Imbalanced Learning & Thresholding

import numpy as np
from sklearn.dummy import DummyClassifier
from sklearn.metrics import accuracy_score, balanced_accuracy_score, f1_score
from sklearn.model_selection import train_test_split

y = np.concatenate([np.zeros(9900), np.ones(100)])
X = np.random.default_rng(0).standard_normal((10000, 5))    # pure noise
X_tr, X_te, y_tr, y_te = train_test_split(X, y, stratify=y, test_size=0.3, random_state=0)

dummy = DummyClassifier(strategy="most_frequent").fit(X_tr, y_tr)
pred = dummy.predict(X_te)
accuracy_score(y_te, pred)            # 0.99
balanced_accuracy_score(y_te, pred)   # 0.50 — the honest version
f1_score(y_te, pred, zero_division=0) # 0.00

from collections import Counter
from imblearn.over_sampling import RandomOverSampler, SMOTE
from imblearn.under_sampling import RandomUnderSampler

print(Counter(y_train))                       # {0: 9900, 1: 100}

over = RandomOverSampler(sampling_strategy=0.5, random_state=0)   # minority to 50% of majority
X_over, y_over = over.fit_resample(X_train, y_train)
print(Counter(y_over))                        # {0: 9900, 1: 4950}

under = RandomUnderSampler(sampling_strategy=0.5, random_state=0)
X_under, y_under = under.fit_resample(X_train, y_train)
print(Counter(y_under))                       # {0: 200, 1: 100}

from imblearn.over_sampling import SMOTE, SMOTENC, BorderlineSMOTE
from imblearn.pipeline import Pipeline as ImbPipeline
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler

# k_neighbors must be < the number of minority rows in every fold.
pipe = ImbPipeline([
    ("scale", StandardScaler()),
    ("smote", SMOTE(k_neighbors=5, random_state=0)),
    ("clf", LogisticRegression(max_iter=2000)),
])

# Mixed data: SMOTENC handles categorical columns by taking the neighbour majority.
SMOTENC(categorical_features=[3, 7], random_state=0)

# Synthesise only near the boundary, where the model is actually confused.
BorderlineSMOTE(random_state=0)

from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.utils.class_weight import compute_class_weight
import numpy as np

classes = np.unique(y_train)
compute_class_weight("balanced", classes=classes, y=y_train)   # array([0.505, 50.0])

LogisticRegression(class_weight="balanced", max_iter=2000).fit(X_train, y_train)
LogisticRegression(class_weight={0: 1.0, 1: 20.0}).fit(X_train, y_train)   # a chosen ratio

# Forests: weight per class, or per bootstrap sample.
RandomForestClassifier(class_weight="balanced_subsample", random_state=0)

# Boosting: one scalar, the negative/positive ratio.
# XGBClassifier(scale_pos_weight=9900 / 100)

import numpy as np
from sklearn.metrics import precision_recall_curve

proba = model.predict_proba(X_val)[:, 1]
precision, recall, thresholds = precision_recall_curve(y_val, proba)

# Requirement: recall >= 0.90. Take the most precise threshold that meets it.
ok = recall[:-1] >= 0.90
best = int(np.argmax(precision[:-1] * ok))
print(f"threshold {thresholds[best]:.3f} -> P {precision[best]:.3f}, R {recall[best]:.3f}")

# Or maximise F1 if there is genuinely no asymmetry between the two errors.
f1 = 2 * precision[:-1] * recall[:-1] / np.clip(precision[:-1] + recall[:-1], 1e-12, None)
chosen = float(thresholds[int(np.argmax(f1))])

pred = (model.predict_proba(X_test)[:, 1] >= chosen).astype(int)

from sklearn.metrics import (average_precision_score, roc_auc_score,
                             PrecisionRecallDisplay, RocCurveDisplay)

proba = model.predict_proba(X_test)[:, 1]
roc_auc_score(y_test, proba)             # 0.94 — flattering
average_precision_score(y_test, proba)   # 0.38 — the honest summary

# Baselines to compare against:
#   ROC-AUC of a random model  = 0.50, whatever the prevalence
#   PR-AUC  of a random model  = the prevalence itself, here 0.01
y_test.mean()

# Use average_precision as the CV scorer, not roc_auc, when positives are rare.
# GridSearchCV(..., scoring="average_precision")

import numpy as np

C_FP, C_FN = 5.0, 400.0          # a manual review vs an unblocked fraud
t_star = C_FP / (C_FP + C_FN)     # 0.0123

proba = model.predict_proba(X_val)[:, 1]
grid = np.linspace(0.001, 0.5, 500)
cost = [
    C_FP * ((proba >= t) & (y_val == 0)).sum() + C_FN * ((proba < t) & (y_val == 1)).sum()
    for t in grid
]
best = float(grid[int(np.argmin(cost))])
print(f"analytic {t_star:.4f}, empirical {best:.4f}, cost {min(cost):,.0f}")

# Per-row costs (the transaction amount) instead of one constant:
# weight each false negative by amount[i] rather than by C_FN.

from sklearn.metrics import precision_score, recall_score
from sklearn.model_selection import train_test_split

# Split FIRST, at the real prevalence, and stratify so each side keeps it.
X_tr, X_te, y_tr, y_te = train_test_split(
    X, y, test_size=0.25, stratify=y, random_state=0)
print(y_tr.mean(), y_te.mean())         # both about 0.01

# Resample the training half only.
X_bal, y_bal = SMOTE(random_state=0).fit_resample(X_tr, y_tr)
model.fit(X_bal, y_bal)

# Evaluate on y_te, untouched.
precision_score(y_te, model.predict(X_te))
recall_score(y_te, model.predict(X_te))

from imblearn.over_sampling import SMOTE
from imblearn.pipeline import Pipeline as ImbPipeline
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import StratifiedKFold, cross_val_score
from sklearn.preprocessing import StandardScaler

# WRONG: the synthesised rows are in every fold's validation half too.
# X_bal, y_bal = SMOTE(random_state=0).fit_resample(X, y)
# cross_val_score(model, X_bal, y_bal, cv=5, scoring="average_precision")

# RIGHT: SMOTE is refitted on each fold's training rows, and never applied
# to the fold's validation rows.
pipe = ImbPipeline([
    ("scale", StandardScaler()),
    ("smote", SMOTE(k_neighbors=5, random_state=0)),
    ("clf", RandomForestClassifier(n_estimators=300, random_state=0)),
])
cv = StratifiedKFold(5, shuffle=True, random_state=0)
cross_val_score(pipe, X, y, cv=cv, scoring="average_precision").mean()

import numpy as np
from sklearn.calibration import CalibratedClassifierCV, calibration_curve
from sklearn.metrics import brier_score_loss

raw = model.predict_proba(X_val)[:, 1]
print(f"mean predicted {raw.mean():.3f} vs actual rate {y_val.mean():.3f}")   # 0.33 vs 0.01

cal = CalibratedClassifierCV(model, method="isotonic", cv=5).fit(X_tr, y_tr)
p = cal.predict_proba(X_val)[:, 1]
print(f"mean predicted {p.mean():.3f}")                     # about 0.01

brier_score_loss(y_val, raw), brier_score_loss(y_val, p)    # the second is much lower
frac_pos, mean_pred = calibration_curve(y_val, p, n_bins=10)