Hyperparameter Optimization

from sklearn.linear_model import LogisticRegression

model = LogisticRegression(C=0.5, penalty="l2", max_iter=2000)   # hyperparameters
model.fit(X_train, y_train)

model.coef_, model.intercept_        # parameters: fitted, not chosen
model.get_params()                    # hyperparameters: chosen, not fitted
model.set_params(C=2.0)               # ...and therefore searchable

# Anything with a trailing underscore was learned. Anything in get_params() was set.
[k for k in vars(model) if k.endswith("_")]

from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import GridSearchCV, StratifiedKFold

grid = {
    "learning_rate": [0.01, 0.05, 0.1],
    "max_depth": [2, 3, 5],
    "subsample": [0.7, 1.0],
    "min_samples_leaf": [1, 5, 20],
}
n_configs = 3 * 3 * 2 * 3                     # 54
print(n_configs, "configs x 5 folds =", n_configs * 5, "fits")

search = GridSearchCV(
    GradientBoostingClassifier(random_state=0), grid,
    cv=StratifiedKFold(5, shuffle=True, random_state=0),
    scoring="average_precision", n_jobs=-1, refit=True,
).fit(X_train, y_train)
search.best_params_, round(search.best_score_, 4)

from scipy.stats import loguniform, randint, uniform
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import RandomizedSearchCV, StratifiedKFold

space = {
    "learning_rate": loguniform(1e-3, 3e-1),   # log scale: 0.001 and 0.01 differ as much
    "max_depth": randint(2, 9),                #             as 0.1 and 1.0
    "subsample": uniform(0.5, 0.5),            # uniform(loc, scale) -> [0.5, 1.0]
    "min_samples_leaf": randint(1, 60),
}
search = RandomizedSearchCV(
    GradientBoostingClassifier(random_state=0), space,
    n_iter=60,                                  # the budget, stated directly
    cv=StratifiedKFold(5, shuffle=True, random_state=0),
    scoring="average_precision", n_jobs=-1, random_state=0,
).fit(X_train, y_train)
search.best_params_

# optuna: define the space in code, so it can be conditional.
import optuna
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import cross_val_score

def objective(trial):
    params = {
        "learning_rate": trial.suggest_float("learning_rate", 1e-3, 3e-1, log=True),
        "max_depth": trial.suggest_int("max_depth", 2, 8),
        "subsample": trial.suggest_float("subsample", 0.5, 1.0),
    }
    model = GradientBoostingClassifier(random_state=0, **params)
    return cross_val_score(model, X_train, y_train, cv=5,
                           scoring="average_precision").mean()

study = optuna.create_study(direction="maximize", sampler=optuna.samplers.TPESampler(seed=0))
study.optimize(objective, n_trials=60)
study.best_params, study.best_value

from sklearn.experimental import enable_halving_search_cv   # noqa: F401
from sklearn.model_selection import HalvingRandomSearchCV
from sklearn.ensemble import RandomForestClassifier
from scipy.stats import randint

search = HalvingRandomSearchCV(
    RandomForestClassifier(random_state=0),
    {"max_depth": randint(2, 30), "min_samples_leaf": randint(1, 40),
     "max_features": ["sqrt", "log2", 0.5]},
    resource="n_samples",      # or "n_estimators" to grow the forest instead
    factor=3,                   # keep 1/3 of candidates, triple their budget
    min_resources="exhaust",
    scoring="average_precision", cv=5, random_state=0, n_jobs=-1,
).fit(X_train, y_train)
search.n_resources_, search.n_candidates_

import numpy as np
from sklearn.model_selection import GridSearchCV, StratifiedKFold

search = GridSearchCV(pipe, grid, cv=StratifiedKFold(5, shuffle=True, random_state=0),
                      scoring="average_precision", return_train_score=True).fit(X_tr, y_tr)

res = search.cv_results_
best = search.best_index_
mean = res["mean_test_score"][best]
std = res["std_test_score"][best]
print(f"best {mean:.4f} +/- {std:.4f} across folds")

# How many candidates are inside one standard error of the winner?
se = std / np.sqrt(5)
tied = int((res["mean_test_score"] > mean - se).sum())
print(f"{tied} configurations are statistically indistinguishable from the best")

import lightgbm as lgb
from sklearn.model_selection import train_test_split

X_fit, X_es, y_fit, y_es = train_test_split(
    X_train, y_train, test_size=0.15, stratify=y_train, random_state=0)

model = lgb.LGBMClassifier(n_estimators=5000, learning_rate=0.02, random_state=0)
model.fit(
    X_fit, y_fit,
    eval_set=[(X_es, y_es)], eval_metric="average_precision",
    callbacks=[lgb.early_stopping(stopping_rounds=100, verbose=False)],
)
model.best_iteration_        # the chosen n_estimators

# scikit-learn's own histogram booster does it internally:
from sklearn.ensemble import HistGradientBoostingClassifier
HistGradientBoostingClassifier(early_stopping=True, validation_fraction=0.15,
                               n_iter_no_change=50, max_iter=5000, random_state=0)

import numpy as np

scores = search.cv_results_["mean_test_score"]
best_so_far = np.maximum.accumulate(scores)          # the curve to plot

noise = search.cv_results_["std_test_score"][search.best_index_] / np.sqrt(5)
reached = best_so_far >= best_so_far[-1] - noise
print(f"within one standard error of the final best after {int(np.argmax(reached)) + 1} trials")

# Optuna keeps the same history, and can prune hopeless trials mid-fit.
# optuna.visualization.plot_optimization_history(study)

import numpy as np

rng = np.random.default_rng(0)
truth = 0.865                             # every candidate is equally good
noise = 0.02                              # fold-to-fold std of the CV estimate

for n_trials in (1, 10, 50, 200):
    reported = [rng.normal(truth, noise, n_trials).max() for _ in range(2000)]
    print(f"{n_trials:>4} trials -> reported {np.mean(reported):.4f} "
          f"(optimism +{np.mean(reported) - truth:.4f})")

# The honest number for the tuned pipeline is an outer split it never touched.
# best_score_ is a selection statistic, not a performance estimate.

import numpy as np
from sklearn.model_selection import GridSearchCV, StratifiedKFold, cross_val_score

inner = StratifiedKFold(5, shuffle=True, random_state=0)
outer = StratifiedKFold(5, shuffle=True, random_state=1)

search = GridSearchCV(pipe, grid, cv=inner, scoring="average_precision", n_jobs=-1)
nested = cross_val_score(search, X, y, cv=outer, scoring="average_precision")

print(f"nested estimate {nested.mean():.4f} +/- {nested.std():.4f}")

# The optimism the inner loop alone would have reported:
search.fit(X, y)
print(f"inner best_score_ {search.best_score_:.4f}  <- do not quote this")

# Ship the pipeline refitted on everything, and report the nested number.
final = search.best_estimator_

from scipy.stats import loguniform, randint, uniform

# Log scale for anything spanning orders of magnitude.
space = {
    "clf__C": loguniform(1e-3, 1e3),
    "clf__gamma": loguniform(1e-5, 1e0),
    "pca__n_components": randint(2, 40),
    "smote__sampling_strategy": uniform(0.1, 0.9),
}

# Conditional structure a flat grid cannot express: only search gamma for rbf.
search_list = [
    {"clf__kernel": ["linear"], "clf__C": [0.01, 0.1, 1, 10, 100]},
    {"clf__kernel": ["rbf"], "clf__C": [0.01, 0.1, 1, 10, 100],
     "clf__gamma": [1e-4, 1e-3, 1e-2, 1e-1]},
]
# GridSearchCV accepts a list of dicts and searches each independently.

# Always include the do-nothing option so you can tell whether a step helped.
{"pca": ["passthrough"], "clf__C": [1.0]}