Feature Selection

import numpy as np
from sklearn.datasets import make_classification
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score

# 8 informative columns, then a rising pile of pure noise.
X, y = make_classification(n_samples=1500, n_features=8, n_informative=8,
                           n_redundant=0, random_state=0)
rng = np.random.default_rng(0)
for extra in (0, 12, 52):
    Xn = np.hstack([X, rng.standard_normal((len(X), extra))])
    score = cross_val_score(RandomForestClassifier(random_state=0),
                            Xn, y, cv=5, scoring="accuracy").mean()
    print(f"{8 + extra:>3} features -> {score:.3f}")

from sklearn.feature_selection import SelectKBest, f_classif, f_regression

sel = SelectKBest(score_func=f_classif, k=20)
X_small = sel.fit_transform(X_train, y_train)

kept = sel.get_support(indices=True)      # column positions retained
scores = sel.scores_                       # the F-statistic per column

# For a numeric target, swap the score function:
SelectKBest(score_func=f_regression, k=20).fit(X_train, y_train)

import numpy as np
from sklearn.feature_selection import mutual_info_classif, mutual_info_regression, chi2

rng = np.random.default_rng(0)
x = rng.uniform(-3, 3, 2000)
y = x ** 2 + rng.normal(0, 0.4, 2000)      # perfectly predictive, not linear

np.corrcoef(x, y)[0, 1]                     # about 0.0 — correlation sees nothing
mutual_info_regression(x.reshape(-1, 1), y, random_state=0)   # about 0.9 nats

# chi2 needs non-negative features; use it on counts or one-hot columns.
chi2(counts_matrix, y_class)

from sklearn.feature_selection import RFECV
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import StratifiedKFold
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler

rfe = RFECV(
    estimator=LogisticRegression(max_iter=2000),
    step=1,                       # drop one column per round; use 0.1 for 10% per round
    cv=StratifiedKFold(5, shuffle=True, random_state=0),
    scoring="roc_auc",
    min_features_to_select=3,
)
pipe = make_pipeline(StandardScaler(), rfe).fit(X_train, y_train)
rfe.n_features_                   # the size RFECV settled on
rfe.ranking_                       # 1 = kept, 2 = dropped last, and so on

import numpy as np
from sklearn.linear_model import LassoCV, LogisticRegression
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.feature_selection import SelectFromModel

# Regression: LassoCV picks alpha by cross-validation.
lasso = make_pipeline(StandardScaler(), LassoCV(cv=5, random_state=0))
lasso.fit(X_train, y_train)
coef = lasso[-1].coef_
print(f"{np.sum(coef != 0)} of {len(coef)} features kept")

# Classification: an L1-penalised logistic regression, wrapped as a selector.
sel = SelectFromModel(
    LogisticRegression(penalty="l1", solver="liblinear", C=0.1),
    threshold=1e-5,
)
sel.fit(X_train_scaled, y_train)

import numpy as np
from sklearn.ensemble import RandomForestClassifier
from sklearn.inspection import permutation_importance

forest = RandomForestClassifier(n_estimators=400, random_state=0).fit(X_train, y_train)
order = np.argsort(forest.feature_importances_)[::-1]
for i in order[:5]:
    print(f"{feature_names[i]:<20} {forest.feature_importances_[i]:.3f}")

# Measured on held-out data instead of on impurity, so it cannot be gamed by cardinality.
perm = permutation_importance(forest, X_test, y_test, n_repeats=20, random_state=0)
perm.importances_mean

import numpy as np
from statsmodels.stats.outliers_influence import variance_inflation_factor

vif = [variance_inflation_factor(X_scaled, j) for j in range(X_scaled.shape[1])]
for name, v in sorted(zip(feature_names, vif), key=lambda t: -t[1])[:5]:
    print(f"{name:<20} VIF {v:6.1f}")

# The usual remedy: drop one of each near-duplicate pair, or combine them.
corr = np.corrcoef(X_scaled, rowvar=False)
dupes = np.argwhere(np.triu(np.abs(corr), k=1) > 0.95)

import numpy as np
from sklearn.feature_selection import SelectKBest, f_classif
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_val_score
from sklearn.pipeline import Pipeline

rng = np.random.default_rng(0)
X = rng.standard_normal((200, 5000))        # pure noise
y = rng.integers(0, 2, 200)                 # unrelated to X

# WRONG: the filter sees every row, including each fold's validation rows.
sel = SelectKBest(f_classif, k=20).fit(X, y)
print(cross_val_score(LogisticRegression(), sel.transform(X), y, cv=5).mean())

# RIGHT: the filter is refitted inside every fold.
pipe = Pipeline([("sel", SelectKBest(f_classif, k=20)), ("clf", LogisticRegression())])
print(cross_val_score(pipe, X, y, cv=5).mean())    # about 0.5, as it must be

from sklearn.ensemble import GradientBoostingClassifier
from sklearn.feature_selection import SelectKBest, SelectFromModel, VarianceThreshold, f_classif
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler

pipe = Pipeline([
    ("drop_constant", VarianceThreshold(threshold=0.0)),   # structurally useless
    ("scale", StandardScaler()),
    ("filter", SelectKBest(f_classif, k=200)),             # cheap width reduction
    ("embed", SelectFromModel(                              # judged jointly
        GradientBoostingClassifier(random_state=0), threshold="median")),
    ("model", GradientBoostingClassifier(random_state=0)),
])
# Every selector above is refitted inside each fold of this search.