Nested Cross-Validation — Honest Hyperparameter Evaluation

import numpy as np
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import (GridSearchCV, cross_val_score,
                                      StratifiedKFold, train_test_split)
from sklearn.datasets import load_breast_cancer

cancer = load_breast_cancer()
X, y = cancer.data, cancer.target

# ━━━━━━━━━━━━━━━━━━━━━━━━━━
# NON-NESTED: biased estimate
# ━━━━━━━━━━━━━━━━━━━━━━━━━━
pipe = Pipeline([("sc", StandardScaler()), ("m", GradientBoostingClassifier(random_state=42))])
param_grid = {"m__n_estimators": [100, 200], "m__learning_rate": [0.05, 0.1], "m__max_depth": [3, 4]}

inner_cv  = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
grid_pipe = GridSearchCV(pipe, param_grid, cv=inner_cv, scoring="roc_auc", n_jobs=-1)

# This IS biased because the same CV folds evaluate hyperparameter selection
non_nested_scores = cross_val_score(grid_pipe, X, y, cv=5, scoring="roc_auc")
print(f"Non-nested CV AUC (BIASED):     {non_nested_scores.mean():.4f} +/- {non_nested_scores.std():.4f}")

# ━━━━━━━━━━━━━━━━━━━━━━━━━━
# NESTED: unbiased estimate
# ━━━━━━━━━━━━━━━━━━━━━━━━━━
outer_cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=1)  # DIFFERENT seed

nested_scores = cross_val_score(
    grid_pipe, X, y,
    cv=outer_cv,            # outer loop: honest evaluation
    scoring="roc_auc",
    n_jobs=-1,
)
# inner loop (inside grid_pipe) handles hyperparameter tuning on outer train data
print(f"Nested CV AUC (UNBIASED):       {nested_scores.mean():.4f} +/- {nested_scores.std():.4f}")
print(f"Optimism bias:                  {non_nested_scores.mean() - nested_scores.mean():+.4f}")
print("  (positive = non-nested was overly optimistic)")

# WHEN DOES BIAS MATTER?
print("\nOptimism bias cases:")
bias_cases = {
    "Large param grid":       "More params to search -> more selection pressure -> more bias",
    "Small dataset":          "Fewer data -> CV estimates noisier -> larger bias",
    "Many models compared":   "Comparing 10 algorithms: pick best by CV -> biased by ~0.02-0.05 AUC",
    "Simple model + 3 params":"Bias typically <0.005 -- often negligible",
}
for case, impact in bias_cases.items():
    print(f"  {case:30s}: {impact}")

# WHEN TO USE NESTED CV
print("\nWhen to use nested CV:")
use_cases = [
    "Publishing research results (referee expects it)",
    "Comparing many algorithms fairly (each tuned independently)",
    "Small datasets (<500 samples) where single test split is unreliable",
    "Reporting to business stakeholders as promised accuracy",
]
skip_cases = [
    "Large datasets (>10k) where hold-out test set is sufficient",
    "During model development / exploration (too slow)",
    "When you have a separate, truly held-out test set (as good or better)",
]
for case in use_cases:
    print(f"  USE:  {case}")
for case in skip_cases:
    print(f"  SKIP: {case}")