Feature Scaling Impact on Different Algorithms

import numpy as np
import pandas as pd
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier

data = load_breast_cancer()
X, y = data.data, data.target

# EXPERIMENT: performance WITH and WITHOUT scaling
models = {
    "KNN (k=5)":            KNeighborsClassifier(n_neighbors=5),
    "SVM (RBF)":            SVC(kernel="rbf", C=1.0, random_state=42),
    "Logistic Regression":  LogisticRegression(max_iter=1000, random_state=42),
    "Decision Tree":        DecisionTreeClassifier(max_depth=5, random_state=42),
    "Random Forest":        RandomForestClassifier(n_estimators=100, random_state=42),
}

print(f"{'Algorithm':<22} {'Without Scaling':>16} {'With Scaling':>14} {'Gain':>8}")
print("-" * 65)

for name, model in models.items():
    # Without scaling
    raw_scores = cross_val_score(model, X, y, cv=5, scoring="accuracy")

    # With scaling
    pipeline = Pipeline([("scaler", StandardScaler()), ("model", model)])
    scaled_scores = cross_val_score(pipeline, X, y, cv=5, scoring="accuracy")

    gain = scaled_scores.mean() - raw_scores.mean()
    sensitivity = "VERY SENSITIVE" if gain > 0.05 else ("moderate" if gain > 0.01 else "not sensitive")
    print(f"{name:<22} {raw_scores.mean():.4f} +/-{raw_scores.std():.4f}  "
          f"{scaled_scores.mean():.4f} +/-{scaled_scores.std():.4f}  "
          f"{gain:+.4f}  ({sensitivity})")

# GRADIENT DESCENT SPEED -- scaling critical for convergence
print("\nGradient descent convergence (iterations to converge):")
convergence_info = {
    "Unscaled (income: 200k, age: 80)":  "May take 10,000+ iterations or fail to converge",
    "Scaled (all features ~ N(0,1))":     "Converges in 100-500 iterations typically",
    "Why?": "Gradient step size balanced across all features -> smooth optimization landscape",
}
for key, val in convergence_info.items():
    print(f"  {key}: {val}")