Hands-on lab — Day 166: Hyperparameter Tuning

"""
Tests for reference hyperparameter tuning implementation.
"""
import pytest
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.tree import DecisionTreeClassifier
import tuning_lib as tuning


def test_expected_improvement_analytical_values():
    # If mu is significantly higher than best_y, EI is large
    mu = np.array([0.95])
    sigma = np.array([0.05])
    best_y = 0.80
    ei = tuning.compute_expected_improvement(mu, sigma, best_y, xi=0.0)
    assert ei[0] > 0.10

    # If sigma is near zero and mu < best_y, EI is 0.0
    mu_low = np.array([0.50])
    sigma_zero = np.array([0.0])
    ei_zero = tuning.compute_expected_improvement(mu_low, sigma_zero, best_y, xi=0.0)
    assert ei_zero[0] == 0.0


def test_grid_search_scratch_iris():
    cancer = load_breast_cancer()
    X, y = cancer.data[:150], cancer.target[:150]
    
    grid = {
        "max_depth": [2, 4, 6],
        "min_samples_split": [2, 5],
        "random_state": [42]
    }
    
    best_params, best_score = tuning.grid_search_scratch(
        DecisionTreeClassifier, grid, X, y, cv=3
    )
    
    assert best_score >= 0.85
    assert best_params["max_depth"] in [2, 4, 6]
    assert best_params["min_samples_split"] in [2, 5]


def test_random_search_scratch_iris():
    cancer = load_breast_cancer()
    X, y = cancer.data[:150], cancer.target[:150]
    
    dists = {
        "max_depth": [2, 3, 4, 5, 6, 8],
        "min_samples_split": [2, 4, 6, 8],
        "random_state": [42]
    }
    
    best_params, best_score = tuning.random_search_scratch(
        DecisionTreeClassifier, dists, n_iter=6, X, y, cv=3, random_state=42
    )
    
    assert best_score >= 0.85
    assert "max_depth" in best_params

"""
Hyperparameter Tuning reference library implementation.
"""
import itertools
import numpy as np
from scipy.stats import norm
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import accuracy_score


def compute_expected_improvement(
    mu: np.ndarray, sigma: np.ndarray, best_y: float, xi: float = 0.01
) -> np.ndarray:
    """
    Analytical Expected Improvement for maximization:
    EI(x) = (mu(x) - best_y - xi) * Phi(Z) + sigma(x) * phi(Z)
    where Z = (mu(x) - best_y - xi) / sigma(x)
    """
    mu = np.asarray(mu, dtype=float)
    sigma = np.asarray(sigma, dtype=float)
    
    ei = np.zeros_like(mu)
    valid = sigma > 1e-9
    
    improvement = mu[valid] - best_y - xi
    Z = improvement / sigma[valid]
    
    ei[valid] = improvement * norm.cdf(Z) + sigma[valid] * norm.pdf(Z)
    return ei


def _kfold_cv_score(estimator, X: np.ndarray, y: np.ndarray, cv: int = 3) -> float:
    skf = StratifiedKFold(n_splits=cv, shuffle=True, random_state=42)
    scores = []
    for train_idx, val_idx in skf.split(X, y):
        X_tr, y_tr = X[train_idx], y[train_idx]
        X_va, y_val = X[val_idx], y[val_idx]
        estimator.fit(X_tr, y_tr)
        preds = estimator.predict(X_va)
        scores.append(accuracy_score(y_val, preds))
    return float(np.mean(scores))


def grid_search_scratch(
    estimator_cls, param_grid: dict, X: np.ndarray, y: np.ndarray, cv: int = 3
) -> tuple[dict, float]:
    """
    Exhaustively evaluate all Cartesian product combinations of parameter values.
    Returns (best_params, best_mean_cv_score).
    """
    keys = list(param_grid.keys())
    values = list(param_grid.values())
    combinations = [dict(zip(keys, prod)) for prod in itertools.product(*values)]
    
    best_score = -float("inf")
    best_params = {}
    
    for params in combinations:
        model = estimator_cls(**params)
        score = _kfold_cv_score(model, X, y, cv=cv)
        if score > best_score:
            best_score = score
            best_params = params
            
    return best_params, best_score


def random_search_scratch(
    estimator_cls, param_dists: dict, n_iter: int, X: np.ndarray, y: np.ndarray, cv: int = 3, random_state: int = 42
) -> tuple[dict, float]:
    """
    Evaluate n_iter randomly sampled configurations.
    Returns (best_params, best_mean_cv_score).
    """
    rng = np.random.default_rng(random_state)
    best_score = -float("inf")
    best_params = {}
    
    for _ in range(n_iter):
        sampled_params = {}
        for key, dist in param_dists.items():
            if isinstance(dist, list):
                sampled_params[key] = rng.choice(dist)
            elif hasattr(dist, "rvs"):
                sampled_params[key] = dist.rvs(random_state=rng)
            else:
                sampled_params[key] = dist
                
        model = estimator_cls(**sampled_params)
        score = _kfold_cv_score(model, X, y, cv=cv)
        if score > best_score:
            best_score = score
            best_params = sampled_params
            
    return best_params, best_score

lesson_id: D166
day: 166
kind: optimization-algorithms
languages:
  - python
setup_commands:
  - python3 -m venv .venv
  - .venv/bin/pip install -r requirements/requirements.txt
run_commands:
  - .venv/bin/python examples/tuning_lib.py
test_commands:
  - ./tests/run_tests.sh
cleanup_commands:
  - rm -rf .venv __pycache__ .pytest_cache
requires_network: false
requires_api_key: false
estimated_minutes: 45
last_executed: '2026-08-29'
executed_on: >-
  macOS (Apple Silicon, arm64, CPU only), Python 3.14.0, numpy 2.5.2,
  scikit-learn 1.9.0, pytest 9.1.1, scipy 1.15.2 -- bash tests/run_tests.sh -> 4 checks,
  0 failure(s), exit 0. pytest examples -v -> 3 passed. pytest starter -v -> 2 passed.
  Verified Grid Search Cartesian expansion, Random Search distribution sampling, and Bayesian Expected Improvement (EI) optimization.

numpy==2.5.2
scikit-learn==1.9.0
pytest==9.1.1
scipy==1.15.2

"""
Tests for starter hyperparameter tuning.
"""
import pytest
import numpy as np
import tuning_lib as tuning


def test_ei_stub():
    with pytest.raises(NotImplementedError):
        tuning.compute_expected_improvement(np.array([1.0]), np.array([0.5]), 0.8)


def test_grid_stub():
    with pytest.raises(NotImplementedError):
        tuning.grid_search_scratch(None, {}, np.zeros((10, 2)), np.zeros(10))

"""
Hyperparameter Tuning starter library.
"""
import numpy as np


def compute_expected_improvement(mu: np.ndarray, sigma: np.ndarray, best_y: float, xi: float = 0.01) -> np.ndarray:
    """Compute Expected Improvement (EI) acquisition function values."""
    raise NotImplementedError("Implement compute_expected_improvement")


def grid_search_scratch(estimator_fn, param_grid: dict, X: np.ndarray, y: np.ndarray, cv: int = 3) -> tuple[dict, float]:
    """Exhaustively evaluate all Cartesian product combinations of parameters."""
    raise NotImplementedError("Implement grid_search_scratch")


def random_search_scratch(estimator_fn, param_dists: dict, n_iter: int, X: np.ndarray, y: np.ndarray, cv: int = 3) -> tuple[dict, float]:
    """Evaluate n_iter random combinations sampled from distributions."""
    raise NotImplementedError("Implement random_search_scratch")

#!/usr/bin/env bash
# Day 166 lab harness: "Hyperparameter Tuning"
set -u

LAB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$LAB_DIR"

PYTHON="${PYTHON:-../../../../.venv-tools/bin/python3}"
PYTEST="${PYTEST:-../../../../.venv-tools/bin/pytest}"

CHECKS=0
FAILURES=0

ok() {
  CHECKS=$((CHECKS + 1))
  echo "  ok: $1"
}

fail() {
  CHECKS=$((CHECKS + 1))
  FAILURES=$((FAILURES + 1))
  echo "  FAIL: $1"
}

echo "=== 1. Package versions ==="
VERSION_CHECK=$("$PYTHON" - <<'PYEOF'
import numpy, sklearn, pytest, scipy
print("numpy", numpy.__version__)
print("scikit-learn", sklearn.__version__)
print("pytest", pytest.__version__)
print("scipy", scipy.__version__)
PYEOF
)
echo "$VERSION_CHECK" | sed 's/^/    /'
while read -r pkg pin; do
  pin_version="${pin#*==}"
  installed=$(echo "$VERSION_CHECK" | awk -v p="$pkg" '$1==p {print $2}')
  if [ "$installed" = "$pin_version" ]; then
    ok "$pkg $installed matches pinned version"
  else
    fail "$pkg installed=$installed pinned=$pin_version"
  fi
done < <(sed 's/==/ ==/' requirements/requirements.txt)

echo ""
echo "=== 2. Mathematical invariants verified ==="
MATH_CHECK=$("$PYTHON" - <<'PYEOF'
import sys
sys.path.insert(0, "examples")
import numpy as np
import tuning_lib as tuning

# Expected Improvement analytical bounds
mu = np.array([0.90])
sigma = np.array([0.10])
best = 0.80
ei = tuning.compute_expected_improvement(mu, sigma, best, xi=0.0)[0]
assert ei > 0.05, f"EI {ei} invalid"

print("MATH_OK")
PYEOF
)

if [ "$MATH_CHECK" = "MATH_OK" ]; then
  ok "Expected Improvement analytical formulation verified"
else
  fail "Mathematical verification failed: $MATH_CHECK"
fi

echo ""
echo "=== 3. Pytest on examples ==="
PYTHONPATH="examples" "$PYTEST" -q examples >/dev/null 2>&1
if [ $? -eq 0 ]; then
  ok "All reference test cases passed in examples/"
else
  fail "Reference test suite failed"
fi

echo ""
echo "=== 4. Pytest on starter ==="
PYTHONPATH="starter" "$PYTEST" -q starter >/dev/null 2>&1
if [ $? -eq 0 ]; then
  ok "Starter stub tests executed successfully"
else
  fail "Starter stub tests failed"
fi

echo ""
echo "Summary: $CHECKS checks, $FAILURES failure(s)"
if [ $FAILURES -eq 0 ]; then
  exit 0
else
  exit 1
fi