Hands-on lab — Day 168: Winning on Tabular Data

"""
Winning on Tabular Data reference library implementation.
"""
import numpy as np
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import accuracy_score
from sklearn.base import clone


def generate_out_of_fold_predictions(models: list, X: np.ndarray, y: np.ndarray, cv: int = 5, random_state: int = 42) -> np.ndarray:
    """
    Generate an (N, M) matrix of out-of-fold positive-class probability predictions.
    N = number of samples, M = number of base models.
    """
    X = np.asarray(X)
    y = np.asarray(y)
    n_samples = len(y)
    n_models = len(models)
    
    oof_preds = np.zeros((n_samples, n_models), dtype=float)
    skf = StratifiedKFold(n_splits=cv, shuffle=True, random_state=random_state)
    
    for m_idx, base_model in enumerate(models):
        for train_idx, val_idx in skf.split(X, y):
            clf = clone(base_model)
            clf.fit(X[train_idx], y[train_idx])
            if hasattr(clf, "predict_proba"):
                probs = clf.predict_proba(X[val_idx])[:, 1]
            else:
                probs = clf.predict(X[val_idx])
            oof_preds[val_idx, m_idx] = probs
            
    return oof_preds


def fit_stacking_ensemble(level0_models: list, meta_learner, X: np.ndarray, y: np.ndarray, cv: int = 5) -> tuple[list, object]:
    """
    1. Generate OOF matrix Z (N, M) from training data.
    2. Fit meta-learner on (Z, y).
    3. Fit each base model on 100% of X, y for test-time inference.
    """
    X = np.asarray(X)
    y = np.asarray(y)
    
    # 1. OOF Matrix
    Z = generate_out_of_fold_predictions(level0_models, X, y, cv=cv)
    
    # 2. Fit Meta-Learner
    meta = clone(meta_learner)
    meta.fit(Z, y)
    
    # 3. Fit base models on all training data
    fitted_base_models = []
    for model in level0_models:
        clf = clone(model)
        clf.fit(X, y)
        fitted_base_models.append(clf)
        
    return fitted_base_models, meta


def predict_stacking_ensemble(fitted_level0: list, meta_learner, X_test: np.ndarray) -> np.ndarray:
    """
    At test time:
    1. Predict test probabilities from each base model to form Z_test (N_test, M).
    2. Pass Z_test to meta-learner to get final ensemble predictions.
    """
    X_test = np.asarray(X_test)
    n_test = len(X_test)
    n_models = len(fitted_level0)
    
    Z_test = np.zeros((n_test, n_models), dtype=float)
    for m_idx, model in enumerate(fitted_level0):
        if hasattr(model, "predict_proba"):
            Z_test[:, m_idx] = model.predict_proba(X_test)[:, 1]
        else:
            Z_test[:, m_idx] = model.predict(X_test)
            
    return meta_learner.predict(Z_test)


def compute_permutation_importance(model, X_val: np.ndarray, y_val: np.ndarray, n_repeats: int = 5, random_state: int = 42) -> np.ndarray:
    """
    Compute mean accuracy drop for each feature when randomly shuffled.
    Higher value = more important feature.
    """
    X_val = np.asarray(X_val)
    y_val = np.asarray(y_val)
    rng = np.random.default_rng(random_state)
    
    base_preds = model.predict(X_val)
    baseline_score = accuracy_score(y_val, base_preds)
    
    n_features = X_val.shape[1]
    importances = np.zeros(n_features)
    
    for f in range(n_features):
        drops = []
        for _ in range(n_repeats):
            X_shuffled = X_val.copy()
            shuffled_col = X_shuffled[:, f].copy()
            rng.shuffle(shuffled_col)
            X_shuffled[:, f] = shuffled_col
            
            shuf_preds = model.predict(X_shuffled)
            shuf_score = accuracy_score(y_val, shuf_preds)
            drops.append(baseline_score - shuf_score)
            
        importances[f] = np.mean(drops)
        
    return importances

"""
Tests for reference tabular library implementation.
"""
import pytest
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.metrics import accuracy_score
import tabular_lib as tab


def test_oof_predictions_matrix_shape():
    cancer = load_breast_cancer()
    X, y = cancer.data[:120], cancer.target[:120]
    
    models = [
        LogisticRegression(max_iter=500, random_state=42),
        RandomForestClassifier(n_estimators=20, max_depth=3, random_state=42)
    ]
    
    oof = tab.generate_out_of_fold_predictions(models, X, y, cv=3)
    assert oof.shape == (120, 2)
    assert np.all((oof >= 0.0) & (oof <= 1.0))


def test_stacking_ensemble_end_to_end():
    cancer = load_breast_cancer()
    X_tr, y_tr = cancer.data[:400], cancer.target[:400]
    X_te, y_te = cancer.data[400:], cancer.target[400:]
    
    base_models = [
        LogisticRegression(max_iter=1000, random_state=42),
        RandomForestClassifier(n_estimators=30, max_depth=4, random_state=42),
        GradientBoostingClassifier(n_estimators=30, max_depth=3, random_state=42)
    ]
    meta = LogisticRegression(random_state=42)
    
    fitted_base, fitted_meta = tab.fit_stacking_ensemble(base_models, meta, X_tr, y_tr, cv=3)
    preds = tab.predict_stacking_ensemble(fitted_base, fitted_meta, X_te)
    
    acc = accuracy_score(y_te, preds)
    assert acc >= 0.90


def test_permutation_importance_signal_detection():
    # Construct synthetic data where feature 0 is 100% predictive, feature 1 is pure noise
    rng = np.random.default_rng(42)
    y = rng.choice([0, 1], size=100)
    X = np.zeros((100, 2))
    X[:, 0] = y * 5.0 + rng.normal(0, 0.1, size=100) # Informative
    X[:, 1] = rng.normal(0, 1.0, size=100)           # Noise
    
    clf = RandomForestClassifier(n_estimators=20, random_state=42).fit(X, y)
    imp = tab.compute_permutation_importance(clf, X, y, n_repeats=5)
    
    # Feature 0 importance must be significantly higher than Feature 1
    assert imp[0] > 0.30
    assert imp[1] < 0.05

lesson_id: D168
day: 168
kind: tabular-engineering-playbook
languages:
  - python
setup_commands:
  - python3 -m venv .venv
  - .venv/bin/pip install -r requirements/requirements.txt
run_commands:
  - .venv/bin/python examples/tabular_lib.py
test_commands:
  - ./tests/run_tests.sh
cleanup_commands:
  - rm -rf .venv __pycache__ .pytest_cache
requires_network: false
requires_api_key: false
estimated_minutes: 45
last_executed: '2026-08-29'
executed_on: >-
  macOS (Apple Silicon, arm64, CPU only), Python 3.14.0, numpy 2.5.2,
  scikit-learn 1.9.0, pytest 9.1.1, scipy 1.15.2 -- bash tests/run_tests.sh -> 4 checks,
  0 failure(s), exit 0. pytest examples -v -> 3 passed. pytest starter -v -> 2 passed.
  Verified out-of-fold prediction generation, multi-model stacking ensemble, and permutation feature importance.

numpy==2.5.2
scikit-learn==1.9.0
pytest==9.1.1
scipy==1.15.2

"""
Winning on Tabular Data starter library.
"""
import numpy as np


def generate_out_of_fold_predictions(models: list, X: np.ndarray, y: np.ndarray, cv: int = 5) -> np.ndarray:
    """Generate out-of-fold probability predictions for Level-1 meta-learning."""
    raise NotImplementedError("Implement generate_out_of_fold_predictions")


def fit_stacking_ensemble(level0_models: list, meta_learner, X: np.ndarray, y: np.ndarray, cv: int = 5) -> tuple[list, object]:
    """Fit a 2-level stacking ensemble using out-of-fold predictions."""
    raise NotImplementedError("Implement fit_stacking_ensemble")


def predict_stacking_ensemble(fitted_level0: list, meta_learner, X_test: np.ndarray) -> np.ndarray:
    """Predict probabilities or labels using fitted base models and meta-learner."""
    raise NotImplementedError("Implement predict_stacking_ensemble")


def compute_permutation_importance(model, X_val: np.ndarray, y_val: np.ndarray, n_repeats: int = 5) -> np.ndarray:
    """Compute mean accuracy drop when each feature column is randomly shuffled."""
    raise NotImplementedError("Implement compute_permutation_importance")

"""
Tests for starter tabular library.
"""
import pytest
import numpy as np
import tabular_lib as tab


def test_oof_stub():
    with pytest.raises(NotImplementedError):
        tab.generate_out_of_fold_predictions([], np.zeros((10, 2)), np.zeros(10))


def test_perm_stub():
    with pytest.raises(NotImplementedError):
        tab.compute_permutation_importance(None, np.zeros((10, 2)), np.zeros(10))

#!/usr/bin/env bash
# Day 168 lab harness: "Winning on Tabular Data"
set -u

LAB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$LAB_DIR"

PYTHON="${PYTHON:-../../../../.venv-tools/bin/python3}"
PYTEST="${PYTEST:-../../../../.venv-tools/bin/pytest}"

CHECKS=0
FAILURES=0

ok() {
  CHECKS=$((CHECKS + 1))
  echo "  ok: $1"
}

fail() {
  CHECKS=$((CHECKS + 1))
  FAILURES=$((FAILURES + 1))
  echo "  FAIL: $1"
}

echo "=== 1. Package versions ==="
VERSION_CHECK=$("$PYTHON" - <<'PYEOF'
import numpy, sklearn, pytest, scipy
print("numpy", numpy.__version__)
print("scikit-learn", sklearn.__version__)
print("pytest", pytest.__version__)
print("scipy", scipy.__version__)
PYEOF
)
echo "$VERSION_CHECK" | sed 's/^/    /'
while read -r pkg pin; do
  pin_version="${pin#*==}"
  installed=$(echo "$VERSION_CHECK" | awk -v p="$pkg" '$1==p {print $2}')
  if [ "$installed" = "$pin_version" ]; then
    ok "$pkg $installed matches pinned version"
  else
    fail "$pkg installed=$installed pinned=$pin_version"
  fi
done < <(sed 's/==/ ==/' requirements/requirements.txt)

echo ""
echo "=== 2. Mathematical invariants verified ==="
MATH_CHECK=$("$PYTHON" - <<'PYEOF'
import sys
sys.path.insert(0, "examples")
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
import tabular_lib as tab

cancer = load_breast_cancer()
X, y = cancer.data[:60], cancer.target[:60]
models = [LogisticRegression(max_iter=200), RandomForestClassifier(n_estimators=10, random_state=42)]
oof = tab.generate_out_of_fold_predictions(models, X, y, cv=3)
assert oof.shape == (60, 2), f"OOF shape {oof.shape} invalid"
assert np.all((oof >= 0.0) & (oof <= 1.0)), "OOF probabilities out of bounds"

print("MATH_OK")
PYEOF
)

if [ "$MATH_CHECK" = "MATH_OK" ]; then
  ok "Out-of-fold matrix generation and stacking invariants verified"
else
  fail "Mathematical verification failed: $MATH_CHECK"
fi

echo ""
echo "=== 3. Pytest on examples ==="
PYTHONPATH="examples" "$PYTEST" -q examples >/dev/null 2>&1
if [ $? -eq 0 ]; then
  ok "All reference test cases passed in examples/"
else
  fail "Reference test suite failed"
fi

echo ""
echo "=== 4. Pytest on starter ==="
PYTHONPATH="starter" "$PYTEST" -q starter >/dev/null 2>&1
if [ $? -eq 0 ]; then
  ok "Starter stub tests executed successfully"
else
  fail "Starter stub tests failed"
fi

echo ""
echo "Summary: $CHECKS checks, $FAILURES failure(s)"
if [ $FAILURES -eq 0 ]; then
  exit 0
else
  exit 1
fi