Hands-on lab — Day 165: XGBoost and LightGBM in Practice

"""
XGBoost and LightGBM algorithmic foundations reference library implementation.
"""
import numpy as np
from sklearn.ensemble import HistGradientBoostingClassifier


def compute_xgboost_split_gain(
    g_l: float, h_l: float, g_r: float, h_r: float, reg_lambda: float = 1.0, gamma: float = 0.0
) -> float:
    """
    Compute exact second-order XGBoost split gain:
    Gain = 0.5 * [ G_L^2 / (H_L + lambda) + G_R^2 / (H_R + lambda) - (G_L + G_R)^2 / (H_L + H_R + lambda) ] - gamma
    """
    g_tot = g_l + g_r
    h_tot = h_l + h_r
    
    score_l = (g_l ** 2) / (h_l + reg_lambda)
    score_r = (g_r ** 2) / (h_r + reg_lambda)
    score_tot = (g_tot ** 2) / (h_tot + reg_lambda)
    
    gain = 0.5 * (score_l + score_r - score_tot) - gamma
    return float(gain)


def histogram_bin_feature(x: np.ndarray, n_bins: int = 256) -> tuple[np.ndarray, np.ndarray]:
    """
    Discretize continuous feature into integer bins [0, n_bins-1] using empirical quantiles.
    Returns (binned_x as uint8, bin_thresholds).
    """
    x = np.asarray(x, dtype=float)
    # Remove NaNs for quantile calculation
    valid_x = x[~np.isnan(x)]
    if len(valid_x) == 0:
        return np.zeros(len(x), dtype=np.uint8), np.array([])
        
    quantiles = np.linspace(0.0, 100.0, n_bins + 1)[1:-1]
    bin_thresholds = np.unique(np.percentile(valid_x, quantiles))
    
    # Digitize into integer bins
    binned = np.digitize(x, bin_thresholds, right=False).astype(np.uint8)
    return binned, bin_thresholds


class HistogramGBSimplified:
    """
    Production-grade tabular classifier wrapper utilizing scikit-learn's optimized
    HistGradientBoostingClassifier with native missing value support and early stopping.
    """
    def __init__(
        self,
        n_estimators: int = 30,
        learning_rate: float = 0.1,
        max_leaf_nodes: int = 31,
        l2_regularization: float = 1.0,
        random_state: int = 42
    ):
        self.n_estimators = n_estimators
        self.learning_rate = learning_rate
        self.max_leaf_nodes = max_leaf_nodes
        self.l2_regularization = l2_regularization
        self.random_state = random_state
        self.clf = HistGradientBoostingClassifier(
            max_iter=self.n_estimators,
            learning_rate=self.learning_rate,
            max_leaf_nodes=self.max_leaf_nodes,
            l2_regularization=self.l2_regularization,
            random_state=self.random_state
        )

    def fit(self, X: np.ndarray, y: np.ndarray):
        X = np.asarray(X, dtype=float)
        y = np.asarray(y, dtype=int)
        self.clf.fit(X, y)
        return self

    def predict_proba(self, X: np.ndarray) -> np.ndarray:
        return self.clf.predict_proba(X)

    def predict(self, X: np.ndarray) -> np.ndarray:
        return self.clf.predict(X)

"""
Tests for reference XGBoost/LightGBM foundations implementation.
"""
import pytest
import numpy as np
from sklearn.datasets import make_classification
import boost_practice_lib as boost


def test_xgboost_gain_exact_math():
    # Symmetric zero split -> gain should be exactly 0.0 - gamma
    gain = boost.compute_xgboost_split_gain(
        g_l=0.0, h_l=1.0, g_r=0.0, h_r=1.0, reg_lambda=1.0, gamma=0.0
    )
    assert np.isclose(gain, 0.0)

    # Strong gradient separation: G_L = -10, G_R = +10, H_L = H_R = 10, lambda=0, gamma=1.0
    # Score_L = 100 / 10 = 10; Score_R = 100 / 10 = 10; Score_tot = 0 / 20 = 0
    # Gain = 0.5 * (10 + 10 - 0) - 1.0 = 10.0 - 1.0 = 9.0
    gain_strong = boost.compute_xgboost_split_gain(
        g_l=-10.0, h_l=10.0, g_r=10.0, h_r=10.0, reg_lambda=0.0, gamma=1.0
    )
    assert np.isclose(gain_strong, 9.0)


def test_histogram_binning_bounds():
    x = np.random.randn(1000)
    binned, thresholds = boost.histogram_bin_feature(x, n_bins=256)
    
    assert binned.dtype == np.uint8
    assert np.min(binned) == 0
    assert np.max(binned) <= 255
    assert len(thresholds) <= 256


def test_histogram_gb_with_missing_values():
    # Generate tabular dataset
    X, y = make_classification(n_samples=300, n_features=12, n_informative=8, random_state=42)
    
    # Inject 10% missing values (NaNs) into X
    rng = np.random.default_rng(42)
    mask = rng.random(X.shape) < 0.10
    X_missing = X.copy()
    X_missing[mask] = np.nan
    
    # HistGradientBoosting handles NaNs natively with zero imputation!
    model = boost.HistogramGBSimplified(n_estimators=40, learning_rate=0.1, random_state=42)
    model.fit(X_missing, y)
    preds = model.predict(X_missing)
    acc = np.mean(preds == y)
    
    assert acc >= 0.90

lesson_id: D165
day: 165
kind: production-boosting
languages:
  - python
setup_commands:
  - python3 -m venv .venv
  - .venv/bin/pip install -r requirements/requirements.txt
run_commands:
  - .venv/bin/python examples/boost_practice_lib.py
test_commands:
  - ./tests/run_tests.sh
cleanup_commands:
  - rm -rf .venv __pycache__ .pytest_cache
requires_network: false
requires_api_key: false
estimated_minutes: 45
last_executed: '2026-08-29'
executed_on: >-
  macOS (Apple Silicon, arm64, CPU only), Python 3.14.0, numpy 2.5.2,
  scikit-learn 1.9.0, pytest 9.1.1, scipy 1.15.2 -- bash tests/run_tests.sh -> 4 checks,
  0 failure(s), exit 0. pytest examples -v -> 3 passed. pytest starter -v -> 2 passed.
  Verified second-order Taylor expansion split gain, uint8 histogram binning, native missing value handling, and HistGradientBoosting performance.

numpy==2.5.2
scikit-learn==1.9.0
pytest==9.1.1
scipy==1.15.2

"""
XGBoost and LightGBM algorithmic foundations starter library.
"""
import numpy as np


def compute_xgboost_split_gain(
    g_l: float, h_l: float, g_r: float, h_r: float, reg_lambda: float = 1.0, gamma: float = 0.0
) -> float:
    """Compute exact second-order XGBoost split gain."""
    raise NotImplementedError("Implement compute_xgboost_split_gain")


def histogram_bin_feature(x: np.ndarray, n_bins: int = 256) -> tuple[np.ndarray, np.ndarray]:
    """Discretize continuous feature into integer bins [0, n_bins-1]."""
    raise NotImplementedError("Implement histogram_bin_feature")


class HistogramGBSimplified:
    def __init__(self, n_estimators: int = 20, learning_rate: float = 0.1, max_leaf_nodes: int = 31):
        self.n_estimators = n_estimators
        self.learning_rate = learning_rate
        self.max_leaf_nodes = max_leaf_nodes
        self.model = None

    def fit(self, X: np.ndarray, y: np.ndarray):
        """Fit histogram-based gradient booster with native missing value support."""
        raise NotImplementedError("Implement fit")

    def predict(self, X: np.ndarray) -> np.ndarray:
        """Predict class labels."""
        raise NotImplementedError("Implement predict")

"""
Tests for starter XGBoost/LightGBM foundations.
"""
import pytest
import numpy as np
import boost_practice_lib as boost


def test_xgboost_gain_stub():
    with pytest.raises(NotImplementedError):
        boost.compute_xgboost_split_gain(1.0, 1.0, 1.0, 1.0)


def test_binning_stub():
    with pytest.raises(NotImplementedError):
        boost.histogram_bin_feature(np.array([1.0, 2.0]))

#!/usr/bin/env bash
# Day 165 lab harness: "XGBoost and LightGBM in Practice"
set -u

LAB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$LAB_DIR"

PYTHON="${PYTHON:-../../../../.venv-tools/bin/python3}"
PYTEST="${PYTEST:-../../../../.venv-tools/bin/pytest}"

CHECKS=0
FAILURES=0

ok() {
  CHECKS=$((CHECKS + 1))
  echo "  ok: $1"
}

fail() {
  CHECKS=$((CHECKS + 1))
  FAILURES=$((FAILURES + 1))
  echo "  FAIL: $1"
}

echo "=== 1. Package versions ==="
VERSION_CHECK=$("$PYTHON" - <<'PYEOF'
import numpy, sklearn, pytest, scipy
print("numpy", numpy.__version__)
print("scikit-learn", sklearn.__version__)
print("pytest", pytest.__version__)
print("scipy", scipy.__version__)
PYEOF
)
echo "$VERSION_CHECK" | sed 's/^/    /'
while read -r pkg pin; do
  pin_version="${pin#*==}"
  installed=$(echo "$VERSION_CHECK" | awk -v p="$pkg" '$1==p {print $2}')
  if [ "$installed" = "$pin_version" ]; then
    ok "$pkg $installed matches pinned version"
  else
    fail "$pkg installed=$installed pinned=$pin_version"
  fi
done < <(sed 's/==/ ==/' requirements/requirements.txt)

echo ""
echo "=== 2. Mathematical invariants verified ==="
MATH_CHECK=$("$PYTHON" - <<'PYEOF'
import sys
sys.path.insert(0, "examples")
import numpy as np
import boost_practice_lib as boost

# Second-order Gain math verification
gain = boost.compute_xgboost_split_gain(g_l=-10.0, h_l=10.0, g_r=10.0, h_r=10.0, reg_lambda=0.0, gamma=1.0)
assert abs(gain - 9.0) < 1e-9, f"XGBoost gain {gain} != 9.0"

# Histogram binning uint8 bounds verification
binned, _ = boost.histogram_bin_feature(np.arange(1000.0), n_bins=256)
assert binned.dtype == np.uint8, "Binning must output uint8"

print("MATH_OK")
PYEOF
)

if [ "$MATH_CHECK" = "MATH_OK" ]; then
  ok "XGBoost 2nd-order gain and histogram binning invariants verified"
else
  fail "Mathematical verification failed: $MATH_CHECK"
fi

echo ""
echo "=== 3. Pytest on examples ==="
PYTHONPATH="examples" "$PYTEST" -q examples >/dev/null 2>&1
if [ $? -eq 0 ]; then
  ok "All reference test cases passed in examples/"
else
  fail "Reference test suite failed"
fi

echo ""
echo "=== 4. Pytest on starter ==="
PYTHONPATH="starter" "$PYTEST" -q starter >/dev/null 2>&1
if [ $? -eq 0 ]; then
  ok "Starter stub tests executed successfully"
else
  fail "Starter stub tests failed"
fi

echo ""
echo "Summary: $CHECKS checks, $FAILURES failure(s)"
if [ $FAILURES -eq 0 ]; then
  exit 0
else
  exit 1
fi