Hands-on lab — Day 155: Logistic Regression

"""
Logistic Regression reference library implementation.
"""
import numpy as np


def sigmoid(z: np.ndarray) -> np.ndarray:
    """
    Compute the logistic sigmoid function in a numerically stable manner:
    sigma(z) = 1 / (1 + exp(-z))
    """
    z = np.asarray(z, dtype=float)
    z_clipped = np.clip(z, -500.0, 500.0)
    return 1.0 / (1.0 + np.exp(-z_clipped))


def predict_proba(X: np.ndarray, w: np.ndarray, b: float) -> np.ndarray:
    """
    Compute predicted probabilities P(y=1|X) given weights w and intercept b.
    """
    X = np.asarray(X, dtype=float)
    w = np.asarray(w, dtype=float)
    z = np.dot(X, w) + b
    return sigmoid(z)


def binary_cross_entropy(y_true: np.ndarray, y_prob: np.ndarray, eps: float = 1e-15) -> float:
    """
    Compute binary cross-entropy (log loss):
    L = - (1/N) * sum(y * log(p) + (1 - y) * log(1 - p))
    """
    y_t = np.asarray(y_true, dtype=float)
    y_p = np.asarray(y_prob, dtype=float)
    y_p_clipped = np.clip(y_p, eps, 1.0 - eps)
    loss = -np.mean(y_t * np.log(y_p_clipped) + (1.0 - y_t) * np.log(1.0 - y_p_clipped))
    return float(loss)


def compute_gradients(X: np.ndarray, y_true: np.ndarray, y_prob: np.ndarray) -> tuple[np.ndarray, float]:
    """
    Compute exact gradients of binary cross-entropy:
    grad_w = (1/N) * X^T (y_prob - y_true)
    grad_b = (1/N) * sum(y_prob - y_true)
    """
    X = np.asarray(X, dtype=float)
    y_t = np.asarray(y_true, dtype=float)
    y_p = np.asarray(y_prob, dtype=float)
    N = len(y_t)
    diff = y_p - y_t
    grad_w = (1.0 / N) * np.dot(X.T, diff)
    grad_b = float((1.0 / N) * np.sum(diff))
    return grad_w, grad_b


def fit_logistic_regression(
    X: np.ndarray,
    y: np.ndarray,
    lr: float = 0.1,
    epochs: int = 1000,
    tol: float = 1e-7
) -> tuple[np.ndarray, float, list[float]]:
    """
    Fit logistic regression model using batch gradient descent.
    """
    X = np.asarray(X, dtype=float)
    y = np.asarray(y, dtype=float)
    N, D = X.shape
    w = np.zeros(D, dtype=float)
    b = 0.0
    history = []

    for epoch in range(epochs):
        probs = predict_proba(X, w, b)
        loss = binary_cross_entropy(y, probs)
        history.append(loss)

        grad_w, grad_b = compute_gradients(X, y, probs)
        w -= lr * grad_w
        b -= lr * grad_b

        if np.linalg.norm(grad_w) < tol and abs(grad_b) < tol:
            break

    return w, b, history


def predict_classes(X: np.ndarray, w: np.ndarray, b: float, threshold: float = 0.5) -> np.ndarray:
    """
    Predict binary class labels (0 or 1) based on decision threshold tau.
    """
    probs = predict_proba(X, w, b)
    return (probs >= threshold).astype(int)

"""
Tests for reference logistic regression implementation.
"""
import pytest
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
import logistic_lib as logreg


def test_sigmoid_properties():
    assert abs(logreg.sigmoid(np.array([0.0]))[0] - 0.5) < 1e-9
    assert logreg.sigmoid(np.array([100.0]))[0] > 0.9999
    assert logreg.sigmoid(np.array([-100.0]))[0] < 0.0001
    
    z = np.array([-3.5, -1.0, 0.0, 1.2, 4.0])
    np.testing.assert_allclose(logreg.sigmoid(-z), 1.0 - logreg.sigmoid(z), atol=1e-9)


def test_binary_cross_entropy_extremes():
    y_true = np.array([1.0, 0.0])
    y_perfect = np.array([0.99999, 0.00001])
    assert logreg.binary_cross_entropy(y_true, y_perfect) < 1e-4

    y_mid = np.array([0.5, 0.5])
    assert abs(logreg.binary_cross_entropy(y_true, y_mid) - np.log(2.0)) < 1e-4


def test_gradient_descent_convergence():
    rng = np.random.default_rng(42)
    X = rng.normal(size=(100, 2))
    y = ((X[:, 0] * 2.0 - X[:, 1] * 1.5 + 0.5) > 0).astype(float)

    w, b, history = logreg.fit_logistic_regression(X, y, lr=0.5, epochs=300)
    assert history[-1] < history[0]
    assert history[-1] < 0.25

    preds = logreg.predict_classes(X, w, b)
    acc = np.mean(preds == y)
    assert acc >= 0.90


def test_breast_cancer_benchmark():
    data = load_breast_cancer()
    X = data.data
    y = data.target
    scaler = StandardScaler()
    X_scaled = scaler.fit_transform(X)

    w, b, history = logreg.fit_logistic_regression(X_scaled, y, lr=0.2, epochs=1000)
    scratch_preds = logreg.predict_classes(X_scaled, w, b)
    scratch_acc = np.mean(scratch_preds == y)

    sk_model = LogisticRegression(C=1e9, solver="lbfgs", max_iter=1000)
    sk_model.fit(X_scaled, y)
    sk_preds = sk_model.predict(X_scaled)
    sk_acc = np.mean(sk_preds == y)

    assert scratch_acc >= 0.95
    assert sk_acc >= 0.95
    assert abs(scratch_acc - sk_acc) < 0.03

lesson_id: D155
day: 155
kind: classification-fundamentals
languages:
  - python
setup_commands:
  - python3 -m venv .venv
  - .venv/bin/pip install -r requirements/requirements.txt
run_commands:
  - .venv/bin/python examples/logistic_lib.py
test_commands:
  - ./tests/run_tests.sh
cleanup_commands:
  - rm -rf .venv __pycache__ .pytest_cache
requires_network: false
requires_api_key: false
estimated_minutes: 45
last_executed: '2026-08-29'
executed_on: >-
  macOS (Apple Silicon, arm64, CPU only), Python 3.14.0, numpy 2.5.2,
  scikit-learn 1.9.0, pytest 9.1.1, scipy 1.15.2 -- bash tests/run_tests.sh -> 4 checks,
  0 failure(s), exit 0. pytest examples -v -> 4 passed. pytest starter -v -> 3 passed.
  Numerical verification reproduced exact sigmoid midpoint, symmetry, and log-loss at p=0.5.

numpy==2.5.2
scikit-learn==1.9.0
pytest==9.1.1
scipy==1.15.2

"""
Logistic Regression library starter implementation.
"""
import numpy as np


def sigmoid(z: np.ndarray) -> np.ndarray:
    """Compute the logistic sigmoid function."""
    raise NotImplementedError("Implement sigmoid")


def predict_proba(X: np.ndarray, w: np.ndarray, b: float) -> np.ndarray:
    """Compute predicted probabilities for class 1."""
    raise NotImplementedError("Implement predict_proba")


def binary_cross_entropy(y_true: np.ndarray, y_prob: np.ndarray, eps: float = 1e-15) -> float:
    """Compute binary cross-entropy (log loss)."""
    raise NotImplementedError("Implement binary_cross_entropy")


def compute_gradients(X: np.ndarray, y_true: np.ndarray, y_prob: np.ndarray) -> tuple[np.ndarray, float]:
    """Compute gradients of loss with respect to w and b."""
    raise NotImplementedError("Implement compute_gradients")


def fit_logistic_regression(
    X: np.ndarray,
    y: np.ndarray,
    lr: float = 0.1,
    epochs: int = 1000,
    tol: float = 1e-6
) -> tuple[np.ndarray, float, list[float]]:
    """Fit logistic regression model using batch gradient descent."""
    raise NotImplementedError("Implement fit_logistic_regression")


def predict_classes(X: np.ndarray, w: np.ndarray, b: float, threshold: float = 0.5) -> np.ndarray:
    """Predict binary class labels (0 or 1) based on probability threshold."""
    raise NotImplementedError("Implement predict_classes")

"""
Tests for starter logistic regression implementation.
"""
import pytest
import numpy as np
import logistic_lib as logreg


def test_sigmoid_stub():
    with pytest.raises(NotImplementedError):
        logreg.sigmoid(np.array([0.0]))


def test_predict_proba_stub():
    with pytest.raises(NotImplementedError):
        logreg.predict_proba(np.zeros((2, 2)), np.zeros(2), 0.0)


def test_binary_cross_entropy_stub():
    with pytest.raises(NotImplementedError):
        logreg.binary_cross_entropy(np.array([1, 0]), np.array([0.8, 0.2]))

#!/usr/bin/env bash
# Day 155 lab harness: "Logistic Regression"
set -u

LAB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$LAB_DIR"

PYTHON="${PYTHON:-../../../../.venv-tools/bin/python3}"
PYTEST="${PYTEST:-../../../../.venv-tools/bin/pytest}"

CHECKS=0
FAILURES=0

ok() {
  CHECKS=$((CHECKS + 1))
  echo "  ok: $1"
}

fail() {
  CHECKS=$((CHECKS + 1))
  FAILURES=$((FAILURES + 1))
  echo "  FAIL: $1"
}

echo "=== 1. Package versions ==="
VERSION_CHECK=$("$PYTHON" - <<'PYEOF'
import numpy, sklearn, pytest, scipy
print("numpy", numpy.__version__)
print("scikit-learn", sklearn.__version__)
print("pytest", pytest.__version__)
print("scipy", scipy.__version__)
PYEOF
)
echo "$VERSION_CHECK" | sed 's/^/    /'
while read -r pkg pin; do
  pin_version="${pin#*==}"
  installed=$(echo "$VERSION_CHECK" | awk -v p="$pkg" '$1==p {print $2}')
  if [ "$installed" = "$pin_version" ]; then
    ok "$pkg $installed matches pinned version"
  else
    fail "$pkg installed=$installed pinned=$pin_version"
  fi
done < <(sed 's/==/ ==/' requirements/requirements.txt)

echo ""
echo "=== 2. Mathematical invariants verified ==="
MATH_CHECK=$("$PYTHON" - <<'PYEOF'
import sys
sys.path.insert(0, "examples")
import numpy as np
import logistic_lib as logreg

s0 = logreg.sigmoid(np.array([0.0]))[0]
assert abs(s0 - 0.5) < 1e-9, f"s(0)={s0}"

z = np.array([-2.5, 0.0, 2.5])
s_pos = logreg.sigmoid(z)
s_neg = logreg.sigmoid(-z)
assert np.allclose(s_neg, 1.0 - s_pos, atol=1e-9), "sigmoid symmetry violated"

y = np.array([1.0, 0.0])
p = np.array([0.5, 0.5])
loss_mid = logreg.binary_cross_entropy(y, p)
assert abs(loss_mid - np.log(2.0)) < 1e-9, f"loss_mid={loss_mid}"

X = np.eye(2)
y_match = np.array([1.0, 0.0])
p_match = np.array([1.0, 0.0])
gw, gb = logreg.compute_gradients(X, y_match, p_match)
assert np.allclose(gw, 0.0) and abs(gb) < 1e-9, "gradients not zero on exact match"

print("MATH_OK")
PYEOF
)

if [ "$MATH_CHECK" = "MATH_OK" ]; then
  ok "All mathematical identities verified exactly"
else
  fail "Mathematical verification failed: $MATH_CHECK"
fi

echo ""
echo "=== 3. Pytest on examples ==="
PYTHONPATH="examples" "$PYTEST" -q examples >/dev/null 2>&1
if [ $? -eq 0 ]; then
  ok "All 4 reference test cases passed in examples/"
else
  fail "Reference test suite failed"
fi

echo ""
echo "=== 4. Pytest on starter ==="
PYTHONPATH="starter" "$PYTEST" -q starter >/dev/null 2>&1
if [ $? -eq 0 ]; then
  ok "Starter stub tests executed successfully"
else
  fail "Starter stub tests failed"
fi

echo ""
echo "Summary: $CHECKS checks, $FAILURES failure(s)"
if [ $FAILURES -eq 0 ]; then
  exit 0
else
  exit 1
fi