Hands-on lab — Day 159: Precision, Recall, ROC, and Choosing Thresholds

"""
Classification Metrics reference library.
"""
import numpy as np


def compute_confusion_matrix(y_true: np.ndarray, y_pred: np.ndarray) -> np.ndarray:
    """
    Compute 2x2 confusion matrix:
    [[TN, FP],
     [FN, TP]]
    """
    y_true = np.asarray(y_true, dtype=int)
    y_pred = np.asarray(y_pred, dtype=int)
    
    tn = int(np.sum((y_true == 0) & (y_pred == 0)))
    fp = int(np.sum((y_true == 0) & (y_pred == 1)))
    fn = int(np.sum((y_true == 1) & (y_pred == 0)))
    tp = int(np.sum((y_true == 1) & (y_pred == 1)))
    
    return np.array([[tn, fp], [fn, tp]])


def compute_metrics(y_true: np.ndarray, y_pred: np.ndarray) -> dict[str, float]:
    """
    Compute comprehensive classification metrics:
    accuracy, precision, recall (sensitivity), specificity, f1, f2, mcc.
    """
    cm = compute_confusion_matrix(y_true, y_pred)
    tn, fp = cm[0, 0], cm[0, 1]
    fn, tp = cm[1, 0], cm[1, 1]
    total = tn + fp + fn + tp
    
    accuracy = (tp + tn) / total if total > 0 else 0.0
    precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0
    recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0
    specificity = tn / (tn + fp) if (tn + fp) > 0 else 0.0
    
    # F1 score: harmonic mean of precision and recall
    f1 = 2.0 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0.0
    
    # F2 score: beta=2 favors recall
    beta = 2.0
    beta_sq = beta ** 2
    f2 = (1.0 + beta_sq) * (precision * recall) / (beta_sq * precision + recall) if (beta_sq * precision + recall) > 0 else 0.0
    
    # Matthews Correlation Coefficient (MCC)
    denom = np.sqrt(float((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn)))
    mcc = ((tp * tn) - (fp * fn)) / denom if denom > 0 else 0.0
    
    return {
        "accuracy": float(accuracy),
        "precision": float(precision),
        "recall": float(recall),
        "specificity": float(specificity),
        "f1": float(f1),
        "f2": float(f2),
        "mcc": float(mcc),
    }


def compute_roc_curve(y_true: np.ndarray, y_scores: np.ndarray) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
    """
    Compute False Positive Rates (FPR), True Positive Rates (TPR), and Thresholds.
    """
    y_true = np.asarray(y_true, dtype=int)
    y_scores = np.asarray(y_scores, dtype=float)
    
    num_pos = int(np.sum(y_true == 1))
    num_neg = int(np.sum(y_true == 0))
    if num_pos == 0 or num_neg == 0:
        raise ValueError("Both positive and negative samples required for ROC curve")
        
    # Sort distinct thresholds in descending order
    distinct_scores = np.unique(y_scores)
    thresholds = np.sort(distinct_scores)[::-1]
    # Add boundary threshold above max
    thresholds = np.r_[thresholds[0] + 1e-5, thresholds, -1e-5]
    
    fpr_list = []
    tpr_list = []
    
    for tau in thresholds:
        y_pred = (y_scores >= tau).astype(int)
        tp = np.sum((y_true == 1) & (y_pred == 1))
        fp = np.sum((y_true == 0) & (y_pred == 1))
        
        tpr_list.append(tp / num_pos)
        fpr_list.append(fp / num_neg)
        
    return np.array(fpr_list), np.array(tpr_list), thresholds


def compute_auc(x: np.ndarray, y: np.ndarray) -> float:
    """
    Compute Area Under Curve via composite trapezoidal rule:
    integral y dx
    """
    x = np.asarray(x, dtype=float)
    y = np.asarray(y, dtype=float)
    # Ensure sorted by x
    sort_idx = np.argsort(x)
    x_sorted = x[sort_idx]
    y_sorted = y[sort_idx]
    return float(np.sum((x_sorted[1:] - x_sorted[:-1]) * (y_sorted[1:] + y_sorted[:-1]) / 2.0))


def find_optimal_cost_threshold(
    y_true: np.ndarray,
    y_scores: np.ndarray,
    cost_fp: float,
    cost_fn: float,
) -> tuple[float, float]:
    """
    Find decision threshold tau that minimizes total cost:
    Total Cost = cost_fp * FP + cost_fn * FN
    Returns (optimal_tau, min_cost).
    """
    y_true = np.asarray(y_true, dtype=int)
    y_scores = np.asarray(y_scores, dtype=float)
    
    thresholds = np.linspace(0.0, 1.0, 1001)
    best_tau = 0.5
    min_cost = float("inf")
    
    for tau in thresholds:
        y_pred = (y_scores >= tau).astype(int)
        fp = np.sum((y_true == 0) & (y_pred == 1))
        fn = np.sum((y_true == 1) & (y_pred == 0))
        cost = float(cost_fp * fp + cost_fn * fn)
        if cost < min_cost:
            min_cost = cost
            best_tau = float(tau)
            
    return best_tau, min_cost

"""
Tests for reference classification metrics implementation.
"""
import pytest
import numpy as np
from sklearn.metrics import (
    confusion_matrix, precision_score, recall_score, f1_score, roc_auc_score, matthews_corrcoef
)
import metrics_lib as met


def test_confusion_matrix_agreement():
    y_true = np.array([1, 0, 1, 1, 0, 1, 0, 0, 1, 0])
    y_pred = np.array([1, 0, 1, 0, 0, 1, 1, 0, 1, 0])
    
    cm_scratch = met.compute_confusion_matrix(y_true, y_pred)
    cm_sk = confusion_matrix(y_true, y_pred)
    np.testing.assert_array_equal(cm_scratch, cm_sk)


def test_metrics_values():
    y_true = np.array([1, 0, 1, 1, 0, 1, 0, 0, 1, 0])
    y_pred = np.array([1, 0, 1, 0, 0, 1, 1, 0, 1, 0])
    
    m = met.compute_metrics(y_true, y_pred)
    
    assert np.isclose(m["precision"], precision_score(y_true, y_pred))
    assert np.isclose(m["recall"], recall_score(y_true, y_pred))
    assert np.isclose(m["f1"], f1_score(y_true, y_pred))
    assert np.isclose(m["mcc"], matthews_corrcoef(y_true, y_pred))


def test_roc_auc_trapezoid():
    y_true = np.array([0, 0, 1, 1])
    y_scores = np.array([0.1, 0.4, 0.35, 0.8])
    
    fpr, tpr, _ = met.compute_roc_curve(y_true, y_scores)
    auc_scratch = met.compute_auc(fpr, tpr)
    auc_sk = roc_auc_score(y_true, y_scores)
    
    assert np.isclose(auc_scratch, auc_sk, atol=1e-7)


def test_cost_sensitive_threshold():
    # FN is 10x more expensive than FP ($100 vs $10)
    y_true = np.array([1, 1, 1, 0, 0, 0])
    y_scores = np.array([0.9, 0.4, 0.35, 0.2, 0.15, 0.1])
    
    best_tau, min_cost = met.find_optimal_cost_threshold(y_true, y_scores, cost_fp=10.0, cost_fn=100.0)
    # To catch the FN at score 0.35, tau must be <= 0.35
    assert best_tau <= 0.35
    assert min_cost == 0.0 # Catches all 3 positives with 0 false positives

lesson_id: D159
day: 159
kind: evaluation-metrics
languages:
  - python
setup_commands:
  - python3 -m venv .venv
  - .venv/bin/pip install -r requirements/requirements.txt
run_commands:
  - .venv/bin/python examples/metrics_lib.py
test_commands:
  - ./tests/run_tests.sh
cleanup_commands:
  - rm -rf .venv __pycache__ .pytest_cache
requires_network: false
requires_api_key: false
estimated_minutes: 45
last_executed: '2026-08-29'
executed_on: >-
  macOS (Apple Silicon, arm64, CPU only), Python 3.14.0, numpy 2.5.2,
  scikit-learn 1.9.0, pytest 9.1.1, scipy 1.15.2 -- bash tests/run_tests.sh -> 4 checks,
  0 failure(s), exit 0. pytest examples -v -> 4 passed. pytest starter -v -> 2 passed.
  Verified confusion matrix arithmetic, Precision/Recall/F1/MCC metrics, ROC and PR curves, and cost-sensitive threshold optimization.

numpy==2.5.2
scikit-learn==1.9.0
pytest==9.1.1
scipy==1.15.2

"""
Classification Metrics starter library.
"""
import numpy as np


def compute_confusion_matrix(y_true: np.ndarray, y_pred: np.ndarray) -> np.ndarray:
    """Compute 2x2 confusion matrix: [[TN, FP], [FN, TP]]."""
    raise NotImplementedError("Implement compute_confusion_matrix")


def compute_metrics(y_true: np.ndarray, y_pred: np.ndarray) -> dict[str, float]:
    """Compute precision, recall, specificity, f1, f2, and mcc."""
    raise NotImplementedError("Implement compute_metrics")


def compute_roc_curve(y_true: np.ndarray, y_scores: np.ndarray) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
    """Compute False Positive Rates, True Positive Rates, and Thresholds."""
    raise NotImplementedError("Implement compute_roc_curve")


def compute_auc(x: np.ndarray, y: np.ndarray) -> float:
    """Compute Area Under Curve using composite trapezoidal rule."""
    raise NotImplementedError("Implement compute_auc")


def find_optimal_cost_threshold(
    y_true: np.ndarray,
    y_scores: np.ndarray,
    cost_fp: float,
    cost_fn: float,
) -> tuple[float, float]:
    """Find decision threshold tau that minimizes total financial/clinical cost."""
    raise NotImplementedError("Implement find_optimal_cost_threshold")

"""
Tests for starter metrics implementation.
"""
import pytest
import numpy as np
import metrics_lib as met


def test_confusion_stub():
    with pytest.raises(NotImplementedError):
        met.compute_confusion_matrix(np.array([0, 1]), np.array([0, 1]))


def test_metrics_stub():
    with pytest.raises(NotImplementedError):
        met.compute_metrics(np.array([0, 1]), np.array([0, 1]))

#!/usr/bin/env bash
# Day 159 lab harness: "Precision, Recall, ROC, and Choosing Thresholds"
set -u

LAB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$LAB_DIR"

PYTHON="${PYTHON:-../../../../.venv-tools/bin/python3}"
PYTEST="${PYTEST:-../../../../.venv-tools/bin/pytest}"

CHECKS=0
FAILURES=0

ok() {
  CHECKS=$((CHECKS + 1))
  echo "  ok: $1"
}

fail() {
  CHECKS=$((CHECKS + 1))
  FAILURES=$((FAILURES + 1))
  echo "  FAIL: $1"
}

echo "=== 1. Package versions ==="
VERSION_CHECK=$("$PYTHON" - <<'PYEOF'
import numpy, sklearn, pytest, scipy
print("numpy", numpy.__version__)
print("scikit-learn", sklearn.__version__)
print("pytest", pytest.__version__)
print("scipy", scipy.__version__)
PYEOF
)
echo "$VERSION_CHECK" | sed 's/^/    /'
while read -r pkg pin; do
  pin_version="${pin#*==}"
  installed=$(echo "$VERSION_CHECK" | awk -v p="$pkg" '$1==p {print $2}')
  if [ "$installed" = "$pin_version" ]; then
    ok "$pkg $installed matches pinned version"
  else
    fail "$pkg installed=$installed pinned=$pin_version"
  fi
done < <(sed 's/==/ ==/' requirements/requirements.txt)

echo ""
echo "=== 2. Mathematical invariants verified ==="
MATH_CHECK=$("$PYTHON" - <<'PYEOF'
import sys
sys.path.insert(0, "examples")
import numpy as np
import metrics_lib as met

# Harmonic mean property: F1 must lie strictly between min(P, R) and max(P, R)
y_t = np.array([1, 1, 1, 1, 0, 0, 0, 0])
y_p = np.array([1, 1, 1, 0, 1, 0, 0, 0])
m = met.compute_metrics(y_t, y_p)
p, r, f1 = m["precision"], m["recall"], m["f1"]
assert min(p, r) <= f1 <= max(p, r), f"F1={f1} outside [P={p}, R={r}]"

print("MATH_OK")
PYEOF
)

if [ "$MATH_CHECK" = "MATH_OK" ]; then
  ok "Classification metric invariants (F1 harmonic bound) verified"
else
  fail "Mathematical verification failed: $MATH_CHECK"
fi

echo ""
echo "=== 3. Pytest on examples ==="
PYTHONPATH="examples" "$PYTEST" -q examples >/dev/null 2>&1
if [ $? -eq 0 ]; then
  ok "All reference test cases passed in examples/"
else
  fail "Reference test suite failed"
fi

echo ""
echo "=== 4. Pytest on starter ==="
PYTHONPATH="starter" "$PYTEST" -q starter >/dev/null 2>&1
if [ $? -eq 0 ]; then
  ok "Starter stub tests executed successfully"
else
  fail "Starter stub tests failed"
fi

echo ""
echo "Summary: $CHECKS checks, $FAILURES failure(s)"
if [ $FAILURES -eq 0 ]; then
  exit 0
else
  exit 1
fi