Hands-on lab — Day 176: Choosing the Right Metric

import numpy as np
from sklearn.metrics import roc_auc_score, average_precision_score

def compute_classification_metrics(y_true, y_pred, y_prob=None, beta=1.0):
    """
    Compute comprehensive classification metrics including MCC and F-beta.
    """
    y_true = np.asarray(y_true, dtype=int)
    y_pred = np.asarray(y_pred, dtype=int)
    
    tp = np.sum((y_true == 1) & (y_pred == 1))
    tn = np.sum((y_true == 0) & (y_pred == 0))
    fp = np.sum((y_true == 0) & (y_pred == 1))
    fn = np.sum((y_true == 1) & (y_pred == 0))
    
    n = len(y_true)
    accuracy = (tp + tn) / max(n, 1)
    
    precision = tp / max(tp + fp, 1e-9)
    recall = tp / max(tp + fn, 1e-9)
    specificity = tn / max(tn + fp, 1e-9)
    
    b2 = beta ** 2
    f_beta = (1.0 + b2) * (precision * recall) / max((b2 * precision) + recall, 1e-9)
    
    mcc_denom = np.sqrt(float((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn)))
    mcc = float((tp * tn) - (fp * fn)) / max(mcc_denom, 1e-9)
    
    res = {
        "accuracy": float(accuracy),
        "precision": float(precision),
        "recall": float(recall),
        "specificity": float(specificity),
        "f_beta": float(f_beta),
        "mcc": float(mcc),
        "confusion_matrix": {"tp": int(tp), "tn": int(tn), "fp": int(fp), "fn": int(fn)}
    }
    
    if y_prob is not None:
        y_prob = np.asarray(y_prob, dtype=float)
        try:
            res["roc_auc"] = float(roc_auc_score(y_true, y_prob))
        except Exception:
            res["roc_auc"] = 0.5
        try:
            res["pr_auc"] = float(average_precision_score(y_true, y_prob))
        except Exception:
            res["pr_auc"] = float(np.mean(y_true))
            
    return res

def find_optimal_cost_threshold(y_true, y_prob, cost_matrix, thresholds=None):
    """
    Calculate the optimal decision threshold minimizing expected financial cost.
    cost_matrix: dict with keys 'c_tp', 'c_tn', 'c_fp', 'c_fn'
    """
    y_true = np.asarray(y_true, dtype=int)
    y_prob = np.asarray(y_prob, dtype=float)
    if thresholds is None:
        thresholds = np.linspace(0.01, 0.99, 99)
        
    best_cost = float("inf")
    best_threshold = 0.5
    
    for t in thresholds:
        y_pred = (y_prob >= t).astype(int)
        tp = np.sum((y_true == 1) & (y_pred == 1))
        tn = np.sum((y_true == 0) & (y_pred == 0))
        fp = np.sum((y_true == 0) & (y_pred == 1))
        fn = np.sum((y_true == 1) & (y_pred == 0))
        
        total_cost = (
            tp * cost_matrix.get("c_tp", 0.0) +
            tn * cost_matrix.get("c_tn", 0.0) +
            fp * cost_matrix.get("c_fp", 0.0) +
            fn * cost_matrix.get("c_fn", 0.0)
        )
        if total_cost < best_cost:
            best_cost = total_cost
            best_threshold = t
            
    return float(best_threshold), float(best_cost)

def compute_regression_metrics(y_true, y_pred):
    """
    Compute standard, robust, and percentage regression metrics.
    """
    y_true = np.asarray(y_true, dtype=float)
    y_pred = np.asarray(y_pred, dtype=float)
    
    errors = y_true - y_pred
    mse = np.mean(errors ** 2)
    rmse = np.sqrt(mse)
    mae = np.mean(np.abs(errors))
    median_ae = np.median(np.abs(errors))
    
    # MAPE with epsilon guard
    mape = np.mean(np.abs(errors) / np.maximum(np.abs(y_true), 1e-6)) * 100.0
    
    # Symmetric MAPE (sMAPE)
    smape_denom = (np.abs(y_true) + np.abs(y_pred)) / 2.0
    smape = np.mean(np.abs(errors) / np.maximum(smape_denom, 1e-6)) * 100.0
    
    ss_tot = np.sum((y_true - np.mean(y_true)) ** 2)
    ss_res = np.sum(errors ** 2)
    r2 = 1.0 - (ss_res / max(ss_tot, 1e-9))
    
    return {
        "mse": float(mse),
        "rmse": float(rmse),
        "mae": float(mae),
        "median_ae": float(median_ae),
        "mape": float(mape),
        "smape": float(smape),
        "r2": float(r2)
    }

def compute_ranking_ndcg(y_true_relevance, y_score, k=5):
    """
    Compute Normalized Discounted Cumulative Gain at Rank K (NDCG@K).
    """
    y_true_relevance = np.asarray(y_true_relevance, dtype=float)
    y_score = np.asarray(y_score, dtype=float)
    
    order = np.argsort(y_score)[::-1][:k]
    rel_at_k = y_true_relevance[order]
    
    gains = (2.0 ** rel_at_k) - 1.0
    discounts = np.log2(np.arange(len(rel_at_k)) + 2.0)
    dcg = np.sum(gains / discounts)
    
    ideal_order = np.argsort(y_true_relevance)[::-1][:k]
    ideal_rel = y_true_relevance[ideal_order]
    ideal_gains = (2.0 ** ideal_rel) - 1.0
    idcg = np.sum(ideal_gains / discounts)
    
    if idcg <= 0.0:
        return 0.0
    return float(dcg / idcg)

import pytest
import numpy as np
from metric_lib import (
    compute_classification_metrics,
    find_optimal_cost_threshold,
    compute_regression_metrics,
    compute_ranking_ndcg
)

def test_classification_metrics():
    y_true = [1, 1, 0, 0, 1, 0]
    y_pred = [1, 0, 0, 0, 1, 1]
    y_prob = [0.9, 0.4, 0.1, 0.2, 0.8, 0.6]
    
    res = compute_classification_metrics(y_true, y_pred, y_prob, beta=1.0)
    assert res["confusion_matrix"]["tp"] == 2
    assert res["confusion_matrix"]["tn"] == 2
    assert res["confusion_matrix"]["fp"] == 1
    assert res["confusion_matrix"]["fn"] == 1
    assert 0.0 <= res["mcc"] <= 1.0
    assert 0.5 <= res["roc_auc"] <= 1.0

def test_cost_optimal_threshold():
    y_true = np.array([1, 1, 1, 0, 0, 0, 0, 0, 0, 0])
    y_prob = np.array([0.85, 0.75, 0.65, 0.55, 0.45, 0.35, 0.25, 0.15, 0.05, 0.01])
    # Massive cost for False Negatives ($1000) vs False Positives ($10)
    cost_matrix = {"c_tp": 0, "c_tn": 0, "c_fp": 10, "c_fn": 1000}
    
    best_t, best_cost = find_optimal_cost_threshold(y_true, y_prob, cost_matrix)
    # The optimal threshold should be low to avoid expensive False Negatives
    assert best_t <= 0.65
    assert best_cost < 100.0

def test_regression_metrics():
    y_true = [10.0, 20.0, 30.0, 40.0]
    y_pred = [12.0, 18.0, 33.0, 38.0]
    
    res = compute_regression_metrics(y_true, y_pred)
    assert res["mae"] == 2.25
    assert res["r2"] > 0.90
    assert res["mape"] > 0.0
    assert res["smape"] > 0.0

def test_ranking_ndcg():
    # Documents graded 0 to 3
    relevance = [3, 2, 3, 0, 1, 2]
    # Model assigns higher scores to relevant documents
    scores = [0.95, 0.80, 0.70, 0.10, 0.30, 0.60]
    
    ndcg = compute_ranking_ndcg(relevance, scores, k=3)
    assert 0.80 <= ndcg <= 1.0

lesson_id: D176
day: 176
kind: applied-ml-evaluation-metrics
languages:
  - python
setup_commands:
  - python3 -m venv .venv
  - .venv/bin/pip install -r requirements/requirements.txt
run_commands:
  - .venv/bin/python examples/choosing_the_right_metric_lib.py
test_commands:
  - ./tests/run_tests.sh
cleanup_commands:
  - rm -rf .venv __pycache__ .pytest_cache
requires_network: false
requires_api_key: false
estimated_minutes: 45
last_executed: '2026-08-29'
executed_on: >-
  macOS (Apple Silicon, arm64, CPU only), Python 3.14.0, scikit-learn 1.9.0, pytest 9.1.1 -- bash tests/run_tests.sh -> 4 checks, 0 failure(s), exit 0. Verified Day 176 implementation.

numpy>=1.24.0
scipy>=1.10.0
scikit-learn>=1.3.0
pytest>=7.4.0

import numpy as np

def compute_classification_metrics(y_true, y_pred, y_prob=None, beta=1.0):
    # TODO: Implement accuracy, precision, recall, specificity, f_beta, and mcc
    pass

def find_optimal_cost_threshold(y_true, y_prob, cost_matrix, thresholds=None):
    # TODO: Find threshold minimizing total financial cost
    pass

def compute_regression_metrics(y_true, y_pred):
    # TODO: Implement mse, rmse, mae, median_ae, mape, smape, and r2
    pass

def compute_ranking_ndcg(y_true_relevance, y_score, k=5):
    # TODO: Implement DCG, IDCG, and NDCG@K
    pass

import pytest
import numpy as np
from metric_lib import (
    compute_classification_metrics,
    find_optimal_cost_threshold,
    compute_regression_metrics,
    compute_ranking_ndcg
)

def test_classification_metrics():
    y_true = [1, 1, 0, 0, 1, 0]
    y_pred = [1, 0, 0, 0, 1, 1]
    y_prob = [0.9, 0.4, 0.1, 0.2, 0.8, 0.6]
    
    res = compute_classification_metrics(y_true, y_pred, y_prob, beta=1.0)
    assert res["confusion_matrix"]["tp"] == 2
    assert res["confusion_matrix"]["tn"] == 2
    assert res["confusion_matrix"]["fp"] == 1
    assert res["confusion_matrix"]["fn"] == 1
    assert 0.0 <= res["mcc"] <= 1.0
    assert 0.5 <= res["roc_auc"] <= 1.0

def test_cost_optimal_threshold():
    y_true = np.array([1, 1, 1, 0, 0, 0, 0, 0, 0, 0])
    y_prob = np.array([0.85, 0.75, 0.65, 0.55, 0.45, 0.35, 0.25, 0.15, 0.05, 0.01])
    # Massive cost for False Negatives ($1000) vs False Positives ($10)
    cost_matrix = {"c_tp": 0, "c_tn": 0, "c_fp": 10, "c_fn": 1000}
    
    best_t, best_cost = find_optimal_cost_threshold(y_true, y_prob, cost_matrix)
    # The optimal threshold should be low to avoid expensive False Negatives
    assert best_t <= 0.65
    assert best_cost < 100.0

def test_regression_metrics():
    y_true = [10.0, 20.0, 30.0, 40.0]
    y_pred = [12.0, 18.0, 33.0, 38.0]
    
    res = compute_regression_metrics(y_true, y_pred)
    assert res["mae"] == 2.25
    assert res["r2"] > 0.90
    assert res["mape"] > 0.0
    assert res["smape"] > 0.0

def test_ranking_ndcg():
    # Documents graded 0 to 3
    relevance = [3, 2, 3, 0, 1, 2]
    # Model assigns higher scores to relevant documents
    scores = [0.95, 0.80, 0.70, 0.10, 0.30, 0.60]
    
    ndcg = compute_ranking_ndcg(relevance, scores, k=3)
    assert 0.80 <= ndcg <= 1.0

#!/bin/bash
set -e
pytest tests/ -v