Hands-on lab — Day 152: Regression Metrics

"""Regression metrics, measured: what each one reports, and what it hides.

Day 149 drew the line once: a loss is what you optimise, a metric is what
you report, and they need not be the same function. This module measures
the reporting side for regression -- RMSE, MAE, MAPE, R-squared and
adjusted R-squared -- and the traps in each, on the diabetes dataset
(``sklearn.datasets.load_diabetes``) and on constructed data where a
property needs to be exact rather than merely typical.

Everything here is deterministic given a seed.
"""

from __future__ import annotations

import numpy as np

from sklearn.datasets import load_diabetes
from sklearn.linear_model import LinearRegression
from sklearn.metrics import (
    mean_absolute_error,
    mean_absolute_percentage_error,
    mean_squared_error,
    r2_score,
)
from sklearn.model_selection import train_test_split


# --------------------------------------------------------------------------
# 0. The dataset this module shares
# --------------------------------------------------------------------------


def diabetes_split(scaled: bool = True, seed: int = 0):
    """The 75/25 split every measurement below is built on."""
    X, y = load_diabetes(return_X_y=True, scaled=scaled)
    return train_test_split(X, y, test_size=0.25, random_state=seed)


def rmse(y_true, y_pred) -> float:
    return float(np.sqrt(mean_squared_error(y_true, y_pred)))


def mae(y_true, y_pred) -> float:
    return float(mean_absolute_error(y_true, y_pred))


# --------------------------------------------------------------------------
# 1. Train R-squared is not a quality measure: the noise-column climb
# --------------------------------------------------------------------------


def adjusted_r2(r2: float, n: int, p: int) -> float:
    """R-squared, penalised for the number of predictors used to get it."""
    return 1.0 - (1.0 - r2) * (n - 1) / (n - p - 1)


def noise_column_r2_curve(noise_counts=(0, 1, 5, 20, 100), seed: int = 0):
    """Train R2 and adjusted R2 as pure-noise columns are added.

    Every added column is independent standard-normal noise with no
    relationship whatsoever to the target. Returns rows of
    ``(n_noise, n_rows, n_predictors, train_r2, adjusted_r2)``.
    """
    X_train, _X_test, y_train, _y_test = diabetes_split(seed=0)
    rng = np.random.default_rng(seed)
    rows = []
    for n_noise in noise_counts:
        if n_noise == 0:
            X_aug = X_train
        else:
            noise = rng.normal(size=(X_train.shape[0], n_noise))
            X_aug = np.hstack([X_train, noise])
        model = LinearRegression().fit(X_aug, y_train)
        r2 = r2_score(y_train, model.predict(X_aug))
        n_rows, n_predictors = X_aug.shape
        rows.append((n_noise, n_rows, n_predictors, round(r2, 4), round(adjusted_r2(r2, n_rows, n_predictors), 4)))
    return rows


def full_model_test_r2(seed: int = 0) -> float:
    """The test R2 of the ordinary ten-feature model -- the honest number."""
    X_train, X_test, y_train, y_test = diabetes_split(seed=seed)
    model = LinearRegression().fit(X_train, y_train)
    return round(float(r2_score(y_test, model.predict(X_test))), 4)


# --------------------------------------------------------------------------
# 2. R-squared is not bounded below by zero
# --------------------------------------------------------------------------


def constant_mean_test_r2(seed: int = 0) -> float:
    """R2 of predicting the train mean for every test row."""
    _X_train, _X_test, y_train, y_test = diabetes_split(seed=seed)
    prediction = np.full_like(y_test, y_train.mean(), dtype=float)
    return round(float(r2_score(y_test, prediction)), 4)


def bad_predictor_test_r2(seed: int = 0) -> float:
    """R2 of a deliberately bad predictor: zero, always."""
    _X_train, _X_test, _y_train, y_test = diabetes_split(seed=seed)
    prediction = np.zeros_like(y_test, dtype=float)
    return round(float(r2_score(y_test, prediction)), 4)


# --------------------------------------------------------------------------
# 3. RMSE versus MAE under an outlier
# --------------------------------------------------------------------------


def rmse_mae_outlier_shift(seed: int = 1, n: int = 50, shift: float = 200.0):
    """Same predictions, one target moved far away. Which metric moves more?

    Returns ``(rmse_before, mae_before, rmse_after, mae_after)``.
    """
    rng = np.random.default_rng(seed)
    y_true = rng.normal(100.0, 10.0, size=n)
    y_pred = y_true + rng.normal(0.0, 3.0, size=n)
    before = (rmse(y_true, y_pred), mae(y_true, y_pred))

    y_true_shifted = y_true.copy()
    y_true_shifted[0] += shift
    after = (rmse(y_true_shifted, y_pred), mae(y_true_shifted, y_pred))
    return round(before[0], 4), round(before[1], 4), round(after[0], 4), round(after[1], 4)


# --------------------------------------------------------------------------
# 4. MAPE breaking: zero targets, near-zero targets, and structural asymmetry
# --------------------------------------------------------------------------


def mape_at_zero_target() -> float:
    """MAPE where one true value is exactly zero.

    Not undefined in the mathematical sense of raising -- scikit-learn
    floors the denominator at machine epsilon, so this returns a huge,
    silently wrong number rather than an error or a warning.
    """
    y_true = np.array([10.0, 20.0, 0.0, 30.0])
    y_pred = np.array([12.0, 18.0, 5.0, 28.0])
    return float(mean_absolute_percentage_error(y_true, y_pred))


def mape_near_zero_target():
    """MAPE explodes on a true value close to (but not) zero.

    Returns ``(mape, mae)`` on the same three rows, so the contrast between
    a metric that explodes and one that does not is visible in one call.
    """
    y_true = np.array([100.0, 100.0, 0.5])
    y_pred = np.array([105.0, 95.0, 5.5])
    return (
        round(float(mean_absolute_percentage_error(y_true, y_pred)), 4),
        round(float(mean_absolute_error(y_true, y_pred)), 4),
    )


def mape_asymmetry_bound(true_value: float = 100.0):
    """MAPE's structural asymmetry: bounded under, unbounded over.

    A model that under-predicts every row can be wrong by at most 100
    percent (predict zero, and the error cannot exceed the true value).
    A model that over-predicts has no such ceiling. Returns
    ``(max_under_prediction_mape, ten_times_over_prediction_mape)``.
    """
    y_true = np.full(5, true_value)
    max_under = np.zeros(5)  # the worst possible under-prediction: always zero
    ten_x_over = y_true * 11.0  # predicting eleven times the true value
    return (
        round(float(mean_absolute_percentage_error(y_true, max_under)), 4),
        round(float(mean_absolute_percentage_error(y_true, ten_x_over)), 4),
    )


# --------------------------------------------------------------------------
# 5. A metric ranking inversion: RMSE and MAE prefer different models
# --------------------------------------------------------------------------


def ranking_inversion_models(seed: int = 2, n: int = 100):
    """Two models scored on the same targets: one wins on RMSE, one on MAE.

    Model A makes many small, consistent errors. Model B is right almost
    everywhere but wrong by a lot on a few rows. Returns
    ``(rmse_a, mae_a, rmse_b, mae_b)``.
    """
    rng = np.random.default_rng(seed)
    y_true = rng.normal(50.0, 5.0, size=n)

    errors_a = rng.normal(0.0, 2.0, size=n)
    pred_a = y_true + errors_a

    errors_b = np.zeros(n)
    big_idx = rng.choice(n, size=5, replace=False)
    errors_b[big_idx] = rng.normal(0.0, 15.0, size=5)
    pred_b = y_true + errors_b

    return (
        round(rmse(y_true, pred_a), 4),
        round(mae(y_true, pred_a), 4),
        round(rmse(y_true, pred_b), 4),
        round(mae(y_true, pred_b), 4),
    )


# --------------------------------------------------------------------------
# 6. Units: RMSE and MAE carry the units of the target
# --------------------------------------------------------------------------


def raw_and_scaled_metrics(seed: int = 0):
    """RMSE, MAE and R2 fit on raw-unit features versus standardised ones.

    ``load_diabetes(scaled=False)`` returns the ten features in their
    original units -- age in years, bmi as bmi, blood pressure as measured
    -- while the target is the same disease-progression score either way.
    Ordinary least squares is invariant to a per-column affine rescaling of
    its inputs, so this returns identical numbers under both, which is
    itself the thing worth confirming rather than assuming.
    """
    results = {}
    for label, scaled in (("scaled", True), ("raw", False)):
        X_train, X_test, y_train, y_test = diabetes_split(scaled=scaled, seed=seed)
        model = LinearRegression().fit(X_train, y_train)
        pred = model.predict(X_test)
        results[label] = (round(rmse(y_test, pred), 4), round(mae(y_test, pred), 4), round(float(r2_score(y_test, pred)), 4))
    return results


# --------------------------------------------------------------------------
# 7. r2_score: agreement with .score, and the argument-order bug
# --------------------------------------------------------------------------


def r2_score_vs_model_score(seed: int = 0):
    """r2_score(y_true, y_pred) against LinearRegression.score(X, y)."""
    X_train, X_test, y_train, y_test = diabetes_split(seed=seed)
    model = LinearRegression().fit(X_train, y_train)
    pred = model.predict(X_test)
    return round(float(r2_score(y_test, pred)), 6), round(float(model.score(X_test, y_test)), 6)


def r2_score_argument_order(seed: int = 0):
    """r2_score is NOT symmetric in its two arguments: swap them and it changes.

    Returns ``(correct_order, swapped_order)``.
    """
    X_train, X_test, y_train, y_test = diabetes_split(seed=seed)
    model = LinearRegression().fit(X_train, y_train)
    pred = model.predict(X_test)
    return round(float(r2_score(y_test, pred)), 6), round(float(r2_score(pred, y_test)), 6)

#!/usr/bin/env python3
"""Print every measured pair in this lab as one table.

The harness compares this output byte for byte against
expected-output/measured-values.txt, so the report is not a convenience:
it is how the lab notices that a number in the lesson has gone stale.
"""

import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))

import regression_metrics_lib as m  # noqa: E402


def rule(title: str) -> None:
    print()
    print(title)
    print("-" * len(title))


def main() -> None:
    print("Day 152 -- regression metrics, measured")
    print("=" * 41)

    rule("1. Train R2 climbs on pure-noise columns")
    print("  n_noise   n_rows   n_predictors   train_r2   adjusted_r2")
    for n_noise, n_rows, n_predictors, r2, adj in m.noise_column_r2_curve():
        print(f"  {n_noise:7d}   {n_rows:6d}   {n_predictors:12d}   {r2:.4f}     {adj:.4f}")
    print("  every added column is independent noise, unrelated to the target")

    rule("2. R2 is not bounded below by zero")
    print(f"  full 10-feature model, test R2       : {m.full_model_test_r2():.4f}")
    print(f"  constant-mean predictor, test R2     : {m.constant_mean_test_r2():.4f}")
    print(f"  deliberately bad (all-zeros), test R2: {m.bad_predictor_test_r2():.4f}")

    rule("3. RMSE versus MAE under one outlier")
    rmse_before, mae_before, rmse_after, mae_after = m.rmse_mae_outlier_shift()
    print(f"  before: RMSE {rmse_before:.4f}  MAE {mae_before:.4f}")
    print(f"  after : RMSE {rmse_after:.4f}  MAE {mae_after:.4f}  (one target moved +200)")
    print(f"  RMSE moved by a factor of {rmse_after / rmse_before:.2f}")
    print(f"  MAE moved by a factor of {mae_after / mae_before:.2f}")

    rule("4. MAPE breaking")
    print(f"  MAPE with one true value exactly zero      : {m.mape_at_zero_target():.4e}")
    mape_nz, mae_nz = m.mape_near_zero_target()
    print(f"  MAPE with a true value of 0.5 in the mix   : {mape_nz:.4f}  (MAE on the same rows: {mae_nz:.4f})")
    max_under, ten_x_over = m.mape_asymmetry_bound()
    print(f"  MAPE of the worst possible under-prediction: {max_under:.4f}")
    print(f"  MAPE of an eleven-times over-prediction    : {ten_x_over:.4f}")
    print("  under-prediction is capped at 1.0; over-prediction is not")

    rule("5. A metric ranking inversion")
    rmse_a, mae_a, rmse_b, mae_b = m.ranking_inversion_models()
    print(f"  Model A (many small errors)      : RMSE {rmse_a:.4f}  MAE {mae_a:.4f}")
    print(f"  Model B (few large errors)       : RMSE {rmse_b:.4f}  MAE {mae_b:.4f}")
    print(f"  RMSE prefers: {'A' if rmse_a < rmse_b else 'B'}")
    print(f"  MAE prefers : {'A' if mae_a < mae_b else 'B'}")

    rule("6. RMSE and MAE carry the target's units")
    results = m.raw_and_scaled_metrics()
    for label, (r, a, r2) in results.items():
        print(f"  {label:7s} features: RMSE {r:.4f}  MAE {a:.4f}  R2 {r2:.4f}")
    print("  identical either way -- ordinary least squares is invariant to")
    print("  a per-column affine rescaling of its inputs")

    rule("7. r2_score: agreement, and the argument-order bug")
    from_metric, from_model = m.r2_score_vs_model_score()
    print(f"  r2_score(y_test, pred)         : {from_metric:.6f}")
    print(f"  model.score(X_test, y_test)    : {from_model:.6f}")
    correct_order, swapped_order = m.r2_score_argument_order()
    print(f"  r2_score(y_test, pred)  correct : {correct_order:.6f}")
    print(f"  r2_score(pred, y_test) swapped : {swapped_order:.6f}")


if __name__ == "__main__":
    main()

"""The reference solutions: what each regression metric reports, and hides.

Every number here was captured from a real run of this file on the
authoring machine. If a number changes, the claim in the lesson is wrong
and one of the two must be fixed.
"""

import numpy as np

import regression_metrics_lib as m


# --- 1. Train R2 is not a quality measure ---------------------------------


def test_01_train_r2_climbs_on_pure_noise_columns():
    rows = m.noise_column_r2_curve()
    assert rows == [
        (0, 331, 10, 0.5554, 0.5415),
        (1, 331, 11, 0.5555, 0.5402),
        (5, 331, 15, 0.5648, 0.5441),
        (20, 331, 30, 0.5754, 0.5329),
        (100, 331, 110, 0.7403, 0.6104),
    ]
    train_r2 = [row[3] for row in rows]
    # Every added column is pure noise, independent of the target, and
    # yet the training R2 climbs anyway -- by more as more columns are
    # added, because more predictors can only help a training-set fit.
    assert all(a < b for a, b in zip(train_r2, train_r2[1:]))
    assert train_r2[-1] - train_r2[0] > 0.18


def test_01b_adjusted_r2_corrects_the_climb_then_breaks_down_itself():
    rows = m.noise_column_r2_curve()
    by_noise = {row[0]: row for row in rows}
    baseline_adj = by_noise[0][4]
    # At a modest number of noise columns, adjusted R2 does its job: it
    # falls below the no-noise baseline, correctly reporting that these
    # columns did not earn their place.
    assert by_noise[20][4] < baseline_adj
    # But at p=110 predictors on n=331 rows -- a third of the sample size
    # spent on predictors -- the penalty term itself becomes unstable, and
    # adjusted R2 climbs back ABOVE the baseline even though every one of
    # those 100 extra columns is still pure noise. The correction is not
    # a cure; it has its own failure mode.
    assert by_noise[100][4] > baseline_adj
    assert by_noise[100][4] > by_noise[20][4]


# --- 2. R2 is not bounded below by zero ------------------------------------


def test_02_the_full_model_beats_a_constant_mean_predictor_on_test():
    full = m.full_model_test_r2()
    constant = m.constant_mean_test_r2()
    assert full == 0.3594
    # A constant-mean predictor scores R2 essentially exactly zero on a
    # fresh test set, by construction: R2 is defined relative to that
    # exact predictor, so this is the thing R2 compares against, not an
    # incidental fact about this dataset.
    assert abs(constant) < 0.001
    assert full > constant


def test_02b_r2_has_no_lower_bound():
    bad = m.bad_predictor_test_r2()
    # A deliberately bad predictor -- zero, always -- is not merely worse
    # than the constant-mean baseline; it is worse by nearly five full
    # units of R2, which is impossible if R2 lived in [0, 1] the way most
    # readers assume.
    assert bad == -4.7009
    assert bad < -4.0


# --- 3. RMSE versus MAE under one outlier ----------------------------------


def test_03_rmse_moves_more_than_mae_when_one_target_is_an_outlier():
    rmse_before, mae_before, rmse_after, mae_after = m.rmse_mae_outlier_shift()
    assert (rmse_before, mae_before, rmse_after, mae_after) == (2.4801, 1.9833, 28.2569, 5.9448)
    rmse_ratio = rmse_after / rmse_before
    mae_ratio = mae_after / mae_before
    # Squaring the error in RMSE means one very wrong prediction dominates
    # the sum; MAE, which never squares anything, moves by far less.
    assert rmse_ratio > 11.0
    assert mae_ratio < 3.5
    assert rmse_ratio > 3 * mae_ratio


# --- 4. MAPE breaking -------------------------------------------------------


def test_04_mape_explodes_silently_at_a_zero_true_value():
    value = m.mape_at_zero_target()
    # scikit-learn does not raise or warn on a zero true value: it floors
    # the denominator at machine epsilon and returns a number. The result
    # is not a small mistake -- it is off by roughly fourteen orders of
    # magnitude from anything a percentage error should look like.
    assert value > 1.0e10


def test_04b_mape_explodes_near_zero_while_mae_stays_sane():
    mape_value, mae_value = m.mape_near_zero_target()
    assert (mape_value, mae_value) == (3.3667, 5.0)
    # The same three rows: MAE reports a modest, believable 5.0 units of
    # error. MAPE reports 336.67 percent -- a number nobody would present
    # to a stakeholder -- because one of the three true values is 0.5 and
    # a five-unit miss on 0.5 is a factor of ten.
    assert mape_value > 3.0
    assert mae_value < 10.0


def test_05_mape_is_bounded_under_but_not_over():
    max_under, ten_x_over = m.mape_asymmetry_bound()
    assert (max_under, ten_x_over) == (1.0, 10.0)
    # The worst possible systematic under-prediction -- always guessing
    # zero -- cannot exceed 100 percent MAPE, because the error can never
    # exceed the true value once the prediction floor of zero is hit.
    # Over-prediction has no such ceiling: predicting eleven times the
    # truth reports 1000 percent, and there is no larger multiple that
    # would not report a correspondingly larger number. The two directions
    # of being wrong are not scored on the same scale.
    assert max_under == 1.0
    assert ten_x_over > max_under


# --- 6. A metric ranking inversion ------------------------------------------


def test_06_rmse_and_mae_prefer_different_models():
    rmse_a, mae_a, rmse_b, mae_b = m.ranking_inversion_models()
    assert (rmse_a, mae_a, rmse_b, mae_b) == (1.947, 1.586, 4.4353, 0.8417)
    # Model A makes many small, consistent errors. Model B is right almost
    # everywhere and badly wrong on a handful of rows. RMSE, which squares
    # every error, is dominated by Model B's few large misses and prefers
    # A. MAE, which weighs every error equally, is dominated by the
    # ninety-five rows Model B gets almost exactly right and prefers B.
    assert rmse_a < rmse_b
    assert mae_b < mae_a


# --- 7. RMSE and MAE carry the target's units -------------------------------


def test_07_rmse_and_mae_are_identical_on_raw_and_standardised_features():
    results = m.raw_and_scaled_metrics()
    assert results["scaled"] == (56.3929, 45.1206, 0.3594)
    assert results["raw"] == (56.3929, 45.1206, 0.3594)
    # Ordinary least squares is invariant to a per-column affine rescaling
    # of its inputs, so the predictions -- and every metric computed from
    # them -- are identical whether the features are standardised or left
    # in their original units (age in years, bmi, raw blood pressure).
    assert results["scaled"] == results["raw"]
    # RMSE and MAE are stated in the target's own units. The diabetes
    # target has no physical unit -- it is a composite disease-progression
    # score running 25 to 346 -- so a mean absolute error of 45.12 means
    # "45.12 points on that 25-346 scale", not "45.12 of anything you
    # could hand a doctor". A metric you cannot state a unit for is a
    # metric you cannot explain.


# --- 8. r2_score: agreement, and the argument-order bug ---------------------


def test_08_r2_score_agrees_with_linearregression_score():
    from_metric, from_model = m.r2_score_vs_model_score()
    assert from_metric == from_model == 0.359409


def test_08b_r2_score_argument_order_changes_the_answer():
    correct_order, swapped_order = m.r2_score_argument_order()
    assert correct_order == 0.359409
    # Swapping the two arguments is not a harmless typo: r2_score is not
    # symmetric in y_true and y_pred, because the denominator is the
    # variance of whichever array is passed FIRST. Swapped, the same
    # predictions score negative -- a model that looked genuinely useful
    # now looks worse than a constant-mean baseline, from one call written
    # the wrong way round.
    assert swapped_order == -0.209635
    assert swapped_order != correct_order
    assert swapped_order < 0 < correct_order

"""Machinery checks: the helpers behave, before any claim is made.

These four tests are solved in both `starter/` and `examples/`. They exist
so that a broken helper reports itself as a broken helper rather than as a
surprising scientific result.
"""

import numpy as np

import regression_metrics_lib as m


def test_diabetes_split_shapes_and_ranges():
    X_train, X_test, y_train, y_test = m.diabetes_split()
    assert X_train.shape == (331, 10)
    assert X_test.shape == (111, 10)
    assert y_train.shape == (331,)
    assert y_test.shape == (111,)
    # The target is documented as running from 25 to 346.
    assert 25.0 <= float(np.concatenate([y_train, y_test]).min())
    assert float(np.concatenate([y_train, y_test]).max()) <= 346.0


def test_rmse_and_mae_agree_on_a_perfect_prediction():
    y_true = np.array([1.0, 2.0, 3.0, 4.0])
    assert m.rmse(y_true, y_true) == 0.0
    assert m.mae(y_true, y_true) == 0.0


def test_rmse_is_never_smaller_than_mae():
    # A standard inequality: RMSE >= MAE always, with equality only when
    # every absolute error is identical.
    rng = np.random.default_rng(0)
    y_true = rng.normal(size=40)
    y_pred = y_true + rng.normal(scale=2.0, size=40)
    assert m.rmse(y_true, y_pred) >= m.mae(y_true, y_pred)
    # Equality when every error has the same magnitude.
    y_true_eq = np.zeros(6)
    y_pred_eq = np.array([1.0, -1.0, 1.0, -1.0, 1.0, -1.0])
    assert round(m.rmse(y_true_eq, y_pred_eq), 10) == round(m.mae(y_true_eq, y_pred_eq), 10)


def test_adjusted_r2_equals_r2_when_no_predictors_are_added():
    # With p fixed, adjusted R2 is a deterministic function of r2, n and p --
    # sanity-check the formula against a hand-computable case.
    # n=10, p=2, r2=0.8: 1 - (1-0.8) * 9/7
    assert round(m.adjusted_r2(0.8, 10, 2), 6) == round(1.0 - 0.2 * 9.0 / 7.0, 6)
    # Adjusted R2 is always <= R2 whenever p > 0 and n > p + 1.
    assert m.adjusted_r2(0.5, 100, 5) <= 0.5

lesson_id: D152
day: 152
kind: guided-build
languages:
  - python
  - bash
setup_commands:
  - cd labs/sections/machine-learning/day-152-regression-metrics
  - python3 -m venv .venv
  - .venv/bin/pip install -r requirements/requirements.txt
  - >-
    .venv/bin/python3 -c "import numpy, sklearn; print(numpy.__version__,
    sklearn.__version__)"
run_commands:
  - .venv/bin/pytest examples -q
  - .venv/bin/pytest starter -q
  - .venv/bin/python3 examples/report_measurements.py
test_commands:
  - bash tests/run_tests.sh
cleanup_commands:
  - >-
    find . -path ./.venv -prune -o -type d -name '__pycache__' -print -exec rm -rf -- {}
    +
  - rm -rf .pytest_cache
  - 'rm -rf .venv  # optional: removes the lab virtual environment'
  - 'git checkout -- starter/  # optional: reset your work'
requires_network: true
requires_api_key: false
estimated_minutes: 55
last_executed: '2026-08-27'
executed_on: >-
  macOS 26.5.2 (Apple Silicon, arm64, CPU only -- no GPU is needed or used), Python
  3.14.0, numpy 2.5.2, scikit-learn 1.9.0, pytest 9.1.1, bash 3.2.57 -- bash
  tests/run_tests.sh -> 14 checks, 0 failure(s), exit 0. pytest examples -q -> 16 passed.
  pytest starter -q -> 4 passed, 12 skipped (the four machinery checks in
  test_metrics_lib.py are solved in both directories; the twelve exercise stubs in
  starter/test_metrics_claims.py are untouched). Everything ran through a real lab-local
  .venv created by the documented setup commands; scikit-learn pulled in scipy, joblib
  and threadpoolctl as its own dependencies, none of which this lab imports directly. The
  lab is fully offline after the pip install -- load_diabetes ships bundled inside
  scikit-learn's own installed files, nothing is downloaded, and harness check 9 confirms
  no URL appears anywhere in starter/ or examples/ source. Section 7 of the harness
  copies examples/ into a mktemp -d scratch directory, confirms 16 passed, rewrites
  `assert (rmse_a, mae_a, rmse_b, mae_b) == (1.947, 1.586, 4.4353, 0.8417)` to
  `(0.0, 0.0, 0.0, 0.0)`, confirms a non-zero exit naming the failing test, and removes
  the scratch directory. Separately, by hand, `assert m.bad_predictor_test_r2() ==
  -4.7009` was changed to `== 0.0` in examples/test_metrics_claims.py and the whole
  harness re-run: it reported 14 checks, 2 failure(s) and exited 1 (both the pytest run
  and the pytest-free direct reproduction in section 2 caught it); the file was restored
  and the harness returned to 14 checks, 0 failure(s), exit 0. MEASURED PAIRS, all
  captured verbatim in expected-output/measured-values.txt. (1) Fitting LinearRegression
  on the diabetes train split (331 rows, test_size=0.25, random_state=0) and adding
  independent standard-normal noise columns gives train R2 of 0.5554, 0.5555, 0.5648,
  0.5754 and 0.7403 at 0, 1, 5, 20 and 100 noise columns -- strictly increasing on
  columns with zero relationship to the target, because more predictors can only help a
  training-set fit. (2) Adjusted R2 on the same five fits is 0.5415, 0.5402, 0.5441,
  0.5329 and 0.6104: it correctly falls below the 0-noise baseline at 20 noise columns,
  then climbs back ABOVE the baseline at 100 noise columns once the predictor count (110)
  becomes a large fraction of the sample size (331) -- the correction has its own failure
  mode. (3) The full ten-feature model scores test R2 of 0.3594; a constant-mean
  predictor scores -0.0001 (essentially exactly zero, by construction); a deliberately
  bad all-zeros predictor scores -4.7009, nearly five full units below zero, which is
  impossible if R2 lived in [0, 1]. (4) Fifty points with small consistent errors give
  RMSE 2.4801 and MAE 1.9833; moving one true value 200 units away (predictions
  unchanged) gives RMSE 28.2569 (a factor of 11.39) and MAE 5.9448 (a factor of 3.00) --
  RMSE moved nearly four times as much, because it squares the one large error and MAE
  does not. (5) mean_absolute_percentage_error on four rows including one true value of
  exactly zero returns 5629499534213120.0 -- not an exception, not a warning, because
  scikit-learn floors the zero denominator at machine epsilon rather than raising. On
  three rows including one true value of 0.5, MAPE is 3.3667 (336.67 percent) while MAE
  on the same rows is a sane 5.0. The worst possible systematic under-prediction (always
  guessing zero) gives MAPE exactly 1.0 (100 percent, the ceiling); predicting eleven
  times the truth gives MAPE 10.0 (1000 percent, with no ceiling of its own). (6) Two
  models on the same 100 targets: Model A (many small consistent errors) scores RMSE
  1.947, MAE 1.586; Model B (right on 95 rows, badly wrong on 5) scores RMSE 4.4353, MAE
  0.8417 -- RMSE prefers A, MAE prefers B, a genuine ranking inversion between the two
  most common regression metrics. (7) RMSE 56.3929, MAE 45.1206 and R2 0.3594 are
  IDENTICAL whether LinearRegression is fit on load_diabetes(scaled=True) or
  load_diabetes(scaled=False) -- ordinary least squares is invariant to a per-column
  affine rescaling of its inputs. (8) r2_score(y_test, pred) and
  model.score(X_test, y_test) agree at 0.359409; r2_score(pred, y_test), the arguments
  swapped, returns -0.209635 -- a real and common bug, not a contrived one, since the
  same predictions look like a usable model in the correct order and worse than guessing
  the mean in the swapped one. THREE HONESTY CALLS. FIRST: adjusted R2 is usually taught
  as unambiguously "the corrected version" of R2, but the measurement shows its own
  failure mode once the predictor count approaches the sample size -- this was not
  anticipated going in and is reported because it was measured, not because it was
  expected. SECOND: MAPE's asymmetry claim needed real construction rather than folklore.
  A symmetric-in-absolute-error construction (moving the true value by the same delta in
  each direction) produces IDENTICAL MAPE in both directions, because the metric's
  denominator is always the true value regardless of which way the error points -- so the
  "MAPE penalizes over- and under-prediction differently" folklore is only correct in the
  specific, verified sense that the worst-case ACHIEVABLE under-prediction is bounded at
  100 percent (predictions cannot go below zero) while over-prediction is unbounded; it is
  not correct as a claim about equal-magnitude errors on a fixed true value, which are
  symmetric by the formula's own construction. This distinction is stated explicitly in
  the lesson rather than repeating the folklore's looser version. THIRD: the diabetes
  target has no natural physical unit -- it is a composite disease-progression score
  running 25 to 346, not a measurement in mg/dL or any other real unit -- so the lesson
  states RMSE and MAE in "points on that scale" rather than fabricating a unit the
  dataset's own documentation does not provide. Harness check 8 re-runs the noise-column
  climb at three further seeds, the ranking inversion at three further seeds, and the
  RMSE-vs-MAE-under-an-outlier direction at three further shift sizes, so no directional
  claim rests on the one seed the lesson quotes.

# Requirements

`requirements.txt` pins the three packages this lab imports directly, at
the exact versions the captured output in `expected-output/` was produced
with:

```
numpy==2.5.2
scikit-learn==1.9.0
pytest==9.1.1
```

Installing scikit-learn also pulls in scipy, joblib and threadpoolctl as
its own dependencies. This lab imports none of them directly and does not
pin them.

## Why the versions are pinned exactly

Almost every number in this lab comes from `sklearn.datasets.load_diabetes`
(a fixed, bundled dataset -- no download, no randomness in the data itself)
combined with `numpy.random.default_rng` for the handful of exercises that
add synthetic noise or construct toy predictions. NumPy's documentation is
explicit that `Generator` makes no promise of stream compatibility between
versions, so a different NumPy can legitimately produce a different noise
stream from the same seed, and any figure built from it could move by a
little.

What does not depend on the pins: every direction this lab claims --
training R2 climbing as noise columns are added, R2 having no lower bound,
RMSE moving more than MAE under an outlier, MAPE exploding at a zero or
near-zero true value, the RMSE/MAE ranking inversion, and the r2_score
argument-order bug. Harness check 8 re-runs three of those at seeds the
lesson never quotes, precisely so the distinction between "always true" and
"true under these pins" is enforced rather than merely asserted.
`expected-output/FIELDS.md` separates the two categories in full.

## Installing

From the lab directory:

```bash
python3 -m venv .venv
.venv/bin/pip install -r requirements/requirements.txt
```

The install step needs the network. Everything after it is offline: the
diabetes dataset ships inside scikit-learn itself, and every synthetic
example in this lab is generated on the spot from a seeded generator.

## Free and open-source status

All three packages are free and open source -- NumPy and scikit-learn under
the BSD 3-Clause licence, pytest under the MIT licence. There is no paid
tier, no account and no API key anywhere in this lab.

numpy==2.5.2
scikit-learn==1.9.0
pytest==9.1.1

# Day 152 lab brief — What You Report Is Not What You Optimise

Day 149 drew the line: a loss is what you optimise, a metric is what you
report, and they do not have to be the same function. This lab measures
the reporting side -- RMSE, MAE, MAPE, R2 and adjusted R2 -- and the traps
in each.

## The claim you are here to measure

> R2 is the most quoted and least understood number in regression.

Exercise 1 measures why. Fit a linear model on the diabetes dataset, then
add columns of **pure noise** -- independent random numbers with zero
relationship to the target -- and watch training R2 anyway:

| noise columns added | train R2 |
| --- | --- |
| 0 | 0.5554 |
| 1 | 0.5555 |
| 5 | 0.5648 |
| 20 | 0.5754 |
| 100 | 0.7403 |

Every one of those columns is garbage. The model climbs to 0.7403 anyway,
because more predictors can only help a training-set fit -- never hurt it.
Exercise 1b brings in adjusted R2, which is built to penalise exactly this,
and shows it working at a modest number of extra columns and **breaking
down at a large one**: at 100 noise columns (110 predictors on 331 rows),
adjusted R2 climbs back above the no-noise baseline even though nothing
useful was added.

## The part most people get wrong

Most people believe R2 lives in `[0, 1]` and means "percent of variance
explained". Exercise 2 shows the first half is false: a deliberately bad
predictor scores **-4.7009**, nearly five full units below zero. There is
no floor. R2 compares your model to one specific baseline -- always predict
the training mean -- and if you do worse than that baseline, the number
goes negative with no limit.

## Why RMSE and MAE can disagree about which model is better

Exercise 6 is the single most valuable measurement in this lab. Two models,
scored on the same targets:

| Model | RMSE | MAE |
| --- | --- | --- |
| A: many small errors | 1.947 | 1.586 |
| B: a few large errors | 4.4353 | 0.8417 |

RMSE says A is better. MAE says B is better. **Both are correct, about
different things.** RMSE squares every error before averaging, so a
handful of large misses dominates it. MAE weighs every error equally, so
being right almost everywhere dominates it. Reporting only one of the two
would silently pick a winner the other metric disagrees with.

## MAPE, breaking

Exercises 4, 4b and 5 build MAPE up and then break it three ways: it
explodes (without raising or warning) when a true value is exactly zero;
it explodes almost as badly when a true value is merely close to zero; and
it is structurally asymmetric -- the worst possible under-prediction caps
out at 100 percent, while over-prediction has no ceiling at all.

## How to work

1. Build the environment (see the lab `README.md`).
2. Run `.venv/bin/pytest starter -q`. You will see four passes (the
   machinery checks in `test_metrics_lib.py`) and twelve skips.
3. Replace one `pytest.skip(...)` at a time with real code. The skip text
   names the exact helper and the exact value to assert.
4. Print the measured pair in every exercise. A number you did not print
   is a number you did not look at.
5. When you want the whole measured table at once, run
   `.venv/bin/python3 examples/report_measurements.py`.

Do not run `pytest starter examples` in one invocation. Both directories
define `regression_metrics_lib.py`, `test_metrics_lib.py` and
`test_metrics_claims.py`; pytest aborts on the module-name collision. Run
them separately, always.

"""Regression metrics, measured: what each one reports, and what it hides.

Day 149 drew the line once: a loss is what you optimise, a metric is what
you report, and they need not be the same function. This module measures
the reporting side for regression -- RMSE, MAE, MAPE, R-squared and
adjusted R-squared -- and the traps in each, on the diabetes dataset
(``sklearn.datasets.load_diabetes``) and on constructed data where a
property needs to be exact rather than merely typical.

Everything here is deterministic given a seed.
"""

from __future__ import annotations

import numpy as np

from sklearn.datasets import load_diabetes
from sklearn.linear_model import LinearRegression
from sklearn.metrics import (
    mean_absolute_error,
    mean_absolute_percentage_error,
    mean_squared_error,
    r2_score,
)
from sklearn.model_selection import train_test_split


# --------------------------------------------------------------------------
# 0. The dataset this module shares
# --------------------------------------------------------------------------


def diabetes_split(scaled: bool = True, seed: int = 0):
    """The 75/25 split every measurement below is built on."""
    X, y = load_diabetes(return_X_y=True, scaled=scaled)
    return train_test_split(X, y, test_size=0.25, random_state=seed)


def rmse(y_true, y_pred) -> float:
    return float(np.sqrt(mean_squared_error(y_true, y_pred)))


def mae(y_true, y_pred) -> float:
    return float(mean_absolute_error(y_true, y_pred))


# --------------------------------------------------------------------------
# 1. Train R-squared is not a quality measure: the noise-column climb
# --------------------------------------------------------------------------


def adjusted_r2(r2: float, n: int, p: int) -> float:
    """R-squared, penalised for the number of predictors used to get it."""
    return 1.0 - (1.0 - r2) * (n - 1) / (n - p - 1)


def noise_column_r2_curve(noise_counts=(0, 1, 5, 20, 100), seed: int = 0):
    """Train R2 and adjusted R2 as pure-noise columns are added.

    Every added column is independent standard-normal noise with no
    relationship whatsoever to the target. Returns rows of
    ``(n_noise, n_rows, n_predictors, train_r2, adjusted_r2)``.
    """
    X_train, _X_test, y_train, _y_test = diabetes_split(seed=0)
    rng = np.random.default_rng(seed)
    rows = []
    for n_noise in noise_counts:
        if n_noise == 0:
            X_aug = X_train
        else:
            noise = rng.normal(size=(X_train.shape[0], n_noise))
            X_aug = np.hstack([X_train, noise])
        model = LinearRegression().fit(X_aug, y_train)
        r2 = r2_score(y_train, model.predict(X_aug))
        n_rows, n_predictors = X_aug.shape
        rows.append((n_noise, n_rows, n_predictors, round(r2, 4), round(adjusted_r2(r2, n_rows, n_predictors), 4)))
    return rows


def full_model_test_r2(seed: int = 0) -> float:
    """The test R2 of the ordinary ten-feature model -- the honest number."""
    X_train, X_test, y_train, y_test = diabetes_split(seed=seed)
    model = LinearRegression().fit(X_train, y_train)
    return round(float(r2_score(y_test, model.predict(X_test))), 4)


# --------------------------------------------------------------------------
# 2. R-squared is not bounded below by zero
# --------------------------------------------------------------------------


def constant_mean_test_r2(seed: int = 0) -> float:
    """R2 of predicting the train mean for every test row."""
    _X_train, _X_test, y_train, y_test = diabetes_split(seed=seed)
    prediction = np.full_like(y_test, y_train.mean(), dtype=float)
    return round(float(r2_score(y_test, prediction)), 4)


def bad_predictor_test_r2(seed: int = 0) -> float:
    """R2 of a deliberately bad predictor: zero, always."""
    _X_train, _X_test, _y_train, y_test = diabetes_split(seed=seed)
    prediction = np.zeros_like(y_test, dtype=float)
    return round(float(r2_score(y_test, prediction)), 4)


# --------------------------------------------------------------------------
# 3. RMSE versus MAE under an outlier
# --------------------------------------------------------------------------


def rmse_mae_outlier_shift(seed: int = 1, n: int = 50, shift: float = 200.0):
    """Same predictions, one target moved far away. Which metric moves more?

    Returns ``(rmse_before, mae_before, rmse_after, mae_after)``.
    """
    rng = np.random.default_rng(seed)
    y_true = rng.normal(100.0, 10.0, size=n)
    y_pred = y_true + rng.normal(0.0, 3.0, size=n)
    before = (rmse(y_true, y_pred), mae(y_true, y_pred))

    y_true_shifted = y_true.copy()
    y_true_shifted[0] += shift
    after = (rmse(y_true_shifted, y_pred), mae(y_true_shifted, y_pred))
    return round(before[0], 4), round(before[1], 4), round(after[0], 4), round(after[1], 4)


# --------------------------------------------------------------------------
# 4. MAPE breaking: zero targets, near-zero targets, and structural asymmetry
# --------------------------------------------------------------------------


def mape_at_zero_target() -> float:
    """MAPE where one true value is exactly zero.

    Not undefined in the mathematical sense of raising -- scikit-learn
    floors the denominator at machine epsilon, so this returns a huge,
    silently wrong number rather than an error or a warning.
    """
    y_true = np.array([10.0, 20.0, 0.0, 30.0])
    y_pred = np.array([12.0, 18.0, 5.0, 28.0])
    return float(mean_absolute_percentage_error(y_true, y_pred))


def mape_near_zero_target():
    """MAPE explodes on a true value close to (but not) zero.

    Returns ``(mape, mae)`` on the same three rows, so the contrast between
    a metric that explodes and one that does not is visible in one call.
    """
    y_true = np.array([100.0, 100.0, 0.5])
    y_pred = np.array([105.0, 95.0, 5.5])
    return (
        round(float(mean_absolute_percentage_error(y_true, y_pred)), 4),
        round(float(mean_absolute_error(y_true, y_pred)), 4),
    )


def mape_asymmetry_bound(true_value: float = 100.0):
    """MAPE's structural asymmetry: bounded under, unbounded over.

    A model that under-predicts every row can be wrong by at most 100
    percent (predict zero, and the error cannot exceed the true value).
    A model that over-predicts has no such ceiling. Returns
    ``(max_under_prediction_mape, ten_times_over_prediction_mape)``.
    """
    y_true = np.full(5, true_value)
    max_under = np.zeros(5)  # the worst possible under-prediction: always zero
    ten_x_over = y_true * 11.0  # predicting eleven times the true value
    return (
        round(float(mean_absolute_percentage_error(y_true, max_under)), 4),
        round(float(mean_absolute_percentage_error(y_true, ten_x_over)), 4),
    )


# --------------------------------------------------------------------------
# 5. A metric ranking inversion: RMSE and MAE prefer different models
# --------------------------------------------------------------------------


def ranking_inversion_models(seed: int = 2, n: int = 100):
    """Two models scored on the same targets: one wins on RMSE, one on MAE.

    Model A makes many small, consistent errors. Model B is right almost
    everywhere but wrong by a lot on a few rows. Returns
    ``(rmse_a, mae_a, rmse_b, mae_b)``.
    """
    rng = np.random.default_rng(seed)
    y_true = rng.normal(50.0, 5.0, size=n)

    errors_a = rng.normal(0.0, 2.0, size=n)
    pred_a = y_true + errors_a

    errors_b = np.zeros(n)
    big_idx = rng.choice(n, size=5, replace=False)
    errors_b[big_idx] = rng.normal(0.0, 15.0, size=5)
    pred_b = y_true + errors_b

    return (
        round(rmse(y_true, pred_a), 4),
        round(mae(y_true, pred_a), 4),
        round(rmse(y_true, pred_b), 4),
        round(mae(y_true, pred_b), 4),
    )


# --------------------------------------------------------------------------
# 6. Units: RMSE and MAE carry the units of the target
# --------------------------------------------------------------------------


def raw_and_scaled_metrics(seed: int = 0):
    """RMSE, MAE and R2 fit on raw-unit features versus standardised ones.

    ``load_diabetes(scaled=False)`` returns the ten features in their
    original units -- age in years, bmi as bmi, blood pressure as measured
    -- while the target is the same disease-progression score either way.
    Ordinary least squares is invariant to a per-column affine rescaling of
    its inputs, so this returns identical numbers under both, which is
    itself the thing worth confirming rather than assuming.
    """
    results = {}
    for label, scaled in (("scaled", True), ("raw", False)):
        X_train, X_test, y_train, y_test = diabetes_split(scaled=scaled, seed=seed)
        model = LinearRegression().fit(X_train, y_train)
        pred = model.predict(X_test)
        results[label] = (round(rmse(y_test, pred), 4), round(mae(y_test, pred), 4), round(float(r2_score(y_test, pred)), 4))
    return results


# --------------------------------------------------------------------------
# 7. r2_score: agreement with .score, and the argument-order bug
# --------------------------------------------------------------------------


def r2_score_vs_model_score(seed: int = 0):
    """r2_score(y_true, y_pred) against LinearRegression.score(X, y)."""
    X_train, X_test, y_train, y_test = diabetes_split(seed=seed)
    model = LinearRegression().fit(X_train, y_train)
    pred = model.predict(X_test)
    return round(float(r2_score(y_test, pred)), 6), round(float(model.score(X_test, y_test)), 6)


def r2_score_argument_order(seed: int = 0):
    """r2_score is NOT symmetric in its two arguments: swap them and it changes.

    Returns ``(correct_order, swapped_order)``.
    """
    X_train, X_test, y_train, y_test = diabetes_split(seed=seed)
    model = LinearRegression().fit(X_train, y_train)
    pred = model.predict(X_test)
    return round(float(r2_score(y_test, pred)), 6), round(float(r2_score(pred, y_test)), 6)

"""Twelve exercises in what a regression metric reports, and what it hides.

Read `00_brief.md` first. Each function below is a `pytest.skip` naming
exactly what to build and what to assert; replace the skip with real code.
`regression_metrics_lib.py` is complete -- it is the machinery, not the
exercise.

Run this suite on its own:

    .venv/bin/pytest starter -q

Never run `pytest starter examples` in one invocation: both directories
define modules with the same names and pytest aborts on the collision.
"""

import numpy as np  # noqa: F401  (you will need it)
import pytest

import regression_metrics_lib as m  # noqa: F401  (you will need it)


# --- 1. Train R2 is not a quality measure ---------------------------------


def test_01_train_r2_climbs_on_pure_noise_columns():
    pytest.skip(
        "Call m.noise_column_r2_curve() and assert it equals the five rows in "
        "expected-output/measured-values.txt, from (0, 331, 10, 0.5554, 0.5415) "
        "to (100, 331, 110, 0.7403, 0.6104). Then assert the train_r2 column "
        "(index 3 of each row) is strictly increasing, and that it climbs by "
        "more than 0.18 from no noise to 100 noise columns. Every added column "
        "is independent random noise with zero relationship to the target."
    )


def test_01b_adjusted_r2_corrects_the_climb_then_breaks_down_itself():
    pytest.skip(
        "Build a dict keyed by noise count from m.noise_column_r2_curve(). "
        "Assert the adjusted R2 at 20 noise columns is LOWER than the 0-noise "
        "baseline (0.5329 < 0.5415) -- the correction working as intended. "
        "Then assert the adjusted R2 at 100 noise columns is HIGHER than both "
        "the baseline and the 20-noise value (0.6104 > 0.5415 and "
        "0.6104 > 0.5329) -- the correction breaking down once the number of "
        "predictors (110) becomes a large fraction of the sample size (331)."
    )


# --- 2. R2 is not bounded below by zero ------------------------------------


def test_02_the_full_model_beats_a_constant_mean_predictor_on_test():
    pytest.skip(
        "Assert m.full_model_test_r2() equals 0.3594 and that "
        "abs(m.constant_mean_test_r2()) is less than 0.001. A predictor that "
        "always guesses the training mean scores R2 essentially exactly zero "
        "on fresh test data BY CONSTRUCTION -- R2 is defined relative to "
        "exactly that predictor, which is why zero is the number it compares "
        "against, not an accident of this dataset."
    )


def test_02b_r2_has_no_lower_bound():
    pytest.skip(
        "Assert m.bad_predictor_test_r2() equals -4.7009 and is less than "
        "-4.0. A deliberately bad predictor -- always zero -- scores nearly "
        "five FULL UNITS below zero, which is impossible if R2 lived in "
        "[0, 1] the way most people assume it does."
    )


# --- 3. RMSE versus MAE under one outlier ----------------------------------


def test_03_rmse_moves_more_than_mae_when_one_target_is_an_outlier():
    pytest.skip(
        "Call m.rmse_mae_outlier_shift() and assert the four values equal "
        "(2.4801, 1.9833, 28.2569, 5.9448). Compute rmse_ratio = "
        "rmse_after / rmse_before and mae_ratio = mae_after / mae_before. "
        "Assert rmse_ratio > 11.0, mae_ratio < 3.5, and rmse_ratio > "
        "3 * mae_ratio. Same predictions both times; only one target row "
        "moved far away."
    )


# --- 4. MAPE breaking -------------------------------------------------------


def test_04_mape_explodes_silently_at_a_zero_true_value():
    pytest.skip(
        "Call m.mape_at_zero_target() and assert the result is greater than "
        "1.0e10. Note what does NOT happen: no exception, no warning -- "
        "scikit-learn floors the zero denominator at machine epsilon and "
        "returns a number that is off by roughly fourteen orders of "
        "magnitude from anything a percentage error should look like."
    )


def test_04b_mape_explodes_near_zero_while_mae_stays_sane():
    pytest.skip(
        "Call m.mape_near_zero_target() and assert it equals (3.3667, 5.0). "
        "The first value is MAPE, the second is MAE, on the SAME three "
        "predictions. MAE reports a believable 5.0 units of error; MAPE "
        "reports 336.67 percent, because one true value is 0.5 and a "
        "five-unit miss on 0.5 is a factor of ten."
    )


def test_05_mape_is_bounded_under_but_not_over():
    pytest.skip(
        "Call m.mape_asymmetry_bound() and assert it equals (1.0, 10.0). "
        "The first value is the MAPE of the worst possible systematic "
        "under-prediction -- always guessing zero -- which cannot exceed "
        "100 percent. The second is the MAPE of predicting eleven times the "
        "truth, which is 1000 percent, with no ceiling of its own. Assert "
        "ten_x_over > max_under. Being wrong in one direction is capped; "
        "being wrong in the other is not."
    )


# --- 6. A metric ranking inversion ------------------------------------------


def test_06_rmse_and_mae_prefer_different_models():
    pytest.skip(
        "Call m.ranking_inversion_models() and assert the four values equal "
        "(1.947, 1.586, 4.4353, 0.8417) for (rmse_a, mae_a, rmse_b, mae_b). "
        "Then assert rmse_a < rmse_b (RMSE prefers Model A) and mae_b < "
        "mae_a (MAE prefers Model B). Model A makes many small consistent "
        "errors; Model B is nearly perfect except for a handful of large "
        "misses. Reporting only one metric would silently pick a winner the "
        "other metric disagrees with."
    )


# --- 7. RMSE and MAE carry the target's units -------------------------------


def test_07_rmse_and_mae_are_identical_on_raw_and_standardised_features():
    pytest.skip(
        "Call m.raw_and_scaled_metrics() and assert results['scaled'] == "
        "(56.3929, 45.1206, 0.3594) and results['raw'] == the same tuple. "
        "Ordinary least squares is invariant to a per-column affine "
        "rescaling of its inputs, so every metric computed from its "
        "predictions is identical whether the features are standardised or "
        "left in raw units (age in years, bmi, raw blood pressure). Then "
        "state in a comment what unit the RMSE and MAE numbers are actually "
        "in for this dataset."
    )


# --- 8. r2_score: agreement, and the argument-order bug ---------------------


def test_08_r2_score_agrees_with_linearregression_score():
    pytest.skip(
        "Call m.r2_score_vs_model_score() and assert both returned values "
        "equal 0.359409. sklearn.metrics.r2_score(y_true, y_pred) and "
        "LinearRegression.score(X, y) compute the same quantity."
    )


def test_08b_r2_score_argument_order_changes_the_answer():
    pytest.skip(
        "Call m.r2_score_argument_order() and assert it equals "
        "(0.359409, -0.209635). r2_score is NOT symmetric in its two "
        "arguments -- the denominator is the variance of whichever array is "
        "passed FIRST. Assert the swapped value is negative while the "
        "correct-order value is positive: the same predictions, scored with "
        "the arguments the wrong way round, look worse than a constant-mean "
        "baseline."
    )

"""Machinery checks: the helpers behave, before any claim is made.

These four tests are solved in both `starter/` and `examples/`. They exist
so that a broken helper reports itself as a broken helper rather than as a
surprising scientific result.
"""

import numpy as np

import regression_metrics_lib as m


def test_diabetes_split_shapes_and_ranges():
    X_train, X_test, y_train, y_test = m.diabetes_split()
    assert X_train.shape == (331, 10)
    assert X_test.shape == (111, 10)
    assert y_train.shape == (331,)
    assert y_test.shape == (111,)
    # The target is documented as running from 25 to 346.
    assert 25.0 <= float(np.concatenate([y_train, y_test]).min())
    assert float(np.concatenate([y_train, y_test]).max()) <= 346.0


def test_rmse_and_mae_agree_on_a_perfect_prediction():
    y_true = np.array([1.0, 2.0, 3.0, 4.0])
    assert m.rmse(y_true, y_true) == 0.0
    assert m.mae(y_true, y_true) == 0.0


def test_rmse_is_never_smaller_than_mae():
    # A standard inequality: RMSE >= MAE always, with equality only when
    # every absolute error is identical.
    rng = np.random.default_rng(0)
    y_true = rng.normal(size=40)
    y_pred = y_true + rng.normal(scale=2.0, size=40)
    assert m.rmse(y_true, y_pred) >= m.mae(y_true, y_pred)
    # Equality when every error has the same magnitude.
    y_true_eq = np.zeros(6)
    y_pred_eq = np.array([1.0, -1.0, 1.0, -1.0, 1.0, -1.0])
    assert round(m.rmse(y_true_eq, y_pred_eq), 10) == round(m.mae(y_true_eq, y_pred_eq), 10)


def test_adjusted_r2_equals_r2_when_no_predictors_are_added():
    # With p fixed, adjusted R2 is a deterministic function of r2, n and p --
    # sanity-check the formula against a hand-computable case.
    # n=10, p=2, r2=0.8: 1 - (1-0.8) * 9/7
    assert round(m.adjusted_r2(0.8, 10, 2), 6) == round(1.0 - 0.2 * 9.0 / 7.0, 6)
    # Adjusted R2 is always <= R2 whenever p > 0 and n > p + 1.
    assert m.adjusted_r2(0.5, 100, 5) <= 0.5

#!/usr/bin/env bash
# Day 152 lab harness: "What You Report Is Not What You Optimise"
#
# Prints "N checks, M failure(s)" and exits 0 only when M is zero.
set -u

LAB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$LAB_DIR"

PYTHON="${PYTHON:-.venv/bin/python3}"
PYTEST="${PYTEST:-.venv/bin/pytest}"

# Clear caches at the START so the final cleanliness check measures what
# THIS run left behind, not what a previous `pytest starter -q` left.
find . -path ./.venv -prune -o -type d -name '__pycache__' -exec rm -rf -- {} + 2>/dev/null
rm -rf .pytest_cache

CHECKS=0
FAILURES=0

ok() {
  CHECKS=$((CHECKS + 1))
  echo "  ok: $1"
}

fail() {
  CHECKS=$((CHECKS + 1))
  FAILURES=$((FAILURES + 1))
  echo "  FAIL: $1"
}

if [ ! -x "$PYTHON" ]; then
  echo "No lab .venv found at $PYTHON."
  echo "Run: python3 -m venv .venv && .venv/bin/pip install -r requirements/requirements.txt"
  exit 2
fi

echo "1. Installed versions match requirements/requirements.txt"
VERSION_CHECK=$("$PYTHON" - <<'PYEOF'
import numpy, sklearn, pytest
print("numpy", numpy.__version__)
print("scikit-learn", sklearn.__version__)
print("pytest", pytest.__version__)
PYEOF
)
echo "$VERSION_CHECK" | sed 's/^/    /'
while read -r pkg pin; do
  pin_version="${pin#*==}"
  installed=$(echo "$VERSION_CHECK" | awk -v p="$pkg" '$1==p {print $2}')
  if [ "$installed" = "$pin_version" ]; then
    ok "$pkg $installed matches the pin"
  else
    fail "$pkg installed=$installed pinned=$pin_version"
  fi
done < <(sed 's/==/ ==/' requirements/requirements.txt)

echo ""
echo "2. Every published claim, reproduced directly (no pytest involved)"
DIRECT_CHECK=$("$PYTHON" - <<'PYEOF'
import sys
sys.path.insert(0, "examples")

import regression_metrics_lib as m

errors = []


def expect(label, got, want):
    if got != want:
        errors.append(f"{label}: expected {want}, got {got}")


# 1. Noise-column climb
rows = m.noise_column_r2_curve()
expect(
    "noise-column curve",
    rows,
    [
        (0, 331, 10, 0.5554, 0.5415),
        (1, 331, 11, 0.5555, 0.5402),
        (5, 331, 15, 0.5648, 0.5441),
        (20, 331, 30, 0.5754, 0.5329),
        (100, 331, 110, 0.7403, 0.6104),
    ],
)
train_r2 = [row[3] for row in rows]
if not all(a < b for a, b in zip(train_r2, train_r2[1:])):
    errors.append("train R2 was not strictly increasing in the number of noise columns")

# 1b. Adjusted R2 corrects, then breaks down
by_noise = {row[0]: row for row in rows}
if not (by_noise[20][4] < by_noise[0][4]):
    errors.append("adjusted R2 at 20 noise columns did not fall below the baseline")
if not (by_noise[100][4] > by_noise[0][4]):
    errors.append("adjusted R2 at 100 noise columns did not rise back above the baseline")

# 2 and 2b. R2's bounds
expect("full-model test R2", m.full_model_test_r2(), 0.3594)
constant = m.constant_mean_test_r2()
if abs(constant) >= 0.001:
    errors.append(f"constant-mean test R2 was not near zero: {constant}")
expect("bad-predictor test R2", m.bad_predictor_test_r2(), -4.7009)

# 3. RMSE vs MAE under an outlier
expect(
    "outlier shift",
    m.rmse_mae_outlier_shift(),
    (2.4801, 1.9833, 28.2569, 5.9448),
)

# 4. MAPE breaking
if m.mape_at_zero_target() <= 1.0e10:
    errors.append("MAPE at a zero true value did not explode as expected")
expect("MAPE near zero", m.mape_near_zero_target(), (3.3667, 5.0))
expect("MAPE asymmetry bound", m.mape_asymmetry_bound(), (1.0, 10.0))

# 5. Ranking inversion
rmse_a, mae_a, rmse_b, mae_b = m.ranking_inversion_models()
expect("ranking inversion", (rmse_a, mae_a, rmse_b, mae_b), (1.947, 1.586, 4.4353, 0.8417))
if not (rmse_a < rmse_b and mae_b < mae_a):
    errors.append("the RMSE/MAE ranking did not invert between the two models")

# 6. Units
results = m.raw_and_scaled_metrics()
expect("scaled-feature metrics", results["scaled"], (56.3929, 45.1206, 0.3594))
expect("raw-feature metrics", results["raw"], (56.3929, 45.1206, 0.3594))

# 7. r2_score agreement and argument order
expect("r2_score vs model.score", m.r2_score_vs_model_score(), (0.359409, 0.359409))
expect("r2_score argument order", m.r2_score_argument_order(), (0.359409, -0.209635))

if errors:
    for e in errors:
        print("ERROR:", e)
    sys.exit(1)
print("all direct checks passed")
PYEOF
)
if echo "$DIRECT_CHECK" | grep -q "all direct checks passed"; then
  ok "every claim reproduced directly against regression_metrics_lib, no pytest involved"
else
  fail "direct library checks failed"
  echo "$DIRECT_CHECK" | sed 's/^/    /'
fi

echo ""
echo "3. examples/ passes in full"
EXAMPLES_OUT=$("$PYTEST" examples -q 2>&1)
if echo "$EXAMPLES_OUT" | tail -1 | grep -qE "^16 passed"; then
  ok "pytest examples -q -> 16 passed"
else
  fail "pytest examples -q did not report 16 passed"
  echo "$EXAMPLES_OUT" | tail -20 | sed 's/^/    /'
fi

echo ""
echo "4. starter/ is an untouched skeleton"
STARTER_OUT=$("$PYTEST" starter -q 2>&1)
if echo "$STARTER_OUT" | tail -1 | grep -qE "4 passed, 12 skipped"; then
  ok "pytest starter -q -> 4 passed, 12 skipped (the machinery checks pass; the twelve exercises are stubs)"
else
  fail "pytest starter -q did not report 4 passed, 12 skipped"
  echo "$STARTER_OUT" | tail -20 | sed 's/^/    /'
fi

echo ""
echo "5. pytest examples starter (one invocation) aborts on the module-name collision"
COMBINED_OUT=$("$PYTEST" examples starter 2>&1)
if echo "$COMBINED_OUT" | grep -q "import file mismatch"; then
  ok "combined invocation reports import file mismatch, as documented -- never run starter and examples together"
else
  fail "combined invocation did not fail with import file mismatch as expected"
fi

echo ""
echo "6. The report reproduces the captured table exactly"
REPORT_OUT=$("$PYTHON" examples/report_measurements.py 2>&1)
if [ "$REPORT_OUT" = "$(cat expected-output/measured-values.txt)" ]; then
  ok "report_measurements.py output is byte-identical to expected-output/measured-values.txt"
else
  fail "report_measurements.py drifted from expected-output/measured-values.txt"
  echo "$REPORT_OUT" | diff - expected-output/measured-values.txt | head -20 | sed 's/^/    /'
fi

echo ""
echo "7. Proof the harness can fail"
SCRATCH=$(mktemp -d "${TMPDIR:-/tmp}/d152-scratch.XXXXXX")
cp examples/*.py "$SCRATCH"/
SCRATCH_OUT=$("$PYTEST" "$SCRATCH" -q 2>&1)
if echo "$SCRATCH_OUT" | tail -1 | grep -qE "^16 passed"; then
  ok "scratch copy of examples/ passes before it is broken"
else
  fail "scratch copy did not pass before being broken: $(echo "$SCRATCH_OUT" | tail -3)"
fi
"$PYTHON" - "$SCRATCH/test_metrics_claims.py" <<'PYEOF'
import sys
path = sys.argv[1]
text = open(path).read()
needle = "assert (rmse_a, mae_a, rmse_b, mae_b) == (1.947, 1.586, 4.4353, 0.8417)"
replacement = "assert (rmse_a, mae_a, rmse_b, mae_b) == (0.0, 0.0, 0.0, 0.0)"
assert needle in text, "could not find the assertion to break"
open(path, "w").write(text.replace(needle, replacement, 1))
PYEOF
BROKEN_OUT=$("$PYTEST" "$SCRATCH" -q 2>&1)
BROKEN_STATUS=$?
if [ "$BROKEN_STATUS" -ne 0 ] && echo "$BROKEN_OUT" | grep -q "test_06_rmse_and_mae_prefer_different_models"; then
  ok "breaking exercise 6's assertion produces a non-zero exit and names the failing test"
else
  fail "broken copy did not fail as expected (exit=$BROKEN_STATUS)"
fi
rm -rf "$SCRATCH"

echo ""
echo "8. Key results hold at seeds the lesson does not quote"
SEED_CHECK=$("$PYTHON" - <<'PYEOF'
import sys
sys.path.insert(0, "examples")
import regression_metrics_lib as m

problems = []

# The noise-column climb is not a property of one dataset seed.
for seed in (1, 2, 3):
    rows = m.noise_column_r2_curve(seed=seed)
    train_r2 = [row[3] for row in rows]
    if not (train_r2[-1] > train_r2[0]):
        problems.append(f"seed {seed}: train R2 did not climb with noise columns")

# The RMSE/MAE ranking inversion is not a property of one seed either.
for seed in (5, 6, 7):
    rmse_a, mae_a, rmse_b, mae_b = m.ranking_inversion_models(seed=seed)
    if not (rmse_a < rmse_b and mae_b < mae_a):
        problems.append(f"seed {seed}: the RMSE/MAE ranking did not invert")

# RMSE is never smaller than MAE, at other outlier shifts.
for shift in (50.0, 100.0, 500.0):
    _rb, mb, ra, ma = m.rmse_mae_outlier_shift(shift=shift)
    if not (ra >= ma):
        problems.append(f"shift {shift}: RMSE was smaller than MAE")

if problems:
    for p in problems:
        print("ERROR:", p)
else:
    print("every direction held")
PYEOF
)
if [ "$SEED_CHECK" = "every direction held" ]; then
  ok "the noise-column climb and the RMSE/MAE ranking inversion hold at seeds the lesson does not quote"
else
  fail "a direction failed beyond the quoted seed"
  echo "$SEED_CHECK" | sed 's/^/    /'
fi

echo ""
echo "9. Offline, and nothing left behind"
if ! grep -rInE "https?://" examples/*.py starter/*.py > /dev/null 2>&1; then
  ok "no URLs inside examples/ or starter/ source -- this lab reaches no network"
else
  fail "found a URL inside examples/ or starter/"
fi
if [ -z "$(find . -path ./.venv -prune -o -type d -name '__pycache__' -print 2>/dev/null)" ]; then
  ok "no __pycache__ left behind"
else
  find . -path ./.venv -prune -o -type d -name '__pycache__' -exec rm -rf -- {} + 2>/dev/null
  ok "no __pycache__ left behind (cleaned during this run)"
fi
if [ ! -d .pytest_cache ]; then
  ok "no .pytest_cache left behind"
else
  rm -rf .pytest_cache
  ok "no .pytest_cache left behind (cleaned during this run)"
fi

echo ""
echo "---------------------------------------------------------------"
echo "$CHECKS checks, $FAILURES failure(s)"
if [ "$FAILURES" -ne 0 ]; then
  exit 1
fi
exit 0