Hands-on lab — Day 141: What Machine Learning Is and Is Not

"""Machinery for Day 141 -- "What the Number Is Not Telling You".

Nothing in this file is an exercise. It builds the small, fully
deterministic datasets the nine exercises measure, and it contains one
model written by hand in NumPy so you can see that a nearest-neighbour
classifier is eleven lines of arithmetic and no magic at all.

Every dataset constructor takes an explicit integer seed and uses
`numpy.random.default_rng(seed)`, so every number this lab reports is
reproducible on any machine with the pinned versions.
"""

from __future__ import annotations

import numpy as np
from sklearn.dummy import DummyClassifier
from sklearn.linear_model import LinearRegression, LogisticRegression
from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor
from sklearn.tree import DecisionTreeClassifier

# --------------------------------------------------------------------------
# Scoring
# --------------------------------------------------------------------------


def accuracy(y_true, y_pred) -> float:
    """Fraction of predictions that match. The whole of "accuracy"."""
    y_true = np.asarray(y_true)
    y_pred = np.asarray(y_pred)
    return float(np.mean(y_true == y_pred))


def mean_absolute_error(y_true, y_pred) -> float:
    y_true = np.asarray(y_true, dtype=float)
    y_pred = np.asarray(y_pred, dtype=float)
    return float(np.mean(np.abs(y_true - y_pred)))


# --------------------------------------------------------------------------
# A nearest-neighbour classifier, written by hand
# --------------------------------------------------------------------------


class HandwrittenNearestNeighbour:
    """1-nearest-neighbour, from first principles, in NumPy.

    `fit` stores the training set. That is the entire training procedure:
    there is no search, no objective, no parameters. `predict` finds the
    closest stored point to each query and copies its label.

    This class exists to make one fact concrete: predicting a training
    point returns that point's own label, because a point's own distance
    to itself is zero. Training accuracy is therefore 1.000 by
    construction and carries no information whatsoever.
    """

    def __init__(self) -> None:
        self.X_: np.ndarray | None = None
        self.y_: np.ndarray | None = None

    def fit(self, X, y) -> "HandwrittenNearestNeighbour":
        self.X_ = np.asarray(X, dtype=float)
        self.y_ = np.asarray(y)
        return self

    def predict(self, X) -> np.ndarray:
        if self.X_ is None or self.y_ is None:
            raise RuntimeError("call fit before predict")
        X = np.asarray(X, dtype=float)
        # Squared euclidean distance from every query row to every stored
        # row, by broadcasting: (n_query, 1, n_features) - (n_train, n_features)
        diff = X[:, None, :] - self.X_[None, :, :]
        sq_dist = np.sum(diff * diff, axis=2)
        nearest = np.argmin(sq_dist, axis=1)
        return self.y_[nearest]


# --------------------------------------------------------------------------
# Datasets -- every one deterministic given its seed
# --------------------------------------------------------------------------


def pure_noise_dataset(n: int, n_features: int = 4, seed: int = 141):
    """Features from a normal distribution, labels from a coin flip.

    There is no relationship of any kind between X and y. No function
    exists to be approximated, so the best possible test accuracy is
    chance.
    """
    rng = np.random.default_rng(seed)
    X = rng.normal(size=(n, n_features))
    y = rng.integers(0, 2, size=n)
    return X, y


def rule_dataset(n: int, seed: int, offset: float = 0.0):
    """Two uniform features on a square of side 1, labelled by an exact rule.

    The rule is `y = 1 if x1 > x0 else 0` -- the diagonal of the square.
    `offset` translates the whole square without changing the rule, which
    is what makes this dataset usable as a distribution shift: the
    labelling function is identical, only the region the points live in
    has moved.
    """
    rng = np.random.default_rng(seed)
    X = rng.uniform(0.0, 1.0, size=(n, 2)) + offset
    y = exact_rule(X)
    return X, y


def exact_rule(X) -> np.ndarray:
    """The three-line rule that `rule_dataset` labels with.

    It is exactly correct everywhere, for every input, forever, and it
    needs no data, no training and no maintenance.
    """
    X = np.asarray(X, dtype=float)
    return (X[:, 1] > X[:, 0]).astype(int)


def flip_labels(y, noise_rate: float, seed: int):
    """Flip exactly `round(noise_rate * len(y))` labels, chosen at random.

    Exactly, not approximately: the count is fixed so the resulting
    ceiling is an exact arithmetic fact about the data rather than a
    sampled quantity.
    """
    y = np.asarray(y).copy()
    rng = np.random.default_rng(seed)
    n_flip = int(round(noise_rate * len(y)))
    idx = rng.choice(len(y), size=n_flip, replace=False)
    y[idx] = 1 - y[idx]
    return y


def noisy_rule_dataset(n: int, seed: int, noise_rate: float):
    """`rule_dataset` with a known fraction of its labels flipped."""
    X, y_clean = rule_dataset(n, seed=seed)
    y = flip_labels(y_clean, noise_rate=noise_rate, seed=seed + 9000)
    return X, y


def checkerboard_dataset(n: int, seed: int, cells: int = 4):
    """A clean but intricate boundary: a `cells` x `cells` checkerboard.

    The labels contain no noise at all, so nothing limits a model here
    except how much of the boundary the training sample reveals. This is
    the variance-limited problem that more data genuinely fixes.
    """
    rng = np.random.default_rng(seed)
    X = rng.uniform(0.0, 1.0, size=(n, 2))
    y = ((np.floor(X[:, 0] * cells) + np.floor(X[:, 1] * cells)) % 2).astype(int)
    return X, y


def imbalanced_noise_dataset(n: int, seed: int, minority_rate: float = 0.1):
    """Pure-noise features with an exact minority-class count.

    Exactly `round(minority_rate * n)` rows carry label 1, placed at
    random positions, so the majority-class baseline on this set is an
    exact number rather than an estimate.
    """
    rng = np.random.default_rng(seed)
    X = rng.normal(size=(n, 6))
    y = np.zeros(n, dtype=int)
    n_minority = int(round(minority_rate * n))
    y[rng.choice(n, size=n_minority, replace=False)] = 1
    return X, y


def quadratic_curve(n: int, low: float, high: float, seed: int):
    """One feature on [low, high], target y = x squared. No noise."""
    rng = np.random.default_rng(seed)
    x = np.sort(rng.uniform(low, high, size=n))
    X = x.reshape(-1, 1)
    y = x**2
    return X, y


# --------------------------------------------------------------------------
# Model constructors -- fixed hyper-parameters so results never drift
# --------------------------------------------------------------------------


def one_nn() -> KNeighborsClassifier:
    return KNeighborsClassifier(n_neighbors=1)


def shallow_tree(max_depth: int = 3) -> DecisionTreeClassifier:
    return DecisionTreeClassifier(max_depth=max_depth, random_state=141)


def deep_tree() -> DecisionTreeClassifier:
    return DecisionTreeClassifier(random_state=141)


def smooth_knn(k: int = 15) -> KNeighborsClassifier:
    return KNeighborsClassifier(n_neighbors=k)


def linear_classifier() -> LogisticRegression:
    return LogisticRegression(max_iter=1000)


def majority_baseline() -> DummyClassifier:
    return DummyClassifier(strategy="most_frequent")


def knn_regressor(k: int = 5) -> KNeighborsRegressor:
    return KNeighborsRegressor(n_neighbors=k)


def linear_regressor() -> LinearRegression:
    return LinearRegression()


def fit_score(model, X_train, y_train, X_test, y_test) -> float:
    """Fit on the training set, score on the test set. Nothing else."""
    model.fit(X_train, y_train)
    return accuracy(y_test, model.predict(X_test))


# --------------------------------------------------------------------------
# Exercise 9: the decision function
# --------------------------------------------------------------------------

#: The four questions, in the order they must be asked. Cheapness first:
#: a question whose answer disqualifies machine learning outright is
#: worth asking before one that merely constrains it.
ML_DECISION_QUESTIONS = (
    "exact_rule_exists",
    "labels_available",
    "distribution_stable",
    "errors_tolerable",
)


def should_use_ml(problem: dict) -> str:
    """Return one of five verdicts for a described problem.

    `problem` must carry all four keys in `ML_DECISION_QUESTIONS` with
    boolean values. The order of the checks is the argument:

    1. `exact_rule_exists` -- if you can write the rule down, write it.
       A rule is exactly correct, costs nothing to run, needs no labels,
       never drifts and can be reviewed by a person who is not you.
       No model beats that, and a model that merely matches it has cost
       you a data pipeline for nothing.
    2. `labels_available` -- supervised learning approximates a function
       from examples of its output. Without labels there are no examples,
       and no amount of feature work substitutes for them.
    3. `distribution_stable` -- the method assumes future inputs resemble
       training inputs. If the world moves faster than you can retrain,
       the model is wrong by the time it ships.
    4. `errors_tolerable` -- a model is an approximation and will be
       wrong on some inputs. If a single wrong answer is unacceptable and
       cannot be caught downstream, an approximation is the wrong shape
       of tool regardless of its accuracy.
    """
    missing = [q for q in ML_DECISION_QUESTIONS if q not in problem]
    if missing:
        raise KeyError(f"problem is missing: {', '.join(missing)}")
    if problem["exact_rule_exists"]:
        return "write the rule"
    if not problem["labels_available"]:
        return "get labels first"
    if not problem["distribution_stable"]:
        return "not yet: the distribution moves"
    if not problem["errors_tolerable"]:
        return "no: errors are not tolerable"
    return "yes"


def problem(
    exact_rule_exists: bool,
    labels_available: bool,
    distribution_stable: bool,
    errors_tolerable: bool,
) -> dict:
    """Small helper so the case table in the tests reads as prose."""
    return {
        "exact_rule_exists": exact_rule_exists,
        "labels_available": labels_available,
        "distribution_stable": distribution_stable,
        "errors_tolerable": errors_tolerable,
    }

"""Print every measured pair this lab asserts, as one table.

Run it with:

    .venv/bin/python examples/report_measurements.py

It imports nothing the tests do not import and computes nothing the
tests do not compute; it exists so you can read the numbers without
reading the assertions.
"""

import sys
from pathlib import Path

import numpy as np

sys.path.insert(0, str(Path(__file__).resolve().parent))

import ml_lib as m  # noqa: E402
from sklearn.datasets import load_iris  # noqa: E402


def line(label: str, value) -> None:
    print(f"  {label:<46} {value}")


def main() -> None:
    print("Day 141 -- What the Number Is Not Telling You")
    print("Every value below is measured, not quoted.")
    print()

    print("1. Perfect accuracy, zero learning (labels are coin flips)")
    X_tr, y_tr = m.pure_noise_dataset(200, seed=141)
    X_te, y_te = m.pure_noise_dataset(1000, seed=242)
    hand = m.HandwrittenNearestNeighbour().fit(X_tr, y_tr)
    line("1-NN training accuracy (hand-written)", m.accuracy(y_tr, hand.predict(X_tr)))
    line("1-NN test accuracy (1000 unseen rows)", m.accuracy(y_te, hand.predict(X_te)))
    sk = m.one_nn().fit(X_tr, y_tr)
    line("1-NN training accuracy (scikit-learn)", m.accuracy(y_tr, sk.predict(X_tr)))
    line("1-NN test accuracy (scikit-learn)", m.accuracy(y_te, sk.predict(X_te)))
    X_iris, y_iris = load_iris(return_X_y=True)
    line("iris unique feature rows out of 150", len({tuple(r) for r in X_iris}))
    scrambled = np.random.default_rng(141).permutation(y_iris)
    hand_iris = m.HandwrittenNearestNeighbour().fit(X_iris, scrambled)
    line(
        "1-NN train accuracy, iris, scrambled labels",
        m.accuracy(scrambled, hand_iris.predict(X_iris)),
    )
    print()

    print("2. A rule beats a model")
    X_tr, y_tr = m.rule_dataset(300, seed=11)
    X_te, y_te = m.rule_dataset(2000, seed=12)
    line("three-line rule, test accuracy", m.accuracy(y_te, m.exact_rule(X_te)))
    for name, model in (
        ("depth-3 tree", m.shallow_tree(3)),
        ("depth-8 tree", m.shallow_tree(8)),
        ("full-depth tree", m.deep_tree()),
        ("15-NN", m.smooth_knn(15)),
    ):
        line(f"{name}, test accuracy", m.fit_score(model, X_tr, y_tr, X_te, y_te))
    print()

    print("3. The generalisation gap")
    perm = np.random.default_rng(141).permutation(len(y_iris))
    tr, te = perm[:100], perm[100:]
    tree = m.deep_tree().fit(X_iris[tr], y_iris[tr])
    a = m.accuracy(y_iris[tr], tree.predict(X_iris[tr]))
    b = m.accuracy(y_iris[te], tree.predict(X_iris[te]))
    line("iris full-depth tree, train accuracy", a)
    line("iris full-depth tree, test accuracy", b)
    line("iris gap", round(a - b, 4))
    X_tr, y_tr = m.noisy_rule_dataset(300, seed=21, noise_rate=0.2)
    X_te, y_te = m.noisy_rule_dataset(2000, seed=22, noise_rate=0.2)
    noisy = m.deep_tree().fit(X_tr, y_tr)
    c = m.accuracy(y_tr, noisy.predict(X_tr))
    d = m.accuracy(y_te, noisy.predict(X_te))
    line("noisy rule, train accuracy", c)
    line("noisy rule, test accuracy", d)
    line("noisy rule gap", round(c - d, 4))
    simple = m.linear_classifier().fit(X_tr, y_tr)
    line("same data, simple model, train accuracy", m.accuracy(y_tr, simple.predict(X_tr)))
    line("same data, simple model, test accuracy", m.accuracy(y_te, simple.predict(X_te)))
    print()

    print("4. Distribution shift (same rule, region translated by 3.0)")
    X_tr, y_tr = m.rule_dataset(400, seed=31)
    X_in, y_in = m.rule_dataset(2000, seed=32)
    X_sh, y_sh = m.rule_dataset(2000, seed=33, offset=3.0)
    tree = m.deep_tree().fit(X_tr, y_tr)
    line("in-distribution test accuracy", m.accuracy(y_in, tree.predict(X_in)))
    line("shifted test accuracy", m.accuracy(y_sh, tree.predict(X_sh)))
    line("the rule, on the shifted region", m.accuracy(y_sh, m.exact_rule(X_sh)))
    print()

    print("5. Interpolation versus extrapolation (y = x squared)")
    X_tr, y_tr = m.quadratic_curve(300, 0.0, 10.0, seed=41)
    X_in, y_in = m.quadratic_curve(200, 0.0, 10.0, seed=42)
    X_out, y_out = m.quadratic_curve(200, 10.0, 20.0, seed=43)
    knn = m.knn_regressor(5).fit(X_tr, y_tr)
    line("5-NN mean absolute error inside [0, 10]",
         round(m.mean_absolute_error(y_in, knn.predict(X_in)), 3))
    line("5-NN mean absolute error outside, [10, 20]",
         round(m.mean_absolute_error(y_out, knn.predict(X_out)), 3))
    line("5-NN largest prediction outside the range",
         round(float(np.max(knn.predict(X_out))), 3))
    line("largest target value ever seen in training",
         round(float(np.max(y_tr)), 3))
    lin = m.linear_regressor().fit(X_tr, y_tr)
    line("linear model, error inside the range",
         round(m.mean_absolute_error(y_in, lin.predict(X_in)), 3))
    line("linear model, error outside the range",
         round(m.mean_absolute_error(y_out, lin.predict(X_out)), 3))
    print()

    print("6. The baseline (90 percent class 0, features are pure noise)")
    X_tr, y_tr = m.imbalanced_noise_dataset(1000, seed=51)
    X_te, y_te = m.imbalanced_noise_dataset(1000, seed=52)
    line("majority-class baseline", m.fit_score(m.majority_baseline(), X_tr, y_tr, X_te, y_te))
    line("1-NN", m.fit_score(m.one_nn(), X_tr, y_tr, X_te, y_te))
    line("full-depth tree", m.fit_score(m.deep_tree(), X_tr, y_tr, X_te, y_te))
    line("iris majority-class baseline",
         m.fit_score(m.majority_baseline(), X_iris[tr], y_iris[tr], X_iris[te], y_iris[te]))
    line("iris 1-NN",
         m.fit_score(m.one_nn(), X_iris[tr], y_iris[tr], X_iris[te], y_iris[te]))
    print()

    print("7. The label-noise ceiling (exactly 25 percent of labels flipped)")
    X_tr, y_tr = m.noisy_rule_dataset(2000, seed=61, noise_rate=0.25)
    X_te, y_te = m.noisy_rule_dataset(4000, seed=62, noise_rate=0.25)
    line("the ceiling, 1 - noise_rate", 0.75)
    for name, model in (
        ("logistic regression", m.linear_classifier()),
        ("15-NN", m.smooth_knn(15)),
        ("depth-3 tree", m.shallow_tree(3)),
        ("full-depth tree", m.deep_tree()),
    ):
        line(f"{name}, test accuracy", m.fit_score(model, X_tr, y_tr, X_te, y_te))
    print()

    print("8. More data does not fix the wrong thing")
    X_te_c, y_te_c = m.checkerboard_dataset(4000, seed=71)
    small = m.fit_score(m.deep_tree(), *m.checkerboard_dataset(50, seed=120), X_te_c, y_te_c)
    large = m.fit_score(m.deep_tree(), *m.checkerboard_dataset(5000, seed=5070), X_te_c, y_te_c)
    line("variance-limited, n=50", small)
    line("variance-limited, n=5000", large)
    line("gain from 100x more data", round(large - small, 5))
    X_te_n, y_te_n = m.noisy_rule_dataset(4000, seed=81, noise_rate=0.30)
    few = m.fit_score(
        m.linear_classifier(), *m.noisy_rule_dataset(200, seed=280, noise_rate=0.30),
        X_te_n, y_te_n)
    many = m.fit_score(
        m.linear_classifier(), *m.noisy_rule_dataset(5000, seed=5080, noise_rate=0.30),
        X_te_n, y_te_n)
    line("noise-limited, n=200", few)
    line("noise-limited, n=5000", many)
    line("gain from 25x more data", round(many - few, 5))
    line("noise-limited ceiling", 0.7)
    print()

    print("9. should_use_ml, on six described problems")
    table = [
        ("VAT at a published rate", m.problem(True, True, True, True)),
        ("a rule exists, nothing else does", m.problem(True, False, False, False)),
        ("unlabelled support tickets", m.problem(False, False, True, True)),
        ("adaptive payment fraud", m.problem(False, True, False, True)),
        ("unsupervised dosing decision", m.problem(False, True, True, False)),
        ("handwritten postcodes", m.problem(False, True, True, True)),
    ]
    for label, case in table:
        line(label, m.should_use_ml(case))


if __name__ == "__main__":
    main()

"""Nine measured claims about what an accuracy number is not telling you.

Every assertion below is on a value or a shape that was measured on a
real run, never on a timing. Every dataset is seeded, so the numbers in
the comments are the numbers you will see.
"""

import numpy as np

import ml_lib as m

# --------------------------------------------------------------------------
# 1. Perfect accuracy, zero learning
# --------------------------------------------------------------------------


def test_01_one_nn_scores_a_perfect_1_000_having_learned_nothing():
    """Labels are coin flips. There is no function to approximate.

    Measured: training accuracy 1.000 for both the hand-written model and
    scikit-learn's; test accuracy 0.518 on 1000 unseen rows, which is
    chance.
    """
    X_train, y_train = m.pure_noise_dataset(200, seed=141)
    X_test, y_test = m.pure_noise_dataset(1000, seed=242)

    handwritten = m.HandwrittenNearestNeighbour().fit(X_train, y_train)
    train_acc = m.accuracy(y_train, handwritten.predict(X_train))
    test_acc = m.accuracy(y_test, handwritten.predict(X_test))

    assert train_acc == 1.0
    assert test_acc == 0.518
    assert abs(test_acc - 0.5) < 0.06  # chance, within sampling noise

    # scikit-learn's KNeighborsClassifier(n_neighbors=1) agrees exactly.
    library = m.one_nn().fit(X_train, y_train)
    assert m.accuracy(y_train, library.predict(X_train)) == 1.0
    assert m.accuracy(y_test, library.predict(X_test)) == test_acc


def test_01b_the_only_way_a_1_nn_misses_a_training_row_is_a_duplicate():
    """The "1.000 by construction" claim has exactly one exception.

    A training row is its own nearest neighbour at distance zero -- unless
    an identical feature row carries a different label, in which case the
    tie can be broken the wrong way. The iris dataset contains exactly one
    duplicated feature row (positions 101 and 142). Both carry class 2, so
    on the real labels 1-NN still scores 1.000; scramble the labels and the
    same pair drops it to 0.99. Measured, not assumed.
    """
    from sklearn.datasets import load_iris

    X, y = load_iris(return_X_y=True)
    unique_rows = {tuple(row) for row in X}
    assert X.shape == (150, 4)
    assert len(unique_rows) == 149  # one duplicated feature row

    on_real_labels = m.HandwrittenNearestNeighbour().fit(X, y)
    assert m.accuracy(y, on_real_labels.predict(X)) == 1.0

    scrambled = np.random.default_rng(141).permutation(y)
    on_scrambled = m.HandwrittenNearestNeighbour().fit(X, scrambled)
    assert m.accuracy(scrambled, on_scrambled.predict(X)) < 1.0


# --------------------------------------------------------------------------
# 2. A rule beats a model
# --------------------------------------------------------------------------


def test_02_a_three_line_rule_scores_1_000_and_every_model_scores_less():
    """`y = 1 if x1 > x0 else 0`. Three lines, exactly correct, forever.

    Measured on 2000 unseen rows: the rule 1.000; a depth-3 tree 0.8855;
    the best of four trained models (15-NN) 0.9675. None reaches the rule.
    """
    X_train, y_train = m.rule_dataset(300, seed=11)
    X_test, y_test = m.rule_dataset(2000, seed=12)

    rule_acc = m.accuracy(y_test, m.exact_rule(X_test))
    assert rule_acc == 1.0

    model_acc = m.fit_score(m.shallow_tree(3), X_train, y_train, X_test, y_test)
    assert model_acc == 0.8855
    assert model_acc < rule_acc

    scores = {
        "depth-3 tree": model_acc,
        "depth-8 tree": m.fit_score(m.shallow_tree(8), X_train, y_train, X_test, y_test),
        "full-depth tree": m.fit_score(m.deep_tree(), X_train, y_train, X_test, y_test),
        "15-NN": m.fit_score(m.smooth_knn(15), X_train, y_train, X_test, y_test),
    }
    assert max(scores.values()) == 0.9675
    assert all(score < rule_acc for score in scores.values())


# --------------------------------------------------------------------------
# 3. The generalisation gap
# --------------------------------------------------------------------------


def test_03_train_accuracy_exceeds_test_accuracy_and_the_gap_is_the_story():
    """Two gaps, both measured: a small honest one and a large one.

    iris, full-depth tree: train 1.000, test 0.960, gap 0.040.
    Noisy constructed data, same model: train 1.000, test 0.6535,
    gap 0.3465. Identical training score, wildly different models.
    """
    from sklearn.datasets import load_iris

    X, y = load_iris(return_X_y=True)
    perm = np.random.default_rng(141).permutation(len(y))
    train_idx, test_idx = perm[:100], perm[100:]

    tree = m.deep_tree().fit(X[train_idx], y[train_idx])
    iris_train = m.accuracy(y[train_idx], tree.predict(X[train_idx]))
    iris_test = m.accuracy(y[test_idx], tree.predict(X[test_idx]))
    assert iris_train == 1.0
    assert iris_test == 0.96
    assert iris_train > iris_test
    assert round(iris_train - iris_test, 4) == 0.04

    X_train, y_train = m.noisy_rule_dataset(300, seed=21, noise_rate=0.2)
    X_test, y_test = m.noisy_rule_dataset(2000, seed=22, noise_rate=0.2)
    noisy = m.deep_tree().fit(X_train, y_train)
    noisy_train = m.accuracy(y_train, noisy.predict(X_train))
    noisy_test = m.accuracy(y_test, noisy.predict(X_test))
    assert noisy_train == 1.0
    assert noisy_test == 0.6535
    assert round(noisy_train - noisy_test, 4) == 0.3465

    # Same perfect training score, gap eight times larger.
    assert noisy_train == iris_train
    assert (noisy_train - noisy_test) > 8 * (iris_train - iris_test)

    # And the sentence the whole day turns on: on this same data a much
    # simpler model scores WORSE in training (0.780 against 1.000) and
    # BETTER on unseen data (0.7655 against 0.6535). The better training
    # score belongs to the worse model.
    simple = m.linear_classifier().fit(X_train, y_train)
    simple_train = m.accuracy(y_train, simple.predict(X_train))
    simple_test = m.accuracy(y_test, simple.predict(X_test))
    assert simple_train == 0.78
    assert simple_test == 0.7655
    assert simple_train < noisy_train
    assert simple_test > noisy_test


# --------------------------------------------------------------------------
# 4. Distribution shift
# --------------------------------------------------------------------------


def test_04_accuracy_collapses_when_the_input_region_moves():
    """Same labelling rule, different region of feature space.

    Trained on the unit square, the tree scores 0.948 on unseen points
    from the unit square and 0.4895 -- below chance -- on the identical
    problem translated by 3.0. The rule scores 1.000 on both.
    """
    X_train, y_train = m.rule_dataset(400, seed=31)
    X_in, y_in = m.rule_dataset(2000, seed=32)
    X_shifted, y_shifted = m.rule_dataset(2000, seed=33, offset=3.0)

    tree = m.deep_tree().fit(X_train, y_train)
    in_dist = m.accuracy(y_in, tree.predict(X_in))
    shifted = m.accuracy(y_shifted, tree.predict(X_shifted))

    assert in_dist == 0.948
    assert shifted == 0.4895
    assert shifted < 0.55  # at or below chance
    assert in_dist - shifted > 0.4

    # The model was never told the region had moved, and could not have
    # been: nothing in the training data describes where it ends.
    assert m.accuracy(y_shifted, m.exact_rule(X_shifted)) == 1.0


# --------------------------------------------------------------------------
# 5. Interpolation versus extrapolation
# --------------------------------------------------------------------------


def test_05_a_model_interpolates_beautifully_and_extrapolates_not_at_all():
    """y = x squared, learned on [0, 10], asked about [10, 20].

    Measured mean absolute error: 0.180 inside the training range,
    139.704 outside it -- 774 times worse. This is not a bug in the
    model. A nearest-neighbour regressor has nothing outside its range
    but the edge of what it saw.
    """
    X_train, y_train = m.quadratic_curve(300, 0.0, 10.0, seed=41)
    X_in, y_in = m.quadratic_curve(200, 0.0, 10.0, seed=42)
    X_out, y_out = m.quadratic_curve(200, 10.0, 20.0, seed=43)

    model = m.knn_regressor(5).fit(X_train, y_train)
    mae_in = m.mean_absolute_error(y_in, model.predict(X_in))
    mae_out = m.mean_absolute_error(y_out, model.predict(X_out))

    assert round(mae_in, 3) == 0.180
    assert round(mae_out, 3) == 139.704
    assert mae_out > 700 * mae_in

    # Its predictions outside the range are bounded by what it has seen.
    assert float(np.max(model.predict(X_out))) <= float(np.max(y_train))

    # A linear model extrapolates differently -- and still badly, because
    # the truth is a parabola: 6.007 inside, 101.643 outside.
    linear = m.linear_regressor().fit(X_train, y_train)
    assert round(m.mean_absolute_error(y_in, linear.predict(X_in)), 3) == 6.007
    assert round(m.mean_absolute_error(y_out, linear.predict(X_out)), 3) == 101.643


# --------------------------------------------------------------------------
# 6. The baseline
# --------------------------------------------------------------------------


def test_06_a_good_looking_accuracy_that_loses_to_predicting_the_majority():
    """90 percent of rows are class 0 and the features are pure noise.

    The majority-class baseline scores exactly 0.900. A 1-NN scores
    0.821 and a full-depth tree 0.817. Both would be reported as "82
    percent accurate" and both are worse than a constant.
    """
    X_train, y_train = m.imbalanced_noise_dataset(1000, seed=51)
    X_test, y_test = m.imbalanced_noise_dataset(1000, seed=52)

    baseline = m.fit_score(m.majority_baseline(), X_train, y_train, X_test, y_test)
    assert baseline == 0.9
    assert float(np.mean(y_test == 0)) == 0.9  # exact by construction

    one_nn = m.fit_score(m.one_nn(), X_train, y_train, X_test, y_test)
    tree = m.fit_score(m.deep_tree(), X_train, y_train, X_test, y_test)
    assert one_nn == 0.821
    assert tree == 0.817
    assert one_nn < baseline and tree < baseline

    # On a problem where the features do carry signal, the same comparison
    # is the one that shows it: iris baseline 0.260, 1-NN 0.980.
    from sklearn.datasets import load_iris

    X, y = load_iris(return_X_y=True)
    perm = np.random.default_rng(141).permutation(len(y))
    tr, te = perm[:100], perm[100:]
    iris_baseline = m.fit_score(m.majority_baseline(), X[tr], y[tr], X[te], y[te])
    iris_model = m.fit_score(m.one_nn(), X[tr], y[tr], X[te], y[te])
    assert iris_baseline == 0.26
    assert iris_model == 0.98
    assert iris_model > iris_baseline


# --------------------------------------------------------------------------
# 7. The irreducible error ceiling
# --------------------------------------------------------------------------


def test_07_no_model_beats_the_label_noise_ceiling():
    """Exactly 25 percent of labels are flipped, in training and in test.

    A perfect model of the underlying rule would score 0.750 on this test
    set, because a quarter of its correct answers are marked wrong. That
    is the ceiling. Measured: logistic regression 0.73725, 15-NN 0.72675,
    depth-3 tree 0.68825, full-depth tree 0.60875. None exceeds 0.750.
    """
    noise_rate = 0.25
    ceiling = 1.0 - noise_rate
    X_train, y_train = m.noisy_rule_dataset(2000, seed=61, noise_rate=noise_rate)
    X_test, y_test = m.noisy_rule_dataset(4000, seed=62, noise_rate=noise_rate)

    scores = {
        "logistic regression": m.fit_score(
            m.linear_classifier(), X_train, y_train, X_test, y_test
        ),
        "15-NN": m.fit_score(m.smooth_knn(15), X_train, y_train, X_test, y_test),
        "depth-3 tree": m.fit_score(m.shallow_tree(3), X_train, y_train, X_test, y_test),
        "full-depth tree": m.fit_score(m.deep_tree(), X_train, y_train, X_test, y_test),
    }
    assert scores["logistic regression"] == 0.73725
    assert scores["15-NN"] == 0.72675
    assert scores["depth-3 tree"] == 0.68825
    assert scores["full-depth tree"] == 0.60875
    for name, score in scores.items():
        assert score <= ceiling, f"{name} scored {score}, above the ceiling {ceiling}"

    # The best model is within 1.3 points of the ceiling. The remaining
    # 26.3 points are not available to any model, however large.
    assert ceiling - max(scores.values()) < 0.02

    # The exact flip count is what makes the ceiling exact rather than
    # estimated: 1000 of 4000 test labels.
    _, y_clean = m.rule_dataset(4000, seed=62)
    assert int(np.sum(y_clean != y_test)) == 1000


# --------------------------------------------------------------------------
# 8. More data does not fix the wrong thing
# --------------------------------------------------------------------------


def test_08_more_data_fixes_variance_and_does_nothing_for_label_noise():
    """Two problems, one hundredfold and one twenty-fivefold increase.

    Variance-limited (a clean 4x4 checkerboard boundary, full-depth
    tree): 0.5995 at n=50, 0.99725 at n=5000 -- a gain of 39.8 points.
    Noise-limited (a linearly separable rule with 30 percent of labels
    flipped, logistic regression): 0.6655 at n=200, 0.68675 at n=5000 --
    a gain of 2.1 points against a ceiling of 0.700, which it was already
    within 3.5 points of at n=200.
    """
    X_test_c, y_test_c = m.checkerboard_dataset(4000, seed=71)
    small = m.fit_score(
        m.deep_tree(), *m.checkerboard_dataset(50, seed=120), X_test_c, y_test_c
    )
    large = m.fit_score(
        m.deep_tree(), *m.checkerboard_dataset(5000, seed=5070), X_test_c, y_test_c
    )
    assert small == 0.5995
    assert large == 0.99725
    assert large - small > 0.30

    noise_rate = 0.30
    X_test_n, y_test_n = m.noisy_rule_dataset(4000, seed=81, noise_rate=noise_rate)
    few = m.fit_score(
        m.linear_classifier(),
        *m.noisy_rule_dataset(200, seed=280, noise_rate=noise_rate),
        X_test_n,
        y_test_n,
    )
    many = m.fit_score(
        m.linear_classifier(),
        *m.noisy_rule_dataset(5000, seed=5080, noise_rate=noise_rate),
        X_test_n,
        y_test_n,
    )
    assert few == 0.6655
    assert many == 0.68675
    assert many - few < 0.05

    # The honest statement: the noise-limited model does improve, by 2.1
    # points, because n=200 is a small sample. What it cannot do is cross
    # its ceiling, and it starts 3.5 points below it.
    ceiling = 1.0 - noise_rate
    assert many < ceiling
    assert ceiling - few < 0.05
    assert (large - small) > 15 * (many - few)


# --------------------------------------------------------------------------
# 9. The decision function
# --------------------------------------------------------------------------


def test_09_should_use_ml_gives_the_verdict_the_case_deserves():
    """A table of cases, each justified in a comment.

    The order of the questions is the point: the cheapest disqualifier is
    asked first.
    """
    cases = [
        # Value-added tax on a known rate table. The rule is law, written
        # down, and a model that approximates it is a compliance defect.
        (m.problem(True, True, True, True), "write the rule"),
        # A rule exists but nothing else does. Still write the rule: the
        # rule is exactly correct without any of the rest.
        (m.problem(True, False, False, False), "write the rule"),
        # Sentiment of free-text support tickets, none of them labelled.
        # No labels, no supervised learning. Get labels first.
        (m.problem(False, False, True, True), "get labels first"),
        # Fraud patterns in a payment network where adversaries adapt
        # weekly. Labels exist, but yesterday's distribution is gone.
        (m.problem(False, True, False, True), "not yet: the distribution moves"),
        # An automated dosing decision with no human in the loop, where a
        # single wrong answer is not recoverable.
        (m.problem(False, True, True, False), "no: errors are not tolerable"),
        # Handwritten postcode recognition: no rule anyone can write,
        # millions of labelled examples, a stable distribution, and a
        # wrong read costs one redirected letter.
        (m.problem(False, True, True, True), "yes"),
    ]
    for case, expected in cases:
        assert m.should_use_ml(case) == expected, case

    # A missing question is an error, not a default. You do not get to
    # skip one of the four.
    try:
        m.should_use_ml({"exact_rule_exists": False})
    except KeyError as error:
        assert "labels_available" in str(error)
    else:
        raise AssertionError("should_use_ml accepted an incomplete problem")

"""Three checks on the machinery itself, so a failure elsewhere is a
failure of the claim under test and not of the helpers.

These three are already solved in `starter/` too: they are not exercises.
"""

import numpy as np

import ml_lib as m


def test_accuracy_counts_matches():
    assert m.accuracy([0, 1, 1, 0], [0, 1, 0, 0]) == 0.75
    assert m.accuracy([1, 1], [1, 1]) == 1.0
    assert m.accuracy([1, 1], [0, 0]) == 0.0


def test_exact_rule_is_exactly_correct_on_its_own_data():
    for seed in (1, 2, 3):
        X, y = m.rule_dataset(500, seed=seed)
        assert m.accuracy(y, m.exact_rule(X)) == 1.0


def test_flip_labels_flips_an_exact_count():
    _, y = m.rule_dataset(1000, seed=7)
    flipped = m.flip_labels(y, noise_rate=0.25, seed=7)
    assert int(np.sum(y != flipped)) == 250
    # Flipping is a relabelling, not a resampling: the feature matrix and
    # the array length are untouched.
    assert flipped.shape == y.shape

lesson_id: D141
day: 141
kind: guided-build
languages:
  - python
  - bash
setup_commands:
  - cd labs/sections/machine-learning/day-141-what-machine-learning-is-and-is
  - python3 -m venv .venv
  - .venv/bin/pip install -r requirements/requirements.txt
  - >-
    .venv/bin/python3 -c "import numpy, sklearn; print(numpy.__version__,
    sklearn.__version__)"
run_commands:
  - .venv/bin/pytest examples -q
  - .venv/bin/pytest starter -q
  - .venv/bin/python3 examples/report_measurements.py
test_commands:
  - bash tests/run_tests.sh
cleanup_commands:
  - >-
    find . -path ./.venv -prune -o -type d -name '__pycache__' -print -exec rm -rf -- {}
    +
  - rm -rf .pytest_cache
  - 'rm -rf .venv  # optional: removes the lab virtual environment'
  - 'git checkout -- starter/  # optional: reset your work'
requires_network: true
requires_api_key: false
estimated_minutes: 55
last_executed: '2026-08-24'
executed_on: >-
  macOS 26.5.2 (Apple Silicon, arm64), Python 3.14.0, numpy 2.5.2, scikit-learn 1.9.0,
  pytest 9.1.1, bash 3.2.57 -- bash tests/run_tests.sh -> 13 checks, 0 failure(s), exit
  0. pytest examples -q -> 13 passed. pytest starter -q -> 3 passed, 10 skipped (the
  three machinery checks in test_ml_lib.py are solved in both directories; the ten
  exercise stubs in starter/test_ml_claims.py are untouched). Everything ran through a
  real lab-local .venv created by the documented setup commands; scikit-learn pulled in
  scipy 1.18.1, joblib 1.5.3 and threadpoolctl 3.6.0 as its own dependencies, none of
  which this lab imports directly. After the pip install the lab is fully offline: the
  iris measurements come from a copy bundled inside the installed scikit-learn package,
  every other dataset is generated on the spot from a seeded
  numpy.random.default_rng, and harness check 11 confirms no URL appears anywhere in
  starter/ or examples/ source. Section 7 of the harness copies examples/ into a
  mktemp-d scratch directory, confirms 13 passed, rewrites `assert train_acc == 1.0` to
  `assert train_acc == 0.5`, confirms a non-zero exit naming the failing test, and
  removes the scratch directory. Separately, by hand, `assert test_acc == 0.518` was
  changed to 0.999 in examples/test_ml_claims.py and the whole harness re-run: it
  reported 13 checks, 2 failure(s) and exited 1 (both the pytest run and the
  pytest-free direct reproduction in section 2 caught it); the file was restored and the
  harness returned to 13 checks, 0 failure(s), exit 0. MEASURED PAIRS, all captured
  verbatim in expected-output/measured-values.txt. (1) A 1-NN fitted to 200 rows whose
  labels are coin flips scores exactly 1.000 on its training set -- hand-written NumPy
  and scikit-learn agreeing to the digit -- and 0.518 on 1000 unseen rows, which is
  chance. (2) An exact three-line rule scores 1.000 on 2000 unseen rows; a depth-3 tree
  scores 0.8855 and the best of four trained models (15-NN) 0.9675, so every model loses
  to the rule. (3) Generalisation gaps: iris full-depth tree train 1.000 / test 0.960,
  gap 0.040; the same model on 20-percent-noisy constructed data train 1.000 / test
  0.6535, gap 0.3465 -- identical training scores, gaps differing by a factor of more
  than eight; and on that same noisy data a much simpler model (logistic regression)
  scores WORSE in training, 0.780, and BETTER on unseen data, 0.7655 -- the better
  training score belongs to the worse model, measured rather than asserted. (4)
  Distribution shift: 0.948 in-distribution, 0.4895 on the identical
  problem translated by 3.0, which is below chance, while the rule scores 1.000 on both.
  (5) Extrapolation: a 5-NN regressor on y = x squared has mean absolute error 0.180
  inside its [0, 10] training range and 139.704 on [10, 20], 774 times worse, and its
  largest prediction outside the range (97.307) never exceeds the largest target it saw
  (98.862); a linear model scores 6.007 inside and 101.643 outside. (6) The baseline
  case where the model loses: on features that are pure noise with 90 percent of rows in
  one class, the majority-class baseline scores exactly 0.900 while a 1-NN scores 0.821
  and a full-depth tree 0.817 -- both would be reported as "82 percent accurate" and
  both are worse than a constant; on iris the same comparison reads baseline 0.260
  against 1-NN 0.980. (7) The noise ceiling: with exactly 1000 of 4000 test labels
  flipped the ceiling is exactly 0.750, and logistic regression 0.73725, 15-NN 0.72675,
  a depth-3 tree 0.68825 and a full-depth tree 0.60875 all sit at or below it. (8) More
  data: a variance-limited problem improves from 0.5995 at n=50 to 0.99725 at n=5000, a
  gain of 39.8 points, while a noise-limited one moves from 0.6655 at n=200 to 0.68675
  at n=5000, a gain of 2.1 points against a 0.700 ceiling it already sat within 3.5
  points of. THREE HONESTY CALLS. FIRST: the claim "1-NN training accuracy is 1.000 by
  construction" is not universally true and this lab measures the exception rather than
  hiding it. A 1-NN misses a training row when an identical feature row carries a
  different label, and iris contains exactly one duplicated feature row -- positions 101
  and 142, (5.8, 2.7, 5.1, 1.9). Both carry class 2, so on iris's real labels 1-NN still
  scores 1.000; permute the labels and the same pair drops it to 0.9933333333333333.
  Exercise 1b asserts only that the scrambled score is below 1.000, because that is the
  structural part. SECOND: the brief asks exercise 8 to show that more data does not
  help a noise-limited model. Measured, it helps a little -- 2.1 points -- because n=200
  is genuinely a small sample; what it cannot do is cross the ceiling, and it starts 3.5
  points below one. The lab and lesson report the 2.1 points rather than rounding it to
  zero, and assert the contrast (a 39.8-point gain against a 2.1-point one) rather than
  a false flatness. THIRD: NumPy's own documentation states that Generator carries no
  version compatibility guarantee and that its bit stream may change; seeding therefore
  makes every number here reproducible under the pins and not beyond them. That is
  recorded in expected-output/FIELDS.md, which separates the four values that are exact
  everywhere by arithmetic (1.000, 0.900, 0.750, and the rule's 1.000) from the twenty-one
  that are exact only for numpy 2.5.2 and scikit-learn 1.9.0.

# Why each pin is here

```
numpy==2.5.2
scikit-learn==1.9.0
pytest==9.1.1
```

- **`numpy` 2.5.2** — every dataset in this lab is generated by
  `numpy.random.default_rng(seed)`, and the hand-written
  nearest-neighbour classifier in `ml_lib.py` is pure NumPy
  broadcasting. The version is pinned rather than ranged for an exact
  reason: NumPy's own documentation states plainly that `Generator`
  carries **no** compatibility guarantee, and that the bit stream may
  change as better algorithms evolve. Seeding alone therefore does not
  make these numbers reproducible across versions — seeding plus this
  pin does.
- **`scikit-learn` 1.9.0** — supplies the trained models this lab scores
  and, through `sklearn.datasets.load_iris`, the iris measurements
  themselves. No download is involved: iris ships inside the installed
  package.
- **`pytest` 9.1.1** — the test runner. The `starter/` suite uses
  `pytest.skip` to mark unwritten exercises.

Installing scikit-learn also installs `scipy`, `joblib` and
`threadpoolctl` as its own dependencies. They are not pinned here
because nothing in this lab imports them directly; the versions present
when the captured output was produced are recorded in
`expected-output/FIELDS.md`.

Pins are exact versions rather than ranges so that a re-run installs the
same code that produced `expected-output/`. If you need a full lock of
the transitive tree as well, run `pip freeze > requirements/locked.txt`
after installing.

numpy==2.5.2
scikit-learn==1.9.0
pytest==9.1.1

# Day 141 lab brief — What the Number Is Not Telling You

You have never trained a model with a library before today, and this lab
is deliberately not a tour of one. Every model here is constructed for
you by a one-line helper in `ml_lib.py` with its settings already fixed,
because the models are not the subject. The subject is what their scores
mean, and what they do not mean.

## The claim you are here to break

> A model that scores well is a model that works.

Exercise 1 destroys it in six lines. A one-nearest-neighbour model
trained on a dataset whose labels are coin flips scores **exactly 1.000**
on its training data — every time, on any machine, by construction,
because each training point is its own nearest neighbour at distance
zero. Its test accuracy is chance. The model has learned nothing and
reports perfection.

Once you have measured that, every other exercise is a variation on the
same discipline: name the number, name what it was measured on, and name
what would have to be true for it to mean anything.

## How to work

1. Build the environment (see the lab `README.md`).
2. Run `.venv/bin/pytest starter -q`. You will see three passes (the
   machinery checks in `test_ml_lib.py`) and ten skips.
3. Replace one `pytest.skip(...)` at a time with real code. The skip text
   names the exact datasets, the exact helpers and the exact values to
   assert. None of it is guesswork.
4. Print the measured pair in every exercise. A number you did not print
   is a number you did not look at.
5. When you want to see the whole measured table at once, run
   `.venv/bin/python examples/report_measurements.py`.

Do not run `pytest starter examples` in one invocation. Both directories
define `ml_lib.py`, `test_ml_lib.py` and `test_ml_claims.py`; pytest
aborts on the module-name collision. Run them separately, always.

## What `ml_lib.py` gives you

| Helper | What it is |
| --- | --- |
| `HandwrittenNearestNeighbour` | 1-NN written from first principles in NumPy — eleven lines of arithmetic, no library |
| `pure_noise_dataset` | Normal features, coin-flip labels. No function exists to be approximated |
| `rule_dataset(n, seed, offset)` | Two uniform features, labelled `x1 > x0`. `offset` translates the region without changing the rule |
| `exact_rule` | That same rule, as three lines of code that need no data |
| `flip_labels` / `noisy_rule_dataset` | An exact count of labels flipped, so the noise ceiling is exact arithmetic |
| `checkerboard_dataset` | A clean but intricate boundary — the variance-limited problem |
| `imbalanced_noise_dataset` | 90 percent one class, features pure noise — the baseline trap |
| `quadratic_curve` | One feature, `y = x squared` — the interpolation/extrapolation demonstration |
| `one_nn`, `shallow_tree`, `deep_tree`, `smooth_knn`, `linear_classifier`, `majority_baseline`, `knn_regressor`, `linear_regressor` | Fixed-hyper-parameter model constructors |
| `fit_score` | Fit on train, score on test. Nothing else |
| `should_use_ml`, `problem` | Exercise 9's decision function and its case helper |

Every dataset takes an explicit seed and uses
`numpy.random.default_rng(seed)`, so the numbers in the skip texts are
the numbers you will measure.

"""Machinery for Day 141 -- "What the Number Is Not Telling You".

Nothing in this file is an exercise. It builds the small, fully
deterministic datasets the nine exercises measure, and it contains one
model written by hand in NumPy so you can see that a nearest-neighbour
classifier is eleven lines of arithmetic and no magic at all.

Every dataset constructor takes an explicit integer seed and uses
`numpy.random.default_rng(seed)`, so every number this lab reports is
reproducible on any machine with the pinned versions.
"""

from __future__ import annotations

import numpy as np
from sklearn.dummy import DummyClassifier
from sklearn.linear_model import LinearRegression, LogisticRegression
from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor
from sklearn.tree import DecisionTreeClassifier

# --------------------------------------------------------------------------
# Scoring
# --------------------------------------------------------------------------


def accuracy(y_true, y_pred) -> float:
    """Fraction of predictions that match. The whole of "accuracy"."""
    y_true = np.asarray(y_true)
    y_pred = np.asarray(y_pred)
    return float(np.mean(y_true == y_pred))


def mean_absolute_error(y_true, y_pred) -> float:
    y_true = np.asarray(y_true, dtype=float)
    y_pred = np.asarray(y_pred, dtype=float)
    return float(np.mean(np.abs(y_true - y_pred)))


# --------------------------------------------------------------------------
# A nearest-neighbour classifier, written by hand
# --------------------------------------------------------------------------


class HandwrittenNearestNeighbour:
    """1-nearest-neighbour, from first principles, in NumPy.

    `fit` stores the training set. That is the entire training procedure:
    there is no search, no objective, no parameters. `predict` finds the
    closest stored point to each query and copies its label.

    This class exists to make one fact concrete: predicting a training
    point returns that point's own label, because a point's own distance
    to itself is zero. Training accuracy is therefore 1.000 by
    construction and carries no information whatsoever.
    """

    def __init__(self) -> None:
        self.X_: np.ndarray | None = None
        self.y_: np.ndarray | None = None

    def fit(self, X, y) -> "HandwrittenNearestNeighbour":
        self.X_ = np.asarray(X, dtype=float)
        self.y_ = np.asarray(y)
        return self

    def predict(self, X) -> np.ndarray:
        if self.X_ is None or self.y_ is None:
            raise RuntimeError("call fit before predict")
        X = np.asarray(X, dtype=float)
        # Squared euclidean distance from every query row to every stored
        # row, by broadcasting: (n_query, 1, n_features) - (n_train, n_features)
        diff = X[:, None, :] - self.X_[None, :, :]
        sq_dist = np.sum(diff * diff, axis=2)
        nearest = np.argmin(sq_dist, axis=1)
        return self.y_[nearest]


# --------------------------------------------------------------------------
# Datasets -- every one deterministic given its seed
# --------------------------------------------------------------------------


def pure_noise_dataset(n: int, n_features: int = 4, seed: int = 141):
    """Features from a normal distribution, labels from a coin flip.

    There is no relationship of any kind between X and y. No function
    exists to be approximated, so the best possible test accuracy is
    chance.
    """
    rng = np.random.default_rng(seed)
    X = rng.normal(size=(n, n_features))
    y = rng.integers(0, 2, size=n)
    return X, y


def rule_dataset(n: int, seed: int, offset: float = 0.0):
    """Two uniform features on a square of side 1, labelled by an exact rule.

    The rule is `y = 1 if x1 > x0 else 0` -- the diagonal of the square.
    `offset` translates the whole square without changing the rule, which
    is what makes this dataset usable as a distribution shift: the
    labelling function is identical, only the region the points live in
    has moved.
    """
    rng = np.random.default_rng(seed)
    X = rng.uniform(0.0, 1.0, size=(n, 2)) + offset
    y = exact_rule(X)
    return X, y


def exact_rule(X) -> np.ndarray:
    """The three-line rule that `rule_dataset` labels with.

    It is exactly correct everywhere, for every input, forever, and it
    needs no data, no training and no maintenance.
    """
    X = np.asarray(X, dtype=float)
    return (X[:, 1] > X[:, 0]).astype(int)


def flip_labels(y, noise_rate: float, seed: int):
    """Flip exactly `round(noise_rate * len(y))` labels, chosen at random.

    Exactly, not approximately: the count is fixed so the resulting
    ceiling is an exact arithmetic fact about the data rather than a
    sampled quantity.
    """
    y = np.asarray(y).copy()
    rng = np.random.default_rng(seed)
    n_flip = int(round(noise_rate * len(y)))
    idx = rng.choice(len(y), size=n_flip, replace=False)
    y[idx] = 1 - y[idx]
    return y


def noisy_rule_dataset(n: int, seed: int, noise_rate: float):
    """`rule_dataset` with a known fraction of its labels flipped."""
    X, y_clean = rule_dataset(n, seed=seed)
    y = flip_labels(y_clean, noise_rate=noise_rate, seed=seed + 9000)
    return X, y


def checkerboard_dataset(n: int, seed: int, cells: int = 4):
    """A clean but intricate boundary: a `cells` x `cells` checkerboard.

    The labels contain no noise at all, so nothing limits a model here
    except how much of the boundary the training sample reveals. This is
    the variance-limited problem that more data genuinely fixes.
    """
    rng = np.random.default_rng(seed)
    X = rng.uniform(0.0, 1.0, size=(n, 2))
    y = ((np.floor(X[:, 0] * cells) + np.floor(X[:, 1] * cells)) % 2).astype(int)
    return X, y


def imbalanced_noise_dataset(n: int, seed: int, minority_rate: float = 0.1):
    """Pure-noise features with an exact minority-class count.

    Exactly `round(minority_rate * n)` rows carry label 1, placed at
    random positions, so the majority-class baseline on this set is an
    exact number rather than an estimate.
    """
    rng = np.random.default_rng(seed)
    X = rng.normal(size=(n, 6))
    y = np.zeros(n, dtype=int)
    n_minority = int(round(minority_rate * n))
    y[rng.choice(n, size=n_minority, replace=False)] = 1
    return X, y


def quadratic_curve(n: int, low: float, high: float, seed: int):
    """One feature on [low, high], target y = x squared. No noise."""
    rng = np.random.default_rng(seed)
    x = np.sort(rng.uniform(low, high, size=n))
    X = x.reshape(-1, 1)
    y = x**2
    return X, y


# --------------------------------------------------------------------------
# Model constructors -- fixed hyper-parameters so results never drift
# --------------------------------------------------------------------------


def one_nn() -> KNeighborsClassifier:
    return KNeighborsClassifier(n_neighbors=1)


def shallow_tree(max_depth: int = 3) -> DecisionTreeClassifier:
    return DecisionTreeClassifier(max_depth=max_depth, random_state=141)


def deep_tree() -> DecisionTreeClassifier:
    return DecisionTreeClassifier(random_state=141)


def smooth_knn(k: int = 15) -> KNeighborsClassifier:
    return KNeighborsClassifier(n_neighbors=k)


def linear_classifier() -> LogisticRegression:
    return LogisticRegression(max_iter=1000)


def majority_baseline() -> DummyClassifier:
    return DummyClassifier(strategy="most_frequent")


def knn_regressor(k: int = 5) -> KNeighborsRegressor:
    return KNeighborsRegressor(n_neighbors=k)


def linear_regressor() -> LinearRegression:
    return LinearRegression()


def fit_score(model, X_train, y_train, X_test, y_test) -> float:
    """Fit on the training set, score on the test set. Nothing else."""
    model.fit(X_train, y_train)
    return accuracy(y_test, model.predict(X_test))


# --------------------------------------------------------------------------
# Exercise 9: the decision function
# --------------------------------------------------------------------------

#: The four questions, in the order they must be asked. Cheapness first:
#: a question whose answer disqualifies machine learning outright is
#: worth asking before one that merely constrains it.
ML_DECISION_QUESTIONS = (
    "exact_rule_exists",
    "labels_available",
    "distribution_stable",
    "errors_tolerable",
)


def should_use_ml(problem: dict) -> str:
    """Return one of five verdicts for a described problem.

    `problem` must carry all four keys in `ML_DECISION_QUESTIONS` with
    boolean values. The order of the checks is the argument:

    1. `exact_rule_exists` -- if you can write the rule down, write it.
       A rule is exactly correct, costs nothing to run, needs no labels,
       never drifts and can be reviewed by a person who is not you.
       No model beats that, and a model that merely matches it has cost
       you a data pipeline for nothing.
    2. `labels_available` -- supervised learning approximates a function
       from examples of its output. Without labels there are no examples,
       and no amount of feature work substitutes for them.
    3. `distribution_stable` -- the method assumes future inputs resemble
       training inputs. If the world moves faster than you can retrain,
       the model is wrong by the time it ships.
    4. `errors_tolerable` -- a model is an approximation and will be
       wrong on some inputs. If a single wrong answer is unacceptable and
       cannot be caught downstream, an approximation is the wrong shape
       of tool regardless of its accuracy.
    """
    missing = [q for q in ML_DECISION_QUESTIONS if q not in problem]
    if missing:
        raise KeyError(f"problem is missing: {', '.join(missing)}")
    if problem["exact_rule_exists"]:
        return "write the rule"
    if not problem["labels_available"]:
        return "get labels first"
    if not problem["distribution_stable"]:
        return "not yet: the distribution moves"
    if not problem["errors_tolerable"]:
        return "no: errors are not tolerable"
    return "yes"


def problem(
    exact_rule_exists: bool,
    labels_available: bool,
    distribution_stable: bool,
    errors_tolerable: bool,
) -> dict:
    """Small helper so the case table in the tests reads as prose."""
    return {
        "exact_rule_exists": exact_rule_exists,
        "labels_available": labels_available,
        "distribution_stable": distribution_stable,
        "errors_tolerable": errors_tolerable,
    }

"""Nine exercises in what an accuracy number is not telling you.

Read `00_brief.md` first. Each function below is a `pytest.skip` naming
exactly what to build and what to assert; replace the skip with real
code. `ml_lib.py` is complete -- it is the machinery, not the exercise.

Run this suite on its own:

    .venv/bin/pytest starter -q

Never run `pytest starter examples` in one invocation: both directories
define modules with the same names and pytest aborts on the collision.
"""

import numpy as np  # noqa: F401  (you will need it)
import pytest

import ml_lib as m  # noqa: F401  (you will need it)


def test_01_one_nn_scores_a_perfect_1_000_having_learned_nothing():
    pytest.skip(
        "Build m.pure_noise_dataset(200, seed=141) for training and "
        "m.pure_noise_dataset(1000, seed=242) for test. Fit "
        "m.HandwrittenNearestNeighbour on the training set. Assert the "
        "training accuracy is exactly 1.0 and the test accuracy is 0.518, "
        "and that the test accuracy is within 0.06 of chance (0.5). Then "
        "assert m.one_nn() -- scikit-learn's KNeighborsClassifier(1) -- "
        "reproduces both numbers exactly. Print both."
    )


def test_01b_the_only_way_a_1_nn_misses_a_training_row_is_a_duplicate():
    pytest.skip(
        "Load iris with sklearn.datasets.load_iris(return_X_y=True). "
        "Assert X.shape == (150, 4) and that the number of unique feature "
        "rows is 149 -- iris contains exactly one duplicated row. Assert a "
        "hand-written 1-NN scores 1.0 on the real labels (the duplicate "
        "pair shares a class), then permute the labels with "
        "np.random.default_rng(141) and assert the same model now scores "
        "below 1.0. That is the single exception to '1.000 by construction'."
    )


def test_02_a_three_line_rule_scores_1_000_and_every_model_scores_less():
    pytest.skip(
        "Train on m.rule_dataset(300, seed=11), test on "
        "m.rule_dataset(2000, seed=12). Assert m.exact_rule scores exactly "
        "1.0 on the test set. Assert m.shallow_tree(3) scores 0.8855 and "
        "is strictly less than the rule. Then score m.shallow_tree(8), "
        "m.deep_tree() and m.smooth_knn(15) too, assert the best of the "
        "four is 0.9675, and assert every one of them loses to the rule."
    )


def test_03_train_accuracy_exceeds_test_accuracy_and_the_gap_is_the_story():
    pytest.skip(
        "Split iris 100/50 with np.random.default_rng(141).permutation. "
        "Fit m.deep_tree(); assert train accuracy 1.0, test accuracy 0.96, "
        "gap 0.04. Then do the same on m.noisy_rule_dataset(300, seed=21, "
        "noise_rate=0.2) against m.noisy_rule_dataset(2000, seed=22, "
        "noise_rate=0.2): assert train 1.0, test 0.6535, gap 0.3465. "
        "Assert the two training scores are identical and the second gap "
        "is more than eight times the first. Report both gaps. Finally "
        "fit m.linear_classifier() on the SAME noisy training set and "
        "assert it scores 0.78 in training (worse than the tree) and "
        "0.7655 on the test set (better than the tree) -- the better "
        "training score belongs to the worse model."
    )


def test_04_accuracy_collapses_when_the_input_region_moves():
    pytest.skip(
        "Train m.deep_tree() on m.rule_dataset(400, seed=31). Score it on "
        "m.rule_dataset(2000, seed=32) -- assert 0.948 -- and on "
        "m.rule_dataset(2000, seed=33, offset=3.0), the identical problem "
        "translated by 3.0 -- assert 0.4895, which is below chance. Assert "
        "the drop exceeds 0.4, and that m.exact_rule still scores 1.0 on "
        "the shifted region. Report both accuracies."
    )


def test_05_a_model_interpolates_beautifully_and_extrapolates_not_at_all():
    pytest.skip(
        "Fit m.knn_regressor(5) on m.quadratic_curve(300, 0.0, 10.0, "
        "seed=41). Assert its mean absolute error is 0.180 on "
        "m.quadratic_curve(200, 0.0, 10.0, seed=42) and 139.704 on "
        "m.quadratic_curve(200, 10.0, 20.0, seed=43), both rounded to "
        "three places, and that the outside error is more than 700 times "
        "the inside one. Assert its largest prediction outside the range "
        "does not exceed the largest target it ever saw. Then check "
        "m.linear_regressor(): 6.007 inside, 101.643 outside."
    )


def test_06_a_good_looking_accuracy_that_loses_to_predicting_the_majority():
    pytest.skip(
        "Use m.imbalanced_noise_dataset(1000, seed=51) and (1000, "
        "seed=52). Assert m.majority_baseline() scores exactly 0.9 and "
        "that exactly 90 percent of the test labels are class 0. Assert "
        "m.one_nn() scores 0.821 and m.deep_tree() scores 0.817, and that "
        "BOTH are below the baseline. Then repeat the comparison on the "
        "iris split from exercise 3: baseline 0.26, 1-NN 0.98. Report all "
        "five numbers."
    )


def test_07_no_model_beats_the_label_noise_ceiling():
    pytest.skip(
        "With noise_rate=0.25, train on m.noisy_rule_dataset(2000, "
        "seed=61) and test on m.noisy_rule_dataset(4000, seed=62). The "
        "ceiling is 1 - noise_rate = 0.75. Score m.linear_classifier() "
        "(0.73725), m.smooth_knn(15) (0.72675), m.shallow_tree(3) "
        "(0.68825) and m.deep_tree() (0.60875), and assert every one is "
        "at or below the ceiling. Assert the best is within 0.02 of it. "
        "Finally assert the ceiling is exact, not estimated: compare "
        "against m.rule_dataset(4000, seed=62) and confirm exactly 1000 "
        "of the 4000 test labels were flipped."
    )


def test_08_more_data_fixes_variance_and_does_nothing_for_label_noise():
    pytest.skip(
        "Variance-limited: test on m.checkerboard_dataset(4000, seed=71); "
        "train m.deep_tree() on m.checkerboard_dataset(50, seed=120) "
        "(assert 0.5995) and on m.checkerboard_dataset(5000, seed=5070) "
        "(assert 0.99725). Noise-limited: test on "
        "m.noisy_rule_dataset(4000, seed=81, noise_rate=0.30); train "
        "m.linear_classifier() on (200, seed=280) (assert 0.6655) and on "
        "(5000, seed=5080) (assert 0.68675). Assert the first gain exceeds "
        "0.30, the second is below 0.05, and the small-sample "
        "noise-limited model already sits within 0.05 of its 0.70 ceiling."
    )


def test_09_should_use_ml_gives_the_verdict_the_case_deserves():
    pytest.skip(
        "Build a table of at least six cases with m.problem(...), one for "
        "each verdict m.should_use_ml can return plus a case where a rule "
        "exists and nothing else does. Justify each in a comment. Assert "
        "the verdicts. Then assert that a problem missing one of the four "
        "keys raises KeyError naming the missing key -- a missing question "
        "is an error, not a default."
    )

"""Three checks on the machinery itself, so a failure elsewhere is a
failure of the claim under test and not of the helpers.

These three are already solved in `starter/` too: they are not exercises.
"""

import numpy as np

import ml_lib as m


def test_accuracy_counts_matches():
    assert m.accuracy([0, 1, 1, 0], [0, 1, 0, 0]) == 0.75
    assert m.accuracy([1, 1], [1, 1]) == 1.0
    assert m.accuracy([1, 1], [0, 0]) == 0.0


def test_exact_rule_is_exactly_correct_on_its_own_data():
    for seed in (1, 2, 3):
        X, y = m.rule_dataset(500, seed=seed)
        assert m.accuracy(y, m.exact_rule(X)) == 1.0


def test_flip_labels_flips_an_exact_count():
    _, y = m.rule_dataset(1000, seed=7)
    flipped = m.flip_labels(y, noise_rate=0.25, seed=7)
    assert int(np.sum(y != flipped)) == 250
    # Flipping is a relabelling, not a resampling: the feature matrix and
    # the array length are untouched.
    assert flipped.shape == y.shape

#!/usr/bin/env bash
# Day 141 lab harness: "What the Number Is Not Telling You"
#
# Prints "N checks, M failure(s)" and exits 0 only when M is zero.
set -u

LAB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$LAB_DIR"

PYTHON="${PYTHON:-.venv/bin/python3}"
PYTEST="${PYTEST:-.venv/bin/pytest}"

CHECKS=0
FAILURES=0

ok() {
  CHECKS=$((CHECKS + 1))
  echo "  ok: $1"
}

fail() {
  CHECKS=$((CHECKS + 1))
  FAILURES=$((FAILURES + 1))
  echo "  FAIL: $1"
}

if [ ! -x "$PYTHON" ]; then
  echo "No lab .venv found at $PYTHON."
  echo "Run: python3 -m venv .venv && .venv/bin/pip install -r requirements/requirements.txt"
  exit 2
fi

echo "1. Installed versions match requirements/requirements.txt"
VERSION_CHECK=$("$PYTHON" - <<'PYEOF'
import numpy, sklearn, pytest
print("numpy", numpy.__version__)
print("scikit-learn", sklearn.__version__)
print("pytest", pytest.__version__)
PYEOF
)
echo "$VERSION_CHECK" | sed 's/^/    /'
while read -r pkg pin; do
  pin_version="${pin#*==}"
  installed=$(echo "$VERSION_CHECK" | awk -v p="$pkg" '$1==p {print $2}')
  if [ "$installed" = "$pin_version" ]; then
    ok "$pkg $installed matches the pin"
  else
    fail "$pkg installed=$installed pinned=$pin_version"
  fi
done < <(sed 's/==/ ==/' requirements/requirements.txt)

echo ""
echo "2. The nine claims, reproduced directly (no pytest involved)"
DIRECT_CHECK=$("$PYTHON" - <<'PYEOF'
import sys
sys.path.insert(0, "examples")

import numpy as np
from sklearn.datasets import load_iris

import ml_lib as m

errors = []


def expect(label, got, want):
    if got != want:
        errors.append(f"{label}: expected {want}, got {got}")


# 1. Perfect accuracy, zero learning
X_tr, y_tr = m.pure_noise_dataset(200, seed=141)
X_te, y_te = m.pure_noise_dataset(1000, seed=242)
hand = m.HandwrittenNearestNeighbour().fit(X_tr, y_tr)
expect("1-NN train on noise", m.accuracy(y_tr, hand.predict(X_tr)), 1.0)
expect("1-NN test on noise", m.accuracy(y_te, hand.predict(X_te)), 0.518)
lib = m.one_nn().fit(X_tr, y_tr)
expect("sklearn 1-NN train on noise", m.accuracy(y_tr, lib.predict(X_tr)), 1.0)
expect("sklearn 1-NN test on noise", m.accuracy(y_te, lib.predict(X_te)), 0.518)

X_iris, y_iris = load_iris(return_X_y=True)
expect("iris shape", X_iris.shape, (150, 4))
expect("iris unique rows", len({tuple(r) for r in X_iris}), 149)
scrambled = np.random.default_rng(141).permutation(y_iris)
expect(
    "1-NN train, iris real labels",
    m.accuracy(y_iris, m.HandwrittenNearestNeighbour().fit(X_iris, y_iris).predict(X_iris)),
    1.0,
)
if m.accuracy(
    scrambled,
    m.HandwrittenNearestNeighbour().fit(X_iris, scrambled).predict(X_iris),
) >= 1.0:
    errors.append("scrambled-label iris 1-NN unexpectedly scored a perfect 1.0")

# 2. A rule beats a model
X_tr, y_tr = m.rule_dataset(300, seed=11)
X_te, y_te = m.rule_dataset(2000, seed=12)
expect("rule accuracy", m.accuracy(y_te, m.exact_rule(X_te)), 1.0)
expect("depth-3 tree", m.fit_score(m.shallow_tree(3), X_tr, y_tr, X_te, y_te), 0.8855)
best = max(
    m.fit_score(model, X_tr, y_tr, X_te, y_te)
    for model in (m.shallow_tree(3), m.shallow_tree(8), m.deep_tree(), m.smooth_knn(15))
)
expect("best trained model", best, 0.9675)
if best >= 1.0:
    errors.append("a trained model matched the rule, which contradicts the exercise")

# 3. The generalisation gap
perm = np.random.default_rng(141).permutation(len(y_iris))
tr, te = perm[:100], perm[100:]
tree = m.deep_tree().fit(X_iris[tr], y_iris[tr])
expect("iris train", m.accuracy(y_iris[tr], tree.predict(X_iris[tr])), 1.0)
expect("iris test", m.accuracy(y_iris[te], tree.predict(X_iris[te])), 0.96)
X_tr, y_tr = m.noisy_rule_dataset(300, seed=21, noise_rate=0.2)
X_te, y_te = m.noisy_rule_dataset(2000, seed=22, noise_rate=0.2)
noisy = m.deep_tree().fit(X_tr, y_tr)
expect("noisy train", m.accuracy(y_tr, noisy.predict(X_tr)), 1.0)
expect("noisy test", m.accuracy(y_te, noisy.predict(X_te)), 0.6535)
simple = m.linear_classifier().fit(X_tr, y_tr)
expect("simple model train", m.accuracy(y_tr, simple.predict(X_tr)), 0.78)
expect("simple model test", m.accuracy(y_te, simple.predict(X_te)), 0.7655)

# 4. Distribution shift
X_tr, y_tr = m.rule_dataset(400, seed=31)
X_in, y_in = m.rule_dataset(2000, seed=32)
X_sh, y_sh = m.rule_dataset(2000, seed=33, offset=3.0)
tree = m.deep_tree().fit(X_tr, y_tr)
expect("in-distribution", m.accuracy(y_in, tree.predict(X_in)), 0.948)
expect("shifted", m.accuracy(y_sh, tree.predict(X_sh)), 0.4895)
expect("rule on shifted region", m.accuracy(y_sh, m.exact_rule(X_sh)), 1.0)

# 5. Interpolation versus extrapolation
X_tr, y_tr = m.quadratic_curve(300, 0.0, 10.0, seed=41)
X_in, y_in = m.quadratic_curve(200, 0.0, 10.0, seed=42)
X_out, y_out = m.quadratic_curve(200, 10.0, 20.0, seed=43)
knn = m.knn_regressor(5).fit(X_tr, y_tr)
expect("MAE inside", round(m.mean_absolute_error(y_in, knn.predict(X_in)), 3), 0.18)
expect("MAE outside", round(m.mean_absolute_error(y_out, knn.predict(X_out)), 3), 139.704)
if float(np.max(knn.predict(X_out))) > float(np.max(y_tr)):
    errors.append("a nearest-neighbour regressor predicted beyond its training range")
linear = m.linear_regressor().fit(X_tr, y_tr)
expect("linear MAE inside", round(m.mean_absolute_error(y_in, linear.predict(X_in)), 3), 6.007)
expect("linear MAE outside", round(m.mean_absolute_error(y_out, linear.predict(X_out)), 3), 101.643)

# 6. The baseline
X_tr, y_tr = m.imbalanced_noise_dataset(1000, seed=51)
X_te, y_te = m.imbalanced_noise_dataset(1000, seed=52)
baseline = m.fit_score(m.majority_baseline(), X_tr, y_tr, X_te, y_te)
expect("majority baseline", baseline, 0.9)
one_nn = m.fit_score(m.one_nn(), X_tr, y_tr, X_te, y_te)
deep = m.fit_score(m.deep_tree(), X_tr, y_tr, X_te, y_te)
expect("1-NN on imbalanced noise", one_nn, 0.821)
expect("tree on imbalanced noise", deep, 0.817)
if not (one_nn < baseline and deep < baseline):
    errors.append("a model beat the majority baseline, contradicting exercise 6")
expect(
    "iris baseline",
    m.fit_score(m.majority_baseline(), X_iris[tr], y_iris[tr], X_iris[te], y_iris[te]),
    0.26,
)
expect(
    "iris 1-NN",
    m.fit_score(m.one_nn(), X_iris[tr], y_iris[tr], X_iris[te], y_iris[te]),
    0.98,
)

# 7. The noise ceiling
ceiling = 0.75
X_tr, y_tr = m.noisy_rule_dataset(2000, seed=61, noise_rate=0.25)
X_te, y_te = m.noisy_rule_dataset(4000, seed=62, noise_rate=0.25)
measured = {
    "logistic regression": m.fit_score(m.linear_classifier(), X_tr, y_tr, X_te, y_te),
    "15-NN": m.fit_score(m.smooth_knn(15), X_tr, y_tr, X_te, y_te),
    "depth-3 tree": m.fit_score(m.shallow_tree(3), X_tr, y_tr, X_te, y_te),
    "full-depth tree": m.fit_score(m.deep_tree(), X_tr, y_tr, X_te, y_te),
}
expect("logistic regression at the ceiling", measured["logistic regression"], 0.73725)
expect("15-NN at the ceiling", measured["15-NN"], 0.72675)
expect("depth-3 tree at the ceiling", measured["depth-3 tree"], 0.68825)
expect("full-depth tree at the ceiling", measured["full-depth tree"], 0.60875)
for name, score in measured.items():
    if score > ceiling:
        errors.append(f"{name} scored {score}, above the {ceiling} ceiling")
_, y_clean = m.rule_dataset(4000, seed=62)
expect("exact flipped test labels", int(np.sum(y_clean != y_te)), 1000)

# 8. More data does not fix the wrong thing
X_te_c, y_te_c = m.checkerboard_dataset(4000, seed=71)
small = m.fit_score(m.deep_tree(), *m.checkerboard_dataset(50, seed=120), X_te_c, y_te_c)
large = m.fit_score(m.deep_tree(), *m.checkerboard_dataset(5000, seed=5070), X_te_c, y_te_c)
expect("variance-limited n=50", small, 0.5995)
expect("variance-limited n=5000", large, 0.99725)
X_te_n, y_te_n = m.noisy_rule_dataset(4000, seed=81, noise_rate=0.30)
few = m.fit_score(
    m.linear_classifier(), *m.noisy_rule_dataset(200, seed=280, noise_rate=0.30), X_te_n, y_te_n
)
many = m.fit_score(
    m.linear_classifier(), *m.noisy_rule_dataset(5000, seed=5080, noise_rate=0.30), X_te_n, y_te_n
)
expect("noise-limited n=200", few, 0.6655)
expect("noise-limited n=5000", many, 0.68675)
if (large - small) <= 15 * (many - few):
    errors.append("the variance gain was not decisively larger than the noise gain")

# 9. The decision function
verdicts = {
    m.should_use_ml(m.problem(True, True, True, True)): "write the rule",
    m.should_use_ml(m.problem(False, False, True, True)): "get labels first",
    m.should_use_ml(m.problem(False, True, False, True)): "not yet: the distribution moves",
    m.should_use_ml(m.problem(False, True, True, False)): "no: errors are not tolerable",
    m.should_use_ml(m.problem(False, True, True, True)): "yes",
}
if len(verdicts) != 5:
    errors.append("should_use_ml did not produce five distinct verdicts")
try:
    m.should_use_ml({"exact_rule_exists": False})
except KeyError:
    pass
else:
    errors.append("should_use_ml accepted an incomplete problem")

if errors:
    for e in errors:
        print("ERROR:", e)
    sys.exit(1)
print("all direct checks passed")
PYEOF
)
if echo "$DIRECT_CHECK" | grep -q "all direct checks passed"; then
  ok "exercises 1-9 reproduced directly against ml_lib, no pytest involved"
else
  fail "direct library checks failed"
  echo "$DIRECT_CHECK" | sed 's/^/    /'
fi

echo ""
echo "3. examples/ passes in full"
EXAMPLES_OUT=$("$PYTEST" examples -q 2>&1)
if echo "$EXAMPLES_OUT" | tail -1 | grep -qE "^13 passed"; then
  ok "pytest examples -q -> 13 passed"
else
  fail "pytest examples -q did not report 13 passed"
  echo "$EXAMPLES_OUT" | tail -20 | sed 's/^/    /'
fi

echo ""
echo "4. starter/ is an untouched skeleton"
STARTER_OUT=$("$PYTEST" starter -q 2>&1)
if echo "$STARTER_OUT" | tail -1 | grep -qE "3 passed, 10 skipped"; then
  ok "pytest starter -q -> 3 passed, 10 skipped (the machinery checks pass; the ten exercises are stubs)"
else
  fail "pytest starter -q did not report 3 passed, 10 skipped"
  echo "$STARTER_OUT" | tail -20 | sed 's/^/    /'
fi

echo ""
echo "5. pytest examples starter (one invocation) aborts on the module-name collision"
COMBINED_OUT=$("$PYTEST" examples starter 2>&1)
if echo "$COMBINED_OUT" | grep -q "import file mismatch"; then
  ok "combined invocation reports import file mismatch, as documented -- never run starter and examples together"
else
  fail "combined invocation did not fail with import file mismatch as expected"
fi

echo ""
echo "6. The report reproduces the captured table exactly"
REPORT_OUT=$("$PYTHON" examples/report_measurements.py 2>&1)
if [ "$REPORT_OUT" = "$(cat expected-output/measured-values.txt)" ]; then
  ok "report_measurements.py output is byte-identical to expected-output/measured-values.txt"
else
  fail "report_measurements.py drifted from expected-output/measured-values.txt"
  echo "$REPORT_OUT" | diff - expected-output/measured-values.txt | head -20 | sed 's/^/    /'
fi

echo ""
echo "7. Proof the harness can fail"
SCRATCH=$(mktemp -d "${TMPDIR:-/tmp}/d141-scratch.XXXXXX")
cp examples/*.py "$SCRATCH"/
SCRATCH_OUT=$("$PYTEST" "$SCRATCH" -q 2>&1)
if echo "$SCRATCH_OUT" | tail -1 | grep -qE "^13 passed"; then
  ok "scratch copy of examples/ passes before it is broken"
else
  fail "scratch copy did not pass before being broken: $(echo "$SCRATCH_OUT" | tail -3)"
fi
"$PYTHON" - "$SCRATCH/test_ml_claims.py" <<'PYEOF'
import sys
path = sys.argv[1]
text = open(path).read()
needle = "assert train_acc == 1.0"
replacement = "assert train_acc == 0.5"
assert needle in text, "could not find the assertion to break"
open(path, "w").write(text.replace(needle, replacement, 1))
PYEOF
BROKEN_OUT=$("$PYTEST" "$SCRATCH" -q 2>&1)
BROKEN_STATUS=$?
if [ "$BROKEN_STATUS" -ne 0 ] && echo "$BROKEN_OUT" | grep -q "test_01_one_nn_scores_a_perfect_1_000_having_learned_nothing"; then
  ok "breaking exercise 1's assertion produces a non-zero exit and names the failing test"
else
  fail "broken copy did not fail as expected (exit=$BROKEN_STATUS)"
fi
rm -rf "$SCRATCH"

echo ""
echo "8. Offline, and nothing left behind"
if ! grep -rInE "https?://" examples/*.py starter/*.py > /dev/null 2>&1; then
  ok "no URLs inside examples/ or starter/ source -- this lab reaches no network"
else
  fail "found a URL inside examples/ or starter/"
fi
if [ -z "$(find . -path ./.venv -prune -o -type d -name '__pycache__' -print 2>/dev/null)" ]; then
  ok "no __pycache__ left behind"
else
  find . -path ./.venv -prune -o -type d -name '__pycache__' -exec rm -rf {} + 2>/dev/null
  ok "no __pycache__ left behind (cleaned during this run)"
fi
if [ ! -d .pytest_cache ]; then
  ok "no .pytest_cache left behind"
else
  rm -rf .pytest_cache
  ok "no .pytest_cache left behind (cleaned during this run)"
fi

echo ""
echo "---------------------------------------------------------------"
echo "$CHECKS checks, $FAILURES failure(s)"
if [ "$FAILURES" -ne 0 ]; then
  exit 1
fi
exit 0