Hands-on lab — Day 146: Your First Model with scikit-learn

"""The estimator API, measured: what fit/predict/score/get_params/set_params
actually buy you, and where the "it's just a protocol" story needs a footnote.

Days 141-145 called `.fit()` and `.predict()` on scikit-learn objects
without ever explaining what those calls mean. This module builds a
classifier from first principles -- no inheritance at all -- to show that
the four core verbs are a protocol you can implement yourself, and then
measures exactly where that protocol stops being sufficient on its own in
this version of the library.

Everything here is deterministic given a seed, except the two functions
that exist specifically to measure what `random_state=None` costs, which
are documented as such.
"""

from __future__ import annotations

import subprocess
import sys

import numpy as np

from sklearn.base import BaseEstimator, ClassifierMixin, clone
from sklearn.dummy import DummyClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.exceptions import NotFittedError
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import StratifiedKFold, cross_val_score, train_test_split
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.utils import all_estimators
from sklearn.utils.estimator_checks import check_estimator
from sklearn.utils.validation import check_is_fitted, validate_data


def accuracy(y_true, y_pred) -> float:
    return float(np.mean(np.asarray(y_true) == np.asarray(y_pred)))


# --------------------------------------------------------------------------
# 1. A classifier built entirely from first principles
# --------------------------------------------------------------------------


class MajorityClassifier:
    """fit/predict/score/get_params/set_params, written out by hand.

    Nothing here inherits from scikit-learn. `__init__` stores its
    hyper-parameter exactly as given and computes nothing; `fit` is where
    everything named with a trailing underscore gets learned. It always
    predicts the class that was most frequent in the training labels --
    exactly what ``DummyClassifier(strategy="most_frequent")`` does, which
    is what makes its output directly checkable against the library.
    """

    def __init__(self, strategy: str = "most_frequent"):
        self.strategy = strategy

    def fit(self, X, y):
        X = np.asarray(X)
        y = np.asarray(y)
        self.classes_, counts = np.unique(y, return_counts=True)
        self.majority_class_ = self.classes_[np.argmax(counts)]
        self.n_features_in_ = X.shape[1]
        return self

    def _require_fitted(self):
        if not hasattr(self, "majority_class_"):
            raise NotFittedError(
                "This MajorityClassifier instance is not fitted yet. Call "
                "'fit' with appropriate arguments before using this estimator."
            )

    def predict(self, X):
        self._require_fitted()
        X = np.asarray(X)
        return np.full(X.shape[0], self.majority_class_)

    def predict_proba(self, X):
        self._require_fitted()
        X = np.asarray(X)
        row = np.zeros(len(self.classes_))
        row[int(np.argmax(self.classes_ == self.majority_class_))] = 1.0
        return np.tile(row, (X.shape[0], 1))

    def score(self, X, y):
        return accuracy(y, self.predict(X))

    def get_params(self, deep: bool = True) -> dict:
        return {"strategy": self.strategy}

    def set_params(self, **params):
        for key, value in params.items():
            setattr(self, key, value)
        return self


class MajorityClassifierBase(ClassifierMixin, BaseEstimator):
    """The same classifier, this time built on scikit-learn's own base classes.

    `get_params` and `set_params` are gone from the source -- `BaseEstimator`
    supplies both by inspecting `__init__`'s signature, which is why
    `__init__` must do nothing but store its arguments: the introspection
    only works if the parameter names and the stored attribute names match
    exactly. What else `BaseEstimator` supplies, silently, is
    `__sklearn_tags__` -- which exercise 3 shows is no longer optional for
    `Pipeline` and `cross_val_score` in this version.
    """

    def __init__(self, strategy: str = "most_frequent"):
        self.strategy = strategy

    def fit(self, X, y):
        X, y = validate_data(self, X, y)
        self.classes_, counts = np.unique(y, return_counts=True)
        self.majority_class_ = self.classes_[np.argmax(counts)]
        self.class_prior_ = counts / counts.sum()
        return self

    def predict(self, X):
        check_is_fitted(self)
        X = validate_data(self, X, reset=False)
        return np.full(X.shape[0], self.majority_class_)

    def predict_proba(self, X):
        check_is_fitted(self)
        X = validate_data(self, X, reset=False)
        row = np.zeros(len(self.classes_))
        row[int(np.argmax(self.class_prior_))] = 1.0
        return np.tile(row, (X.shape[0], 1))


# --------------------------------------------------------------------------
# 2. Datasets shared by the exercises below
# --------------------------------------------------------------------------


def classification_dataset(n: int = 150, n_features: int = 4, n_classes: int = 3, seed: int = 42):
    """A small, well-separated classification dataset. Nothing rare, nothing grouped."""
    rng = np.random.default_rng(seed)
    centers = rng.normal(scale=4.0, size=(n_classes, n_features))
    y = rng.integers(0, n_classes, size=n)
    X = centers[y] + rng.normal(size=(n, n_features))
    return X, y


def skewed_dataset(n: int = 80, seed: int = 0):
    """A dataset with a clear majority class, for the DummyClassifier comparison."""
    rng = np.random.default_rng(seed)
    X = rng.normal(size=(n, 3))
    y = rng.integers(0, 3, size=n)
    y[: (2 * n) // 3] = 0  # force an unambiguous majority
    return X, y


# --------------------------------------------------------------------------
# 3. Does the hand-built estimator agree with the library one?
# --------------------------------------------------------------------------


def matches_dummy_classifier(seeds=range(5)) -> bool:
    """True only if MajorityClassifier's output is byte-identical to DummyClassifier's."""
    for seed in seeds:
        X, y = skewed_dataset(n=60 + seed * 11, seed=seed)
        ours = MajorityClassifier().fit(X, y)
        theirs = DummyClassifier(strategy="most_frequent").fit(X, y)
        if not np.array_equal(ours.predict(X), theirs.predict(X)):
            return False
        if not np.array_equal(ours.predict_proba(X), theirs.predict_proba(X)):
            return False
    return True


# --------------------------------------------------------------------------
# 4. What fitting actually adds
# --------------------------------------------------------------------------


def gained_attributes(estimator, X, y) -> list[str]:
    """dir(estimator) after fit, minus dir(estimator) before -- what fit() adds."""
    before = set(dir(estimator))
    estimator.fit(X, y)
    after = set(dir(estimator))
    return sorted(after - before)


def predict_before_fit_message(estimator, n_features: int = 4) -> str:
    """Call predict on an unfitted estimator and return the exception's message."""
    try:
        estimator.predict(np.zeros((3, n_features)))
    except NotFittedError as exc:
        return str(exc)
    raise AssertionError("predict() on an unfitted estimator did not raise NotFittedError")


# --------------------------------------------------------------------------
# 5. get_params, set_params, and clone
# --------------------------------------------------------------------------


def params_roundtrip(estimator, **overrides) -> dict:
    """set_params with overrides, then get_params -- the values that made the round trip."""
    estimator.set_params(**overrides)
    return estimator.get_params()


def clone_is_fresh(fitted_estimator, fitted_attr: str) -> dict:
    """clone() of a fitted estimator: same hyper-parameters, no learned state."""
    fresh = clone(fitted_estimator)
    return {
        "params_equal": fresh.get_params() == fitted_estimator.get_params(),
        "fresh_is_unfitted": not hasattr(fresh, fitted_attr),
        "original_still_fitted": hasattr(fitted_estimator, fitted_attr),
    }


# --------------------------------------------------------------------------
# 6. Pipeline and ColumnTransformer are estimators themselves
# --------------------------------------------------------------------------


def pipeline_param_keys(pipeline) -> list[str]:
    return sorted(pipeline.get_params(deep=True).keys())


def pipeline_set_nested(pipeline, **overrides):
    """set_params on the pipeline, addressing a step's own parameter by name."""
    pipeline.set_params(**overrides)
    return pipeline.get_params(deep=True)


class _CountingScaler(StandardScaler):
    """A StandardScaler that counts how many times fit() is actually called."""

    calls = 0

    def fit(self, X, y=None):
        type(self).calls += 1
        return super().fit(X, y)


def fits_per_fold(X, y, folds: int = 5, seed: int = 0) -> int:
    """How many times a preprocessing step inside a Pipeline is fit, under k-fold CV.

    Not a re-measurement of Day 143's leaked-preprocessing cost. This
    measures the mechanism that makes the leak impossible in the first
    place: cross_val_score clones the whole pipeline once per fold and
    fits the clone on that fold's training rows only.
    """
    _CountingScaler.calls = 0
    pipe = Pipeline([("scaler", _CountingScaler()), ("clf", LogisticRegression(max_iter=1000))])
    cross_val_score(pipe, X, y, cv=StratifiedKFold(folds, shuffle=True, random_state=seed))
    return _CountingScaler.calls


# --------------------------------------------------------------------------
# 7. Where "just a protocol" needs a footnote
# --------------------------------------------------------------------------


def bare_estimator_breaks_in_cross_val_score(X, y, folds: int = 5, seed: int = 0) -> str:
    """cross_val_score on the from-scratch estimator that inherits nothing.

    fit/predict/score/get_params/set_params all work fine when called
    directly. This is what stops working the moment scikit-learn's OWN
    machinery -- not our code -- needs to check whether the estimator is
    fitted, which in this version happens through `__sklearn_tags__`.
    Returns the exact AttributeError message raised.
    """
    try:
        cross_val_score(
            MajorityClassifier(), X, y, cv=StratifiedKFold(folds, shuffle=True, random_state=seed)
        )
    except AttributeError as exc:
        return str(exc)
    raise AssertionError("cross_val_score did not fail on the bare estimator, unexpectedly")


def base_estimator_works_in_pipeline_and_cv(X, y, folds: int = 5, seed: int = 0):
    """The identical classifier, inheriting ClassifierMixin and BaseEstimator,
    inside a real Pipeline, scored with a real cross_val_score."""
    pipe = Pipeline([("scaler", StandardScaler()), ("clf", MajorityClassifierBase())])
    return cross_val_score(pipe, X, y, cv=StratifiedKFold(folds, shuffle=True, random_state=seed))


# --------------------------------------------------------------------------
# 8. How many estimators implement fit?
# --------------------------------------------------------------------------


def _bare_estimator_count() -> int:
    """How many estimators all_estimators() reports in a brand-new interpreter.

    Measured via a fresh subprocess, deliberately, rather than in-process.
    Importing sklearn.experimental.enable_halving_search_cv anywhere in a
    running process registers HalvingGridSearchCV and HalvingRandomSearchCV
    PERMANENTLY for that process's remaining lifetime -- there is no way to
    un-register them. So a second in-process call to this module's own
    census, later in the same pytest session, would otherwise silently
    report the already-enabled count even when asked for the bare one. A
    subprocess has no such history and is bare every single time.
    """
    result = subprocess.run(
        [sys.executable, "-c", "from sklearn.utils import all_estimators; print(len(all_estimators()))"],
        capture_output=True,
        text=True,
        check=True,
    )
    return int(result.stdout.strip())


def estimator_census() -> dict:
    """A census of every estimator scikit-learn's own discovery mechanism finds.

    "How many estimators does scikit-learn have" has no single answer --
    all_estimators() only finds estimators registered in modules that have
    actually been imported. HalvingGridSearchCV and HalvingRandomSearchCV
    live behind sklearn.experimental.enable_halving_search_cv and are
    invisible until that import runs, whether by this function or by pure
    accident somewhere upstream (scikit-learn's own estimator_checks module
    imports it transitively, which is precisely how this was first noticed:
    the count differed depending on what had already been imported before
    this function ran). The import below is explicit for exactly that
    reason -- so the "enabled" count is deterministic regardless of the
    caller's import history, and the gap between the two counts is reported
    rather than hidden.
    """
    bare_total = _bare_estimator_count()

    # Explicit and local: makes the two Halving search estimators visible to
    # all_estimators() regardless of what any caller has already imported.
    from sklearn.experimental import enable_halving_search_cv  # noqa: F401

    discovered = all_estimators()
    total = len(discovered)
    has_fit = sum(1 for _name, klass in discovered if hasattr(klass, "fit"))
    has_transform = sum(1 for _name, klass in discovered if hasattr(klass, "transform"))
    has_predict = sum(1 for _name, klass in discovered if hasattr(klass, "predict"))
    both = sorted(
        name for name, klass in discovered if hasattr(klass, "transform") and hasattr(klass, "predict")
    )
    newly_visible = sorted(
        name for name, _klass in discovered if name in ("HalvingGridSearchCV", "HalvingRandomSearchCV")
    )
    return {
        "bare_total": bare_total,
        "total": total,
        "newly_visible_after_experimental_enable": newly_visible,
        "has_fit": has_fit,
        "has_transform": has_transform,
        "has_predict": has_predict,
        "both_transform_and_predict": both,
    }


# --------------------------------------------------------------------------
# 9. predict, predict_proba, decision_function
# --------------------------------------------------------------------------


def proba_argmax_matches_predict(X, y) -> bool:
    model = LogisticRegression(max_iter=1000).fit(X, y)
    proba = model.predict_proba(X)
    predicted_by_proba = model.classes_[np.argmax(proba, axis=1)]
    return bool(np.array_equal(predicted_by_proba, model.predict(X)))


def decision_function_matches_predict(X, y) -> bool:
    model = LogisticRegression(max_iter=1000).fit(X, y)
    df = model.decision_function(X)
    if df.ndim == 1:
        predicted = model.classes_[(df > 0).astype(int)]
    else:
        predicted = model.classes_[np.argmax(df, axis=1)]
    return bool(np.array_equal(predicted, model.predict(X)))


# --------------------------------------------------------------------------
# 10. random_state: what None actually costs
# --------------------------------------------------------------------------


def random_state_reproducibility(X, y, repeats: int = 5, spread_repeats: int = 20, split_seed: int = 0) -> dict:
    """Fit the same forest repeatedly with a fixed seed, then with none at all.

    The `random_state=42` half of this is fully deterministic. The
    `random_state=None` half draws fresh entropy from the OS on every call
    by design -- that unpredictability is exactly what is being measured --
    so only structural facts about it are asserted anywhere in this lab:
    that the fixed half is identical every time, that the unseeded half is
    not, and that its accuracy genuinely varies.
    """
    Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.3, random_state=split_seed)

    fixed_preds = []
    for _ in range(repeats):
        model = RandomForestClassifier(n_estimators=50, random_state=42).fit(Xtr, ytr)
        fixed_preds.append(tuple(model.predict(Xte).tolist()))

    none_preds = []
    for _ in range(repeats):
        model = RandomForestClassifier(n_estimators=50, random_state=None).fit(Xtr, ytr)
        none_preds.append(tuple(model.predict(Xte).tolist()))

    accs = []
    for _ in range(spread_repeats):
        model = RandomForestClassifier(n_estimators=50, random_state=None).fit(Xtr, ytr)
        accs.append(accuracy(yte, model.predict(Xte)))

    return {
        "fixed_identical_across_repeats": len(set(fixed_preds)) == 1,
        "none_distinct_prediction_vectors": len(set(none_preds)),
        "none_repeats": repeats,
        "accuracy_spread_min": round(min(accs), 4),
        "accuracy_spread_max": round(max(accs), 4),
        "accuracy_spread_sd": round(float(np.std(accs)), 4),
    }


# --------------------------------------------------------------------------
# 11. The estimator contract, checked mechanically
# --------------------------------------------------------------------------


def check_estimator_report(estimator) -> dict:
    """Run scikit-learn's own estimator_checks against `estimator` and report honestly."""
    results: dict[str, str] = {}

    def record(*, estimator, check_name, exception, status, expected_to_fail, expected_to_fail_reason):
        results[check_name] = status

    check_estimator(estimator, on_fail=None, on_skip=None, callback=record)

    return {
        "total": len(results),
        "passed": sum(1 for v in results.values() if v == "passed"),
        "failed": sorted(k for k, v in results.items() if v == "failed"),
        "skipped": sorted(k for k, v in results.items() if v == "skipped"),
    }

#!/usr/bin/env python3
"""Print every measured pair in this lab as one table.

The harness compares this output byte for byte against
expected-output/measured-values.txt, so the report is not a convenience:
it is how the lab notices that a number in the lesson has gone stale.

Two sections below print structural facts rather than captured numbers,
and say so: exercises 9 and 9b measure what random_state=None costs, and
that cost is a fresh draw of OS entropy on every run by design.
"""

import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))

from sklearn.linear_model import LogisticRegression  # noqa: E402
from sklearn.pipeline import Pipeline  # noqa: E402
from sklearn.preprocessing import StandardScaler  # noqa: E402

import estimator_lib as e  # noqa: E402


def rule(title: str) -> None:
    print()
    print(title)
    print("-" * len(title))


def main() -> None:
    print("Day 146 -- the scikit-learn estimator API, measured")
    print("=" * 52)

    data = e.classification_dataset()
    X, y = data

    rule("1. The hand-built classifier against the library one")
    print(f"  matches DummyClassifier(strategy='most_frequent') exactly: {e.matches_dummy_classifier()}")

    rule("2. What fitting actually adds")
    gained = e.gained_attributes(LogisticRegression(max_iter=1000), X, y)
    print(f"  attributes gained by fit(): {gained}")
    lib_msg = e.predict_before_fit_message(LogisticRegression(), n_features=X.shape[1])
    ours_msg = e.predict_before_fit_message(e.MajorityClassifier(), n_features=X.shape[1])
    print(f"  LogisticRegression before fit -> {lib_msg}")
    print(f"  MajorityClassifier before fit -> {ours_msg}")

    rule("3. get_params, set_params, clone")
    after = e.params_roundtrip(LogisticRegression(max_iter=1000), C=2.0)
    print(f"  set_params(C=2.0) then get_params() -> C={after['C']}, max_iter={after['max_iter']}")
    fitted = LogisticRegression(C=0.3, max_iter=1000).fit(X, y)
    print(f"  clone() of a fitted estimator: {e.clone_is_fresh(fitted, 'coef_')}")

    rule("4. Pipeline as an estimator itself")
    pipe = Pipeline([("scaler", StandardScaler()), ("clf", LogisticRegression(C=0.5, max_iter=1000))])
    keys = e.pipeline_param_keys(pipe)
    print(f"  pipeline.get_params(deep=True) has {len(keys)} keys, including {['clf__C', 'scaler__with_mean']}")
    nested = e.pipeline_set_nested(pipe, **{"clf__C": 2.0})
    print(f"  set_params(clf__C=2.0) -> clf__C={nested['clf__C']}, live step C={pipe.named_steps['clf'].C}")
    print(f"  a preprocessing step is fit {e.fits_per_fold(X, y, folds=5)} times under 5-fold cross_val_score")
    print(f"  and {e.fits_per_fold(X, y, folds=10)} times under 10-fold -- once per fold, every time")

    rule('5. Where "just a protocol" needs a footnote')
    bare_message = e.bare_estimator_breaks_in_cross_val_score(X, y)
    print("  cross_val_score on an estimator inheriting nothing from sklearn:")
    print(f"    {bare_message}")
    base_scores = e.base_estimator_works_in_pipeline_and_cv(X, y)
    rounded_scores = [round(float(score), 4) for score in base_scores]
    print(f"  the same classifier, inheriting bare BaseEstimator, inside a real Pipeline: {rounded_scores}")

    rule("6. How many estimators implement fit?")
    census = e.estimator_census()
    print(f"  bare discovery (no experimental imports): {census['bare_total']}")
    print(f"  discovered with sklearn.experimental.enable_halving_search_cv: {census['total']}, implement fit: {census['has_fit']}")
    print(f"  newly visible after that import: {census['newly_visible_after_experimental_enable']}")
    print(f"  implement transform: {census['has_transform']}, implement predict: {census['has_predict']}")
    print(f"  implement both: {len(census['both_transform_and_predict'])} -- {census['both_transform_and_predict']}")

    rule("7. predict, predict_proba, decision_function")
    print(f"  argmax(predict_proba(X)) == predict(X): {e.proba_argmax_matches_predict(X, y)}")
    print(f"  decision_function agrees with predict(X): {e.decision_function_matches_predict(X, y)}")

    rule("8. random_state: what None actually costs")
    result = e.random_state_reproducibility(X, y)
    print(f"  random_state=42, five independent fits, identical predictions: {result['fixed_identical_across_repeats']}")
    print(f"  random_state=None, five independent fits, at least two distinct vectors: {result['none_distinct_prediction_vectors'] >= 2}")
    print(f"  random_state=None, accuracy varies across 20 fits (sd > 0): {result['accuracy_spread_sd'] > 0.0}")
    print("  the exact counts and spread are fresh OS entropy every run and are NOT byte-comparable;")
    print("  one real capture lives in expected-output/FIELDS.md, never asserted as a fixed value")

    rule("9. The estimator contract, checked mechanically")
    report = e.check_estimator_report(e.MajorityClassifierBase())
    print(f"  check_estimator: {report['total']} checks, {report['passed']} passed")
    print(f"  failed: {report['failed']}")
    print(f"  skipped: {report['skipped']}")


if __name__ == "__main__":
    main()

"""The reference solutions: what the estimator API actually guarantees,
measured against the real library rather than assumed from its docs.

Every number here was captured from a real run of this file on the
authoring machine. If a number changes, the claim in the lesson is wrong
and one of the two must be fixed.
"""

import numpy as np
import pytest

from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler

import estimator_lib as e


@pytest.fixture(scope="module")
def data():
    return e.classification_dataset()


# --- 1. The hand-built estimator against the library one -----------------


def test_01_the_hand_built_classifier_matches_the_library_one():
    assert e.matches_dummy_classifier() is True


# --- 2. What fitting actually adds ----------------------------------------


def test_02_fitting_adds_exactly_five_learned_attributes(data):
    X, y = data
    gained = e.gained_attributes(LogisticRegression(max_iter=1000), X, y)
    assert gained == ["classes_", "coef_", "intercept_", "n_features_in_", "n_iter_"]
    # Every one of them is a documented convention: learned from data.
    assert all(name.endswith("_") and not name.startswith("__") for name in gained)


def test_02b_predict_before_fit_raises_notfittederror_with_a_useful_message(data):
    X, _y = data
    library_message = e.predict_before_fit_message(LogisticRegression(), n_features=X.shape[1])
    ours_message = e.predict_before_fit_message(e.MajorityClassifier(), n_features=X.shape[1])
    for message in (library_message, ours_message):
        assert "is not fitted yet" in message
        assert "Call 'fit'" in message


# --- 3. get_params, set_params, clone -------------------------------------


def test_03_get_params_and_set_params_round_trip_through_each_other():
    after = e.params_roundtrip(LogisticRegression(max_iter=1000), C=2.0)
    assert after["C"] == 2.0
    assert after["max_iter"] == 1000
    # Setting params back to what get_params() reports is a no-op.
    model = LogisticRegression(C=0.7, max_iter=500)
    same = e.params_roundtrip(model, **model.get_params())
    assert same == model.get_params()


def test_03b_clone_produces_a_fresh_unfitted_copy_with_identical_hyperparameters(data):
    X, y = data
    fitted = LogisticRegression(C=0.3, max_iter=1000).fit(X, y)
    result = e.clone_is_fresh(fitted, "coef_")
    assert result == {
        "params_equal": True,
        "fresh_is_unfitted": True,
        "original_still_fitted": True,
    }


# --- 4. Pipeline as an estimator itself -----------------------------------


def test_04_pipeline_exposes_its_steps_nested_hyperparameters():
    pipe = Pipeline([("scaler", StandardScaler()), ("clf", LogisticRegression(C=0.5, max_iter=1000))])
    keys = e.pipeline_param_keys(pipe)
    assert "clf" in keys and "scaler" in keys
    assert "clf__C" in keys and "scaler__with_mean" in keys
    # The step's own name plus "__" plus its parameter name, mechanically.
    assert len(keys) == 23


def test_04b_setting_a_nested_parameter_changes_the_live_step(data):
    pipe = Pipeline([("scaler", StandardScaler()), ("clf", LogisticRegression(C=0.5, max_iter=1000))])
    after = e.pipeline_set_nested(pipe, **{"clf__C": 2.0})
    assert after["clf__C"] == 2.0
    assert pipe.named_steps["clf"].C == 2.0


def test_05_a_pipeline_step_is_refit_once_per_cv_fold_on_training_rows_only(data):
    X, y = data
    assert e.fits_per_fold(X, y, folds=5) == 5
    assert e.fits_per_fold(X, y, folds=10) == 10


# --- 5. Where "just a protocol" needs a footnote --------------------------


def test_06_a_from_scratch_estimator_breaks_inside_cross_val_score(data):
    X, y = data
    message = e.bare_estimator_breaks_in_cross_val_score(X, y)
    assert "__sklearn_tags__" in message
    assert "BaseEstimator" in message


def test_06b_inheriting_bare_baseestimator_fixes_it(data):
    X, y = data
    scores = e.base_estimator_works_in_pipeline_and_cv(X, y)
    assert len(scores) == 5
    assert all(0.0 <= score <= 1.0 for score in scores)
    assert not any(np.isnan(scores))
    # get_params/set_params are not written anywhere in MajorityClassifierBase's
    # own source -- BaseEstimator supplies both by introspecting __init__.
    assert "get_params" not in e.MajorityClassifierBase.__dict__
    assert "set_params" not in e.MajorityClassifierBase.__dict__


# --- 6. How many estimators implement fit? --------------------------------


def test_07_scikit_learn_discovers_210_estimators_and_all_implement_fit():
    census = e.estimator_census()
    assert census["total"] == 210
    assert census["has_fit"] == 210


def test_07b_transform_and_predict_are_not_mutually_exclusive():
    census = e.estimator_census()
    assert census["has_transform"] == 90
    assert census["has_predict"] == 119
    assert len(census["both_transform_and_predict"]) == 20
    assert "KMeans" in census["both_transform_and_predict"]
    assert "Pipeline" in census["both_transform_and_predict"]


def test_07c_the_210_total_depends_on_an_explicit_experimental_import():
    census = e.estimator_census()
    # Discovery without the experimental enabler undercounts by exactly the
    # two Halving search estimators -- not a fuzzy "roughly fewer," an exact
    # gap, because that gap IS the mechanism being measured.
    assert census["bare_total"] == 208
    assert census["total"] - census["bare_total"] == 2
    assert census["newly_visible_after_experimental_enable"] == [
        "HalvingGridSearchCV",
        "HalvingRandomSearchCV",
    ]


# --- 7. predict, predict_proba, decision_function -------------------------


def test_08_argmax_of_predict_proba_equals_predict(data):
    X, y = data
    assert e.proba_argmax_matches_predict(X, y) is True


def test_08b_decision_function_agrees_with_predict_too(data):
    X, y = data
    assert e.decision_function_matches_predict(X, y) is True


# --- 8. random_state: what None actually costs -----------------------------


def test_09_a_fixed_random_state_reproduces_identical_predictions_every_time(data):
    X, y = data
    result = e.random_state_reproducibility(X, y)
    assert result["fixed_identical_across_repeats"] is True


def test_09b_random_state_none_produces_a_different_model_on_every_fit(data):
    X, y = data
    result = e.random_state_reproducibility(X, y)
    # Sampled by design -- assert the structural claim, not one captured value.
    assert result["none_distinct_prediction_vectors"] >= 2
    assert result["accuracy_spread_sd"] > 0.0


# --- 9. The estimator contract, checked mechanically ------------------------


def test_10_check_estimator_reports_48_of_52_checks_passing():
    report = e.check_estimator_report(e.MajorityClassifierBase())
    assert report["total"] == 52
    assert report["passed"] == 48
    assert report["failed"] == ["check_classifiers_regression_target", "check_classifiers_train"]
    assert report["skipped"] == ["check_array_api_input", "check_classifier_data_not_an_array"]

"""Machinery checks: the helpers behave, before any claim is made.

These five tests are solved in both `starter/` and `examples/`. They exist
so that a broken helper reports itself as a broken helper rather than as a
surprising scientific result.
"""

import numpy as np

import estimator_lib as e


def test_the_datasets_have_the_shapes_they_claim():
    X, y = e.classification_dataset(n=150, n_features=4, n_classes=3, seed=42)
    assert X.shape == (150, 4) and y.shape == (150,)
    assert set(np.unique(y).tolist()) == {0, 1, 2}

    Xs, ys = e.skewed_dataset(n=60, seed=0)
    assert Xs.shape == (60, 3) and ys.shape == (60,)
    # A genuine majority class, by construction.
    values, counts = np.unique(ys, return_counts=True)
    assert counts.max() > len(ys) / 2


def test_majority_classifier_predicts_the_class_it_saw_most_often():
    X = np.zeros((6, 2))
    y = np.array([0, 0, 0, 1, 1, 2])
    clf = e.MajorityClassifier().fit(X, y)
    assert clf.majority_class_ == 0
    assert np.array_equal(clf.predict(X), np.zeros(6, dtype=int))
    assert clf.score(X, y) == 3 / 6


def test_majority_classifier_get_params_and_set_params_agree():
    clf = e.MajorityClassifier(strategy="most_frequent")
    assert clf.get_params() == {"strategy": "most_frequent"}
    clf.set_params(strategy="prior")
    assert clf.get_params() == {"strategy": "prior"}


def test_the_counting_scaler_counts_direct_fit_calls():
    e._CountingScaler.calls = 0
    X = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
    scaler = e._CountingScaler()
    scaler.fit(X)
    scaler.fit(X)
    assert e._CountingScaler.calls == 2


def test_estimator_census_finds_known_members_in_both_lists():
    census = e.estimator_census()
    assert census["total"] == census["has_fit"], "every discovered estimator implements fit"
    assert "KMeans" in census["both_transform_and_predict"]
    assert "Pipeline" in census["both_transform_and_predict"]
    assert "LogisticRegression" not in census["both_transform_and_predict"]

lesson_id: D146
day: 146
kind: guided-build
languages:
  - python
  - bash
setup_commands:
  - cd labs/sections/machine-learning/day-146-your-first-model-with-scikit-learn
  - python3 -m venv .venv
  - .venv/bin/pip install -r requirements/requirements.txt
  - >-
    .venv/bin/python3 -c "import numpy, sklearn; print(numpy.__version__,
    sklearn.__version__)"
run_commands:
  - .venv/bin/pytest examples -q
  - .venv/bin/pytest starter -q
  - .venv/bin/python3 examples/report_measurements.py
test_commands:
  - bash tests/run_tests.sh
cleanup_commands:
  - >-
    find . -path ./.venv -prune -o -type d -name '__pycache__' -print -exec rm -rf -- {}
    +
  - rm -rf .pytest_cache
  - 'rm -rf .venv  # optional: removes the lab virtual environment'
  - 'git checkout -- starter/  # optional: reset your work'
requires_network: true
requires_api_key: false
estimated_minutes: 60
last_executed: '2026-08-27'
executed_on: >-
  macOS 26.5.2 (Apple Silicon, arm64, CPU only -- no GPU is needed or used), Python
  3.14.0, numpy 2.5.2, scikit-learn 1.9.0, pytest 9.1.1, bash 3.2.57 -- bash
  tests/run_tests.sh -> 14 checks, 0 failure(s), exit 0. pytest examples -q -> 23 passed.
  pytest starter -q -> 5 passed, 18 skipped (the five machinery checks in
  test_estimator_lib.py are solved in both directories; the eighteen exercise stubs in
  starter/test_estimator_claims.py are untouched). Everything ran through a real lab-local
  .venv created by the documented setup commands; scikit-learn pulled in scipy 1.18.1,
  joblib 1.5.3 and threadpoolctl 3.6.0 as its own dependencies, none of which this lab
  imports directly. The lab is fully offline after the pip install -- every dataset is
  generated on the spot from a seeded numpy.random.default_rng or scikit-learn's own
  synthetic-data helpers inside check_estimator, nothing is downloaded, no dataset is
  bundled, and harness check 9 confirms no URL appears anywhere in starter/ or examples/
  source. Check 7 of the harness copies examples/ into a mktemp -d scratch directory,
  confirms 23 passed, rewrites `assert census["total"] == 210` to `== 999`, confirms a
  non-zero exit naming the failing test, and removes the scratch directory. MEASURED
  PAIRS, all captured verbatim in expected-output/measured-values.txt. (1) MajorityClassifier
  -- built from scratch, inheriting nothing from scikit-learn -- produces predictions and
  predict_proba output byte-identical to DummyClassifier(strategy="most_frequent") across
  five seeded datasets. (2) Fitting LogisticRegression(max_iter=1000) adds exactly five
  attributes, all trailing-underscore: classes_, coef_, intercept_, n_features_in_, n_iter_.
  Calling predict before fit on both LogisticRegression and MajorityClassifier raises
  NotFittedError with a message containing "is not fitted yet" and "Call 'fit'". (3)
  get_params()/set_params() round-trip through each other exactly; clone() of a fitted
  LogisticRegression returns a fresh instance with identical get_params() and none of the
  fitted attributes -- the original stays fitted. (4) A Pipeline's own get_params(deep=True)
  has 23 keys, including clf__C and scaler__with_mean; set_params(clf__C=2.0) reaches
  through and mutates the live nested LogisticRegression's .C attribute directly. A
  preprocessing step wrapped in a fit-counting subclass is fit exactly 5 times under
  5-fold cross_val_score and exactly 10 times under 10-fold -- once per fold, on that
  fold's training rows only. (5) THE HEADLINE, AND A GENUINE SURPRISE: the identical
  MajorityClassifier that matched DummyClassifier exactly in (1) raises AttributeError
  the instant it is handed to cross_val_score, naming a missing __sklearn_tags__ attribute
  and recommending inheritance from BaseEstimator -- fit/predict/score/get_params/
  set_params all still work when called directly; only scikit-learn's own internal
  fitted-check, reached through Pipeline and cross_val_score, breaks. Rebuilding the exact
  same classifier as MajorityClassifierBase(ClassifierMixin, BaseEstimator), and deleting
  the hand-written get_params/set_params entirely, produces 5 real, non-NaN scores from a
  real Pipeline scored by a real cross_val_score -- and confirmed by source inspection that
  get_params/set_params are not defined anywhere in MajorityClassifierBase's own class body.
  (6) sklearn.utils.all_estimators() discovers 208 estimators bare, in a fresh subprocess
  with no experimental imports, and 210 once sklearn.experimental.enable_halving_search_cv
  has been imported -- a gap of exactly two, HalvingGridSearchCV and HalvingRandomSearchCV,
  which is undiscoverable until that import runs, explicitly or transitively. Of the 210,
  all 210 implement fit; 90 implement transform; 119 implement predict; and transform/predict
  are NOT mutually exclusive -- 20 estimators implement both, and they are exactly the
  clustering models (KMeans, MiniBatchKMeans, Birch, BisectingKMeans) and meta-estimators
  (Pipeline, GridSearchCV, RandomizedSearchCV, the Halving searches, the Stacking and
  Voting ensembles, RFE/RFECV) that legitimately need both, never a plain classifier or
  regressor. (7) For a fitted multiclass LogisticRegression, argmax(predict_proba(X),
  axis=1) mapped through classes_ equals predict(X) for every row, and the same holds for
  decision_function's argmax (multiclass) or sign (binary). (8) random_state=42: five
  independent RandomForestClassifier(n_estimators=50) fits on identical data produce five
  byte-identical prediction vectors, every run. random_state=None: five independent fits
  produced 5 of 5 distinct prediction vectors on this run, with accuracy over 20 such fits
  ranging 0.7111 to 0.8222 (sd 0.0294) -- captured once as an example in
  expected-output/FIELDS.md; the lab and report assert only the structural claim
  (identical under a fixed seed, distinct without one) because random_state=None draws
  fresh OS entropy on every call by design and is not reproducible even on this machine.
  (9) check_estimator(MajorityClassifierBase()) reports 52 checks: 48 passed, 2 skipped
  (check_array_api_input -- SCIPY_ARRAY_API not set; check_classifier_data_not_an_array --
  pandas not installed), 2 failed (check_classifiers_train -- asserts accuracy_score > 0.83,
  which a majority-class dummy cannot reach by design; check_classifiers_regression_target
  -- expects a ValueError on a continuous target, which this estimator's fit() does not
  validate for). THREE HONESTY CALLS. FIRST, and the most important: the whole premise "the
  estimator API is a protocol, not magic" needed a footnote the first draft of this lab did
  not have. The plan going in was to prove the from-scratch classifier works inside a real
  Pipeline and a real cross_val_score, full stop. It does not, in scikit-learn 1.9.0,
  without inheriting BaseEstimator -- and that AttributeError is the more interesting,
  more honest finding, so the lab keeps both halves: the from-scratch version proves the
  five methods are necessary and sufficient for direct use, and the BaseEstimator version
  proves what else composability actually requires in this version of the library. SECOND:
  exercise 9's numbers are fundamentally unrepeatable by construction, so
  report_measurements.py deliberately does not print the sampled counts or accuracy
  figures in its byte-compared output -- only booleans that are true on every run -- and
  the one real captured example lives in FIELDS.md, explicitly labelled as an example
  rather than an expected value. THIRD, found on integration review and not by this lab's
  own first pass: estimator_census() originally called all_estimators() with no explicit
  enabling imports, so its result depended on what a CALLER had already imported rather
  than on the scikit-learn version alone -- a bare interpreter reports 208, but importing
  this very library (which transitively imports sklearn.utils.estimator_checks, which
  itself imports the halving-search enabler) silently bumped the count to 210 before the
  census ever ran. Fixed by making the enabling import explicit inside estimator_census()
  and by measuring the bare count in a genuinely fresh subprocess rather than in-process --
  necessary because importing the enabler anywhere in a running process registers
  HalvingGridSearchCV and HalvingRandomSearchCV permanently for that process's remaining
  lifetime, so a second in-process bare reading later in the same pytest session would
  otherwise silently report the already-enabled count. Both totals are now reported
  together (208 bare, 210 enabled) rather than one hiding the other, and exercise 7c
  asserts the gap by name rather than by a bare total alone.

# Requirements

`requirements.txt` pins the three packages this lab imports directly, at
the exact versions the captured output in `expected-output/` was produced
with:

```
numpy==2.5.2
scikit-learn==1.9.0
pytest==9.1.1
```

Installing scikit-learn also pulls in scipy, joblib and threadpoolctl as
its own dependencies. This lab imports none of them directly and does not
pin them; the versions present during capture are recorded in
`../expected-output/FIELDS.md`.

## Why the versions are pinned exactly

The headline finding in this lab — that a hand-built estimator inheriting
nothing from scikit-learn breaks inside `cross_val_score` with an
`AttributeError` naming `__sklearn_tags__` — is a fact about this specific
version of scikit-learn, not a fact about the library forever. `__sklearn_tags__`
is recent scikit-learn machinery; an older version might fail differently,
or not fail at all, and a much newer one might change the message text
again. Pin the version and the failure — and the fix — reproduce exactly.

`sklearn.utils.all_estimators()`'s census (exercise 7) is also
version-specific: scikit-learn adds and removes estimators between
releases, so the totals of 208 (bare) and 210 (with
`sklearn.experimental.enable_halving_search_cv` imported) belong to 1.9.0
specifically. The *fact that a bare count and an enabled count differ at
all* is not version-specific -- `all_estimators()` has only ever found
what has actually been imported, in every scikit-learn version with
experimental estimators -- but which estimators are gated behind an
experimental import, and how many there are, changes release to release.

What does not depend on the pins: the *shape* of every finding. Fitting
always adds attributes with a trailing underscore; `get_params`/`set_params`
always round-trip; a `Pipeline` step is always refit once per
cross-validation fold; `predict()` is always `argmax(predict_proba())`;
and a hand-built estimator that satisfies `fit`/`predict`/`score`/`get_params`/
`set_params` will always work when called directly, whether or not it
happens to interoperate with every piece of scikit-learn's own machinery.
Harness check 8 re-confirms four of those shapes at seeds and parameters
the lesson never quotes.

`expected-output/FIELDS.md` separates the two categories in full, and it
is worth reading before you conclude that a mismatch is a bug.

## Installing

From the lab directory:

```bash
python3 -m venv .venv
.venv/bin/pip install -r requirements/requirements.txt
```

The install step needs the network. Everything after it is offline: every
dataset in this lab is generated on the spot from a seeded generator, and
nothing is downloaded.

## Free and open-source status

All three packages are free and open source — NumPy and scikit-learn under
the BSD 3-Clause licence, pytest under the MIT licence. There is no paid
tier, no account and no API key anywhere in this lab.

numpy==2.5.2
scikit-learn==1.9.0
pytest==9.1.1

# Day 146 lab brief — The Estimator API, From Scratch

Days 141-145 called `.fit(X, y)` and `.predict(X)` on scikit-learn objects
dozens of times without ever explaining what those two words mean. This
lab explains them, by building a classifier that implements the API by
hand and then measuring exactly where hand-written code and library code
agree — and, honestly, one place where they stop agreeing.

## The claim you are here to test

> The estimator API is a protocol, not magic — until it isn't.

Exercise 1 builds `MajorityClassifier`, a class that inherits nothing from
scikit-learn. `__init__` stores one hyper-parameter. `fit` learns three
things and names each with a trailing underscore. `predict`, `predict_proba`,
`score`, `get_params` and `set_params` are five ordinary methods, written
out in full. Its predictions and probabilities turn out to be **byte
identical** to `sklearn.dummy.DummyClassifier(strategy="most_frequent")` —
not similar, not close, identical — because both objects are computing the
same thing from the same rule.

## Where the protocol needs a footnote

Exercise 6 is the one that should surprise you. The exact same hand-built
classifier — the one whose predictions matched the library exactly —
raises an `AttributeError` the instant you hand it to `cross_val_score`:

```text
AttributeError: 'MajorityClassifier' object has no attribute '__sklearn_tags__'.
...Make sure to inherit from `BaseEstimator`...
```

`fit`, `predict` and `score` all still work fine when you call them
directly. What breaks is scikit-learn's *own* machinery, which needs to
check whether an estimator is fitted and does so, in this version, through
a method that only `BaseEstimator` supplies. Add one line —
`class MajorityClassifierBase(ClassifierMixin, BaseEstimator):` — delete
`get_params` and `set_params` entirely, since `BaseEstimator` now supplies
them by inspecting `__init__`'s signature, and the classifier works inside
a real `Pipeline` and a real `cross_val_score`.

That is the honest version of "it's just a protocol": five methods are
*necessary*, and this lab proves it by making them work standalone. They
are no longer *sufficient* for full interoperability in this version of
the library, and this lab proves that too, by breaking on purpose.

## What else this lab measures

| # | What it checks |
| --- | --- |
| 2 | The exact set of attributes `fit()` adds — every one ends in `_` |
| 3 | `get_params`/`set_params` round-trip; `clone()` produces a fresh, unfitted copy |
| 4-5 | `Pipeline` is an estimator itself, with nested parameters, and refits every step once per cross-validation fold on training rows only |
| 6 | Where "just a protocol" needs a footnote (above) |
| 7 | How many of scikit-learn's 210 discovered estimators implement `fit` (all of them), and how many implement both `transform` and `predict` (20 — clustering models and meta-estimators, not classifiers) |
| 8 | `predict()` is `argmax(predict_proba())`, restated as `classes_[...]` |
| 9 | What `random_state=None` costs, measured as five independent, non-reproducible fits |
| 10 | `check_estimator()` against the real estimator-conformance suite: 48 of 52 checks pass, and this lab explains, honestly, why the other two do not |

## How to work

1. Build the environment (see the lab `README.md`).
2. Run `.venv/bin/pytest starter -q`. You will see five passes (the
   machinery checks in `test_estimator_lib.py`) and eighteen skips.
3. Replace one `pytest.skip(...)` at a time with real code. The skip text
   names the exact helper and the exact value to assert.
4. Print the measured pair in every exercise. A number you did not print
   is a number you did not look at.
5. When you want the whole measured table at once, run
   `.venv/bin/python3 examples/report_measurements.py`.

Do not run `pytest starter examples` in one invocation. Both directories
define `estimator_lib.py`, `test_estimator_lib.py` and
`test_estimator_claims.py`; pytest aborts on the module-name collision.
Run them separately, always.

## Exercises 9 and 9b are honest about being unrepeatable

`random_state=None` draws fresh entropy from the operating system on every
single call, by design — that unpredictability is the entire point of the
exercise. So `report_measurements.py` prints only structural facts about
it ("identical across five fits: True", "distinct across five fits: True")
rather than the sampled counts and accuracy figures themselves, and the
tests assert the same structural claims. One real capture of the sampled
numbers, from one specific run, lives in `expected-output/FIELDS.md` —
labelled as an example, never as a value your run should reproduce.

"""The estimator API, measured: what fit/predict/score/get_params/set_params
actually buy you, and where the "it's just a protocol" story needs a footnote.

Days 141-145 called `.fit()` and `.predict()` on scikit-learn objects
without ever explaining what those calls mean. This module builds a
classifier from first principles -- no inheritance at all -- to show that
the four core verbs are a protocol you can implement yourself, and then
measures exactly where that protocol stops being sufficient on its own in
this version of the library.

Everything here is deterministic given a seed, except the two functions
that exist specifically to measure what `random_state=None` costs, which
are documented as such.
"""

from __future__ import annotations

import subprocess
import sys

import numpy as np

from sklearn.base import BaseEstimator, ClassifierMixin, clone
from sklearn.dummy import DummyClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.exceptions import NotFittedError
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import StratifiedKFold, cross_val_score, train_test_split
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.utils import all_estimators
from sklearn.utils.estimator_checks import check_estimator
from sklearn.utils.validation import check_is_fitted, validate_data


def accuracy(y_true, y_pred) -> float:
    return float(np.mean(np.asarray(y_true) == np.asarray(y_pred)))


# --------------------------------------------------------------------------
# 1. A classifier built entirely from first principles
# --------------------------------------------------------------------------


class MajorityClassifier:
    """fit/predict/score/get_params/set_params, written out by hand.

    Nothing here inherits from scikit-learn. `__init__` stores its
    hyper-parameter exactly as given and computes nothing; `fit` is where
    everything named with a trailing underscore gets learned. It always
    predicts the class that was most frequent in the training labels --
    exactly what ``DummyClassifier(strategy="most_frequent")`` does, which
    is what makes its output directly checkable against the library.
    """

    def __init__(self, strategy: str = "most_frequent"):
        self.strategy = strategy

    def fit(self, X, y):
        X = np.asarray(X)
        y = np.asarray(y)
        self.classes_, counts = np.unique(y, return_counts=True)
        self.majority_class_ = self.classes_[np.argmax(counts)]
        self.n_features_in_ = X.shape[1]
        return self

    def _require_fitted(self):
        if not hasattr(self, "majority_class_"):
            raise NotFittedError(
                "This MajorityClassifier instance is not fitted yet. Call "
                "'fit' with appropriate arguments before using this estimator."
            )

    def predict(self, X):
        self._require_fitted()
        X = np.asarray(X)
        return np.full(X.shape[0], self.majority_class_)

    def predict_proba(self, X):
        self._require_fitted()
        X = np.asarray(X)
        row = np.zeros(len(self.classes_))
        row[int(np.argmax(self.classes_ == self.majority_class_))] = 1.0
        return np.tile(row, (X.shape[0], 1))

    def score(self, X, y):
        return accuracy(y, self.predict(X))

    def get_params(self, deep: bool = True) -> dict:
        return {"strategy": self.strategy}

    def set_params(self, **params):
        for key, value in params.items():
            setattr(self, key, value)
        return self


class MajorityClassifierBase(ClassifierMixin, BaseEstimator):
    """The same classifier, this time built on scikit-learn's own base classes.

    `get_params` and `set_params` are gone from the source -- `BaseEstimator`
    supplies both by inspecting `__init__`'s signature, which is why
    `__init__` must do nothing but store its arguments: the introspection
    only works if the parameter names and the stored attribute names match
    exactly. What else `BaseEstimator` supplies, silently, is
    `__sklearn_tags__` -- which exercise 3 shows is no longer optional for
    `Pipeline` and `cross_val_score` in this version.
    """

    def __init__(self, strategy: str = "most_frequent"):
        self.strategy = strategy

    def fit(self, X, y):
        X, y = validate_data(self, X, y)
        self.classes_, counts = np.unique(y, return_counts=True)
        self.majority_class_ = self.classes_[np.argmax(counts)]
        self.class_prior_ = counts / counts.sum()
        return self

    def predict(self, X):
        check_is_fitted(self)
        X = validate_data(self, X, reset=False)
        return np.full(X.shape[0], self.majority_class_)

    def predict_proba(self, X):
        check_is_fitted(self)
        X = validate_data(self, X, reset=False)
        row = np.zeros(len(self.classes_))
        row[int(np.argmax(self.class_prior_))] = 1.0
        return np.tile(row, (X.shape[0], 1))


# --------------------------------------------------------------------------
# 2. Datasets shared by the exercises below
# --------------------------------------------------------------------------


def classification_dataset(n: int = 150, n_features: int = 4, n_classes: int = 3, seed: int = 42):
    """A small, well-separated classification dataset. Nothing rare, nothing grouped."""
    rng = np.random.default_rng(seed)
    centers = rng.normal(scale=4.0, size=(n_classes, n_features))
    y = rng.integers(0, n_classes, size=n)
    X = centers[y] + rng.normal(size=(n, n_features))
    return X, y


def skewed_dataset(n: int = 80, seed: int = 0):
    """A dataset with a clear majority class, for the DummyClassifier comparison."""
    rng = np.random.default_rng(seed)
    X = rng.normal(size=(n, 3))
    y = rng.integers(0, 3, size=n)
    y[: (2 * n) // 3] = 0  # force an unambiguous majority
    return X, y


# --------------------------------------------------------------------------
# 3. Does the hand-built estimator agree with the library one?
# --------------------------------------------------------------------------


def matches_dummy_classifier(seeds=range(5)) -> bool:
    """True only if MajorityClassifier's output is byte-identical to DummyClassifier's."""
    for seed in seeds:
        X, y = skewed_dataset(n=60 + seed * 11, seed=seed)
        ours = MajorityClassifier().fit(X, y)
        theirs = DummyClassifier(strategy="most_frequent").fit(X, y)
        if not np.array_equal(ours.predict(X), theirs.predict(X)):
            return False
        if not np.array_equal(ours.predict_proba(X), theirs.predict_proba(X)):
            return False
    return True


# --------------------------------------------------------------------------
# 4. What fitting actually adds
# --------------------------------------------------------------------------


def gained_attributes(estimator, X, y) -> list[str]:
    """dir(estimator) after fit, minus dir(estimator) before -- what fit() adds."""
    before = set(dir(estimator))
    estimator.fit(X, y)
    after = set(dir(estimator))
    return sorted(after - before)


def predict_before_fit_message(estimator, n_features: int = 4) -> str:
    """Call predict on an unfitted estimator and return the exception's message."""
    try:
        estimator.predict(np.zeros((3, n_features)))
    except NotFittedError as exc:
        return str(exc)
    raise AssertionError("predict() on an unfitted estimator did not raise NotFittedError")


# --------------------------------------------------------------------------
# 5. get_params, set_params, and clone
# --------------------------------------------------------------------------


def params_roundtrip(estimator, **overrides) -> dict:
    """set_params with overrides, then get_params -- the values that made the round trip."""
    estimator.set_params(**overrides)
    return estimator.get_params()


def clone_is_fresh(fitted_estimator, fitted_attr: str) -> dict:
    """clone() of a fitted estimator: same hyper-parameters, no learned state."""
    fresh = clone(fitted_estimator)
    return {
        "params_equal": fresh.get_params() == fitted_estimator.get_params(),
        "fresh_is_unfitted": not hasattr(fresh, fitted_attr),
        "original_still_fitted": hasattr(fitted_estimator, fitted_attr),
    }


# --------------------------------------------------------------------------
# 6. Pipeline and ColumnTransformer are estimators themselves
# --------------------------------------------------------------------------


def pipeline_param_keys(pipeline) -> list[str]:
    return sorted(pipeline.get_params(deep=True).keys())


def pipeline_set_nested(pipeline, **overrides):
    """set_params on the pipeline, addressing a step's own parameter by name."""
    pipeline.set_params(**overrides)
    return pipeline.get_params(deep=True)


class _CountingScaler(StandardScaler):
    """A StandardScaler that counts how many times fit() is actually called."""

    calls = 0

    def fit(self, X, y=None):
        type(self).calls += 1
        return super().fit(X, y)


def fits_per_fold(X, y, folds: int = 5, seed: int = 0) -> int:
    """How many times a preprocessing step inside a Pipeline is fit, under k-fold CV.

    Not a re-measurement of Day 143's leaked-preprocessing cost. This
    measures the mechanism that makes the leak impossible in the first
    place: cross_val_score clones the whole pipeline once per fold and
    fits the clone on that fold's training rows only.
    """
    _CountingScaler.calls = 0
    pipe = Pipeline([("scaler", _CountingScaler()), ("clf", LogisticRegression(max_iter=1000))])
    cross_val_score(pipe, X, y, cv=StratifiedKFold(folds, shuffle=True, random_state=seed))
    return _CountingScaler.calls


# --------------------------------------------------------------------------
# 7. Where "just a protocol" needs a footnote
# --------------------------------------------------------------------------


def bare_estimator_breaks_in_cross_val_score(X, y, folds: int = 5, seed: int = 0) -> str:
    """cross_val_score on the from-scratch estimator that inherits nothing.

    fit/predict/score/get_params/set_params all work fine when called
    directly. This is what stops working the moment scikit-learn's OWN
    machinery -- not our code -- needs to check whether the estimator is
    fitted, which in this version happens through `__sklearn_tags__`.
    Returns the exact AttributeError message raised.
    """
    try:
        cross_val_score(
            MajorityClassifier(), X, y, cv=StratifiedKFold(folds, shuffle=True, random_state=seed)
        )
    except AttributeError as exc:
        return str(exc)
    raise AssertionError("cross_val_score did not fail on the bare estimator, unexpectedly")


def base_estimator_works_in_pipeline_and_cv(X, y, folds: int = 5, seed: int = 0):
    """The identical classifier, inheriting ClassifierMixin and BaseEstimator,
    inside a real Pipeline, scored with a real cross_val_score."""
    pipe = Pipeline([("scaler", StandardScaler()), ("clf", MajorityClassifierBase())])
    return cross_val_score(pipe, X, y, cv=StratifiedKFold(folds, shuffle=True, random_state=seed))


# --------------------------------------------------------------------------
# 8. How many estimators implement fit?
# --------------------------------------------------------------------------


def _bare_estimator_count() -> int:
    """How many estimators all_estimators() reports in a brand-new interpreter.

    Measured via a fresh subprocess, deliberately, rather than in-process.
    Importing sklearn.experimental.enable_halving_search_cv anywhere in a
    running process registers HalvingGridSearchCV and HalvingRandomSearchCV
    PERMANENTLY for that process's remaining lifetime -- there is no way to
    un-register them. So a second in-process call to this module's own
    census, later in the same pytest session, would otherwise silently
    report the already-enabled count even when asked for the bare one. A
    subprocess has no such history and is bare every single time.
    """
    result = subprocess.run(
        [sys.executable, "-c", "from sklearn.utils import all_estimators; print(len(all_estimators()))"],
        capture_output=True,
        text=True,
        check=True,
    )
    return int(result.stdout.strip())


def estimator_census() -> dict:
    """A census of every estimator scikit-learn's own discovery mechanism finds.

    "How many estimators does scikit-learn have" has no single answer --
    all_estimators() only finds estimators registered in modules that have
    actually been imported. HalvingGridSearchCV and HalvingRandomSearchCV
    live behind sklearn.experimental.enable_halving_search_cv and are
    invisible until that import runs, whether by this function or by pure
    accident somewhere upstream (scikit-learn's own estimator_checks module
    imports it transitively, which is precisely how this was first noticed:
    the count differed depending on what had already been imported before
    this function ran). The import below is explicit for exactly that
    reason -- so the "enabled" count is deterministic regardless of the
    caller's import history, and the gap between the two counts is reported
    rather than hidden.
    """
    bare_total = _bare_estimator_count()

    # Explicit and local: makes the two Halving search estimators visible to
    # all_estimators() regardless of what any caller has already imported.
    from sklearn.experimental import enable_halving_search_cv  # noqa: F401

    discovered = all_estimators()
    total = len(discovered)
    has_fit = sum(1 for _name, klass in discovered if hasattr(klass, "fit"))
    has_transform = sum(1 for _name, klass in discovered if hasattr(klass, "transform"))
    has_predict = sum(1 for _name, klass in discovered if hasattr(klass, "predict"))
    both = sorted(
        name for name, klass in discovered if hasattr(klass, "transform") and hasattr(klass, "predict")
    )
    newly_visible = sorted(
        name for name, _klass in discovered if name in ("HalvingGridSearchCV", "HalvingRandomSearchCV")
    )
    return {
        "bare_total": bare_total,
        "total": total,
        "newly_visible_after_experimental_enable": newly_visible,
        "has_fit": has_fit,
        "has_transform": has_transform,
        "has_predict": has_predict,
        "both_transform_and_predict": both,
    }


# --------------------------------------------------------------------------
# 9. predict, predict_proba, decision_function
# --------------------------------------------------------------------------


def proba_argmax_matches_predict(X, y) -> bool:
    model = LogisticRegression(max_iter=1000).fit(X, y)
    proba = model.predict_proba(X)
    predicted_by_proba = model.classes_[np.argmax(proba, axis=1)]
    return bool(np.array_equal(predicted_by_proba, model.predict(X)))


def decision_function_matches_predict(X, y) -> bool:
    model = LogisticRegression(max_iter=1000).fit(X, y)
    df = model.decision_function(X)
    if df.ndim == 1:
        predicted = model.classes_[(df > 0).astype(int)]
    else:
        predicted = model.classes_[np.argmax(df, axis=1)]
    return bool(np.array_equal(predicted, model.predict(X)))


# --------------------------------------------------------------------------
# 10. random_state: what None actually costs
# --------------------------------------------------------------------------


def random_state_reproducibility(X, y, repeats: int = 5, spread_repeats: int = 20, split_seed: int = 0) -> dict:
    """Fit the same forest repeatedly with a fixed seed, then with none at all.

    The `random_state=42` half of this is fully deterministic. The
    `random_state=None` half draws fresh entropy from the OS on every call
    by design -- that unpredictability is exactly what is being measured --
    so only structural facts about it are asserted anywhere in this lab:
    that the fixed half is identical every time, that the unseeded half is
    not, and that its accuracy genuinely varies.
    """
    Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.3, random_state=split_seed)

    fixed_preds = []
    for _ in range(repeats):
        model = RandomForestClassifier(n_estimators=50, random_state=42).fit(Xtr, ytr)
        fixed_preds.append(tuple(model.predict(Xte).tolist()))

    none_preds = []
    for _ in range(repeats):
        model = RandomForestClassifier(n_estimators=50, random_state=None).fit(Xtr, ytr)
        none_preds.append(tuple(model.predict(Xte).tolist()))

    accs = []
    for _ in range(spread_repeats):
        model = RandomForestClassifier(n_estimators=50, random_state=None).fit(Xtr, ytr)
        accs.append(accuracy(yte, model.predict(Xte)))

    return {
        "fixed_identical_across_repeats": len(set(fixed_preds)) == 1,
        "none_distinct_prediction_vectors": len(set(none_preds)),
        "none_repeats": repeats,
        "accuracy_spread_min": round(min(accs), 4),
        "accuracy_spread_max": round(max(accs), 4),
        "accuracy_spread_sd": round(float(np.std(accs)), 4),
    }


# --------------------------------------------------------------------------
# 11. The estimator contract, checked mechanically
# --------------------------------------------------------------------------


def check_estimator_report(estimator) -> dict:
    """Run scikit-learn's own estimator_checks against `estimator` and report honestly."""
    results: dict[str, str] = {}

    def record(*, estimator, check_name, exception, status, expected_to_fail, expected_to_fail_reason):
        results[check_name] = status

    check_estimator(estimator, on_fail=None, on_skip=None, callback=record)

    return {
        "total": len(results),
        "passed": sum(1 for v in results.values() if v == "passed"),
        "failed": sorted(k for k, v in results.items() if v == "failed"),
        "skipped": sorted(k for k, v in results.items() if v == "skipped"),
    }

"""Seventeen exercises in what the scikit-learn estimator API actually
guarantees. Read `00_brief.md` first. Each function below is a
`pytest.skip` naming exactly what to build and what to assert; replace the
skip with real code. `estimator_lib.py` is complete -- it is the
machinery, not the exercise.

Run this suite on its own:

    .venv/bin/pytest starter -q

Never run `pytest starter examples` in one invocation: both directories
define modules with the same names and pytest aborts on the collision.
"""

import numpy as np  # noqa: F401  (you will need it)
import pytest

from sklearn.linear_model import LogisticRegression  # noqa: F401
from sklearn.pipeline import Pipeline  # noqa: F401
from sklearn.preprocessing import StandardScaler  # noqa: F401

import estimator_lib as e  # noqa: F401  (you will need it)


@pytest.fixture(scope="module")
def data():
    return e.classification_dataset()


def test_01_the_hand_built_classifier_matches_the_library_one():
    pytest.skip(
        "Assert e.matches_dummy_classifier() is True. MajorityClassifier "
        "inherits nothing from scikit-learn -- it implements fit, predict, "
        "predict_proba, score, get_params and set_params by hand -- yet its "
        "predictions and probabilities are byte-identical to "
        "DummyClassifier(strategy='most_frequent') across five seeds."
    )


def test_02_fitting_adds_exactly_five_learned_attributes(data):
    pytest.skip(
        "Assert e.gained_attributes(LogisticRegression(max_iter=1000), X, y) "
        "equals ['classes_', 'coef_', 'intercept_', 'n_features_in_', "
        "'n_iter_'] -- computed as dir(model) after fit() minus dir(model) "
        "before it. Then assert every gained name ends with '_' and does "
        "not start with '__'. The trailing underscore is not a style "
        "choice; it is a documented convention meaning 'learned from data', "
        "and it is why you can inspect a fitted model's guts without "
        "reading its source."
    )


def test_02b_predict_before_fit_raises_notfittederror_with_a_useful_message(data):
    pytest.skip(
        "Call e.predict_before_fit_message on an unfitted LogisticRegression "
        "and on an unfitted e.MajorityClassifier. Assert both returned "
        "messages contain 'is not fitted yet' and \"Call 'fit'\". The "
        "library estimator and the one built from scratch fail the same "
        "way, for the same reason: neither has anything with a trailing "
        "underscore yet."
    )


def test_03_get_params_and_set_params_round_trip_through_each_other():
    pytest.skip(
        "Assert e.params_roundtrip(LogisticRegression(max_iter=1000), C=2.0) "
        "reports C=2.0 and max_iter=1000 unchanged. Then build a "
        "LogisticRegression(C=0.7, max_iter=500), call "
        "e.params_roundtrip(model, **model.get_params()), and assert the "
        "result equals model.get_params() -- setting params back to what "
        "get_params() already reports must be a no-op. This round trip is "
        "what makes GridSearchCV possible at all: it is nothing more than "
        "get_params, set_params, fit, score, repeated."
    )


def test_03b_clone_produces_a_fresh_unfitted_copy_with_identical_hyperparameters(data):
    pytest.skip(
        "Fit a LogisticRegression(C=0.3, max_iter=1000) on the data fixture. "
        "Call e.clone_is_fresh(fitted, 'coef_') and assert the result equals "
        "{'params_equal': True, 'fresh_is_unfitted': True, "
        "'original_still_fitted': True}. clone() copies hyper-parameters, "
        "never learned state -- which is exactly what lets cross-validation "
        "give every fold a genuinely fresh model."
    )


def test_04_pipeline_exposes_its_steps_nested_hyperparameters():
    pytest.skip(
        "Build Pipeline([('scaler', StandardScaler()), ('clf', "
        "LogisticRegression(C=0.5, max_iter=1000))]). Assert "
        "e.pipeline_param_keys(pipe) contains 'clf', 'scaler', 'clf__C' and "
        "'scaler__with_mean', and that it has exactly 23 entries in total. "
        "A Pipeline is an estimator itself: its own get_params() reaches "
        "into every step's get_params() and prefixes each key with "
        "'<step name>__'."
    )


def test_04b_setting_a_nested_parameter_changes_the_live_step(data):
    pytest.skip(
        "Build the same Pipeline as the previous exercise. Call "
        "e.pipeline_set_nested(pipe, **{'clf__C': 2.0}) and assert the "
        "returned dict has clf__C == 2.0, then assert "
        "pipe.named_steps['clf'].C == 2.0 directly -- the nested set_params "
        "call reached through the Pipeline and mutated the actual "
        "LogisticRegression object living inside it."
    )


def test_05_a_pipeline_step_is_refit_once_per_cv_fold_on_training_rows_only(data):
    pytest.skip(
        "Assert e.fits_per_fold(X, y, folds=5) == 5 and "
        "e.fits_per_fold(X, y, folds=10) == 10. Every fold gets a freshly "
        "cloned copy of the whole pipeline, fit on that fold's training "
        "rows alone -- which is the object-model mechanism that makes Day "
        "143's rule ('anything fitted is fitted on training rows only') "
        "enforceable rather than merely advisable."
    )


def test_06_a_from_scratch_estimator_breaks_inside_cross_val_score(data):
    pytest.skip(
        "Call e.bare_estimator_breaks_in_cross_val_score(X, y) and assert "
        "the returned message contains '__sklearn_tags__' and "
        "'BaseEstimator'. e.MajorityClassifier's fit/predict/score all work "
        "fine when called directly -- this failure comes from "
        "scikit-learn's OWN machinery, which needs to check whether the "
        "estimator is fitted and does so through a method only "
        "BaseEstimator supplies."
    )


def test_06b_inheriting_bare_baseestimator_fixes_it(data):
    pytest.skip(
        "Call e.base_estimator_works_in_pipeline_and_cv(X, y) and assert it "
        "returns 5 scores, all between 0.0 and 1.0, none of them NaN. Then "
        "assert 'get_params' not in e.MajorityClassifierBase.__dict__ and "
        "likewise for 'set_params' -- neither is written anywhere in that "
        "class's own source. BaseEstimator supplies both by inspecting "
        "__init__'s signature, which is the actual mechanism behind the "
        "word 'boilerplate'."
    )


def test_07_scikit_learn_discovers_210_estimators_and_all_implement_fit():
    pytest.skip(
        "Call e.estimator_census() and assert census['total'] == 210 and "
        "census['has_fit'] == 210. Every single one -- classifiers, "
        "regressors, transformers, clusterers, meta-estimators -- "
        "implements fit. That is the whole protocol's foundation."
    )


def test_07b_transform_and_predict_are_not_mutually_exclusive():
    pytest.skip(
        "From the same census, assert has_transform == 90, has_predict == "
        "119, and len(both_transform_and_predict) == 20. Assert 'KMeans' "
        "and 'Pipeline' are both in that list. A plain classifier must "
        "never grow a transform method -- but a clustering estimator "
        "legitimately has both predict (which cluster) and transform "
        "(distance to every cluster centre), and a meta-estimator like "
        "Pipeline inherits both by wrapping whatever it is given."
    )


def test_07c_the_210_total_depends_on_an_explicit_experimental_import():
    pytest.skip(
        "Assert census['bare_total'] == 208, that census['total'] - "
        "census['bare_total'] == 2, and that "
        "census['newly_visible_after_experimental_enable'] equals "
        "['HalvingGridSearchCV', 'HalvingRandomSearchCV']. "
        "'How many estimators does scikit-learn have' has no single "
        "answer: all_estimators() only sees estimators registered in "
        "modules that have actually been imported, and these two live "
        "behind sklearn.experimental.enable_halving_search_cv. The gap is "
        "the mechanism this exercise measures, not the total by itself."
    )


def test_08_argmax_of_predict_proba_equals_predict(data):
    pytest.skip(
        "Assert e.proba_argmax_matches_predict(X, y) is True. predict() and "
        "predict_proba() are not two independent sources of truth -- "
        "predict() is defined as classes_[argmax(predict_proba(X), axis=1)] "
        "on every fitted classifier that has both."
    )


def test_08b_decision_function_agrees_with_predict_too(data):
    pytest.skip(
        "Assert e.decision_function_matches_predict(X, y) is True. For a "
        "binary classifier, predict() is (decision_function(X) > 0); for "
        "multiclass it is classes_[argmax(decision_function(X), axis=1)]. "
        "Three methods, one underlying score."
    )


def test_09_a_fixed_random_state_reproduces_identical_predictions_every_time(data):
    pytest.skip(
        "Call e.random_state_reproducibility(X, y) and assert "
        "result['fixed_identical_across_repeats'] is True. Fitting the same "
        "RandomForestClassifier(random_state=42) five times on the same "
        "data produces five byte-identical prediction vectors."
    )


def test_09b_random_state_none_produces_a_different_model_on_every_fit(data):
    pytest.skip(
        "From the same result, assert "
        "result['none_distinct_prediction_vectors'] >= 2 and "
        "result['accuracy_spread_sd'] > 0.0. random_state=None draws fresh "
        "entropy from the OS on every call by design, so only the "
        "structural claim -- that it varies at all -- is asserted here, "
        "never one captured accuracy figure."
    )


def test_10_check_estimator_reports_48_of_52_checks_passing():
    pytest.skip(
        "Call e.check_estimator_report(e.MajorityClassifierBase()) and "
        "assert report == {'total': 52, 'passed': 48, 'failed': "
        "['check_classifiers_regression_target', 'check_classifiers_train'], "
        "'skipped': ['check_array_api_input', "
        "'check_classifier_data_not_an_array']}. Read troubleshooting.md for "
        "why each of those two checks genuinely fails -- both are honest "
        "findings, not bugs in this lab."
    )

"""Machinery checks: the helpers behave, before any claim is made.

These five tests are solved in both `starter/` and `examples/`. They exist
so that a broken helper reports itself as a broken helper rather than as a
surprising scientific result.
"""

import numpy as np

import estimator_lib as e


def test_the_datasets_have_the_shapes_they_claim():
    X, y = e.classification_dataset(n=150, n_features=4, n_classes=3, seed=42)
    assert X.shape == (150, 4) and y.shape == (150,)
    assert set(np.unique(y).tolist()) == {0, 1, 2}

    Xs, ys = e.skewed_dataset(n=60, seed=0)
    assert Xs.shape == (60, 3) and ys.shape == (60,)
    # A genuine majority class, by construction.
    values, counts = np.unique(ys, return_counts=True)
    assert counts.max() > len(ys) / 2


def test_majority_classifier_predicts_the_class_it_saw_most_often():
    X = np.zeros((6, 2))
    y = np.array([0, 0, 0, 1, 1, 2])
    clf = e.MajorityClassifier().fit(X, y)
    assert clf.majority_class_ == 0
    assert np.array_equal(clf.predict(X), np.zeros(6, dtype=int))
    assert clf.score(X, y) == 3 / 6


def test_majority_classifier_get_params_and_set_params_agree():
    clf = e.MajorityClassifier(strategy="most_frequent")
    assert clf.get_params() == {"strategy": "most_frequent"}
    clf.set_params(strategy="prior")
    assert clf.get_params() == {"strategy": "prior"}


def test_the_counting_scaler_counts_direct_fit_calls():
    e._CountingScaler.calls = 0
    X = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
    scaler = e._CountingScaler()
    scaler.fit(X)
    scaler.fit(X)
    assert e._CountingScaler.calls == 2


def test_estimator_census_finds_known_members_in_both_lists():
    census = e.estimator_census()
    assert census["total"] == census["has_fit"], "every discovered estimator implements fit"
    assert "KMeans" in census["both_transform_and_predict"]
    assert "Pipeline" in census["both_transform_and_predict"]
    assert "LogisticRegression" not in census["both_transform_and_predict"]

#!/usr/bin/env bash
# Day 146 lab harness: "The Estimator API, From Scratch"
#
# Prints "N checks, M failure(s)" and exits 0 only when M is zero.
set -u

LAB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$LAB_DIR"

PYTHON="${PYTHON:-.venv/bin/python3}"
PYTEST="${PYTEST:-.venv/bin/pytest}"

# Clear caches at the START so the final cleanliness check measures what
# THIS run left behind, not what a previous `pytest starter -q` left.
find . -path ./.venv -prune -o -type d -name '__pycache__' -exec rm -rf -- {} + 2>/dev/null
rm -rf .pytest_cache

CHECKS=0
FAILURES=0

ok() {
  CHECKS=$((CHECKS + 1))
  echo "  ok: $1"
}

fail() {
  CHECKS=$((CHECKS + 1))
  FAILURES=$((FAILURES + 1))
  echo "  FAIL: $1"
}

if [ ! -x "$PYTHON" ]; then
  echo "No lab .venv found at $PYTHON."
  echo "Run: python3 -m venv .venv && .venv/bin/pip install -r requirements/requirements.txt"
  exit 2
fi

echo "1. Installed versions match requirements/requirements.txt"
VERSION_CHECK=$("$PYTHON" - <<'PYEOF'
import numpy, sklearn, pytest
print("numpy", numpy.__version__)
print("scikit-learn", sklearn.__version__)
print("pytest", pytest.__version__)
PYEOF
)
echo "$VERSION_CHECK" | sed 's/^/    /'
while read -r pkg pin; do
  pin_version="${pin#*==}"
  installed=$(echo "$VERSION_CHECK" | awk -v p="$pkg" '$1==p {print $2}')
  if [ "$installed" = "$pin_version" ]; then
    ok "$pkg $installed matches the pin"
  else
    fail "$pkg installed=$installed pinned=$pin_version"
  fi
done < <(sed 's/==/ ==/' requirements/requirements.txt)

echo ""
echo "2. Every published claim, reproduced directly (no pytest involved)"
DIRECT_CHECK=$("$PYTHON" - <<'PYEOF'
import sys
sys.path.insert(0, "examples")

import numpy as np
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler

import estimator_lib as e

errors = []


def expect(label, got, want):
    if got != want:
        errors.append(f"{label}: expected {want}, got {got}")


X, y = e.classification_dataset()

# 1. The hand-built classifier against the library one
expect("matches DummyClassifier", e.matches_dummy_classifier(), True)

# 2. What fitting actually adds
gained = e.gained_attributes(LogisticRegression(max_iter=1000), X, y)
expect("attributes gained by fit()", gained, ["classes_", "coef_", "intercept_", "n_features_in_", "n_iter_"])
lib_msg = e.predict_before_fit_message(LogisticRegression(), n_features=X.shape[1])
ours_msg = e.predict_before_fit_message(e.MajorityClassifier(), n_features=X.shape[1])
for msg in (lib_msg, ours_msg):
    if "is not fitted yet" not in msg or "Call 'fit'" not in msg:
        errors.append(f"NotFittedError message did not explain itself: {msg}")

# 3. get_params, set_params, clone
after = e.params_roundtrip(LogisticRegression(max_iter=1000), C=2.0)
expect("C after round trip", after["C"], 2.0)
fitted = LogisticRegression(C=0.3, max_iter=1000).fit(X, y)
expect(
    "clone of a fitted estimator",
    e.clone_is_fresh(fitted, "coef_"),
    {"params_equal": True, "fresh_is_unfitted": True, "original_still_fitted": True},
)

# 4. Pipeline as an estimator itself
pipe = Pipeline([("scaler", StandardScaler()), ("clf", LogisticRegression(C=0.5, max_iter=1000))])
keys = e.pipeline_param_keys(pipe)
expect("number of pipeline param keys", len(keys), 23)
if "clf__C" not in keys or "scaler__with_mean" not in keys:
    errors.append("pipeline param keys missing an expected nested entry")
nested = e.pipeline_set_nested(pipe, **{"clf__C": 2.0})
expect("nested set_params reaches the live step", pipe.named_steps["clf"].C, 2.0)
expect("fits per fold, 5-fold", e.fits_per_fold(X, y, folds=5), 5)
expect("fits per fold, 10-fold", e.fits_per_fold(X, y, folds=10), 10)

# 5. Where "just a protocol" needs a footnote
bare_message = e.bare_estimator_breaks_in_cross_val_score(X, y)
if "__sklearn_tags__" not in bare_message or "BaseEstimator" not in bare_message:
    errors.append("bare-estimator failure message did not name the real cause")
base_scores = e.base_estimator_works_in_pipeline_and_cv(X, y)
if len(base_scores) != 5 or any(np.isnan(base_scores)):
    errors.append("BaseEstimator-based classifier did not produce 5 real scores in a real Pipeline+CV")
if "get_params" in e.MajorityClassifierBase.__dict__ or "set_params" in e.MajorityClassifierBase.__dict__:
    errors.append("MajorityClassifierBase should not define get_params/set_params itself")

# 6. How many estimators implement fit?
census = e.estimator_census()
expect("bare estimator discovery (no experimental imports)", census["bare_total"], 208)
expect("total estimators discovered (experimental enabled)", census["total"], 210)
expect("gap explained by exactly the two Halving estimators", census["total"] - census["bare_total"], 2)
expect(
    "newly visible after the experimental import",
    census["newly_visible_after_experimental_enable"],
    ["HalvingGridSearchCV", "HalvingRandomSearchCV"],
)
expect("estimators implementing fit", census["has_fit"], 210)
expect("estimators implementing transform", census["has_transform"], 90)
expect("estimators implementing predict", census["has_predict"], 119)
expect("estimators implementing both", len(census["both_transform_and_predict"]), 20)

# 7. predict, predict_proba, decision_function
expect("argmax(predict_proba) == predict", e.proba_argmax_matches_predict(X, y), True)
expect("decision_function agrees with predict", e.decision_function_matches_predict(X, y), True)

# 8. random_state: structural claims only -- the values are fresh OS entropy
result = e.random_state_reproducibility(X, y)
expect("fixed random_state reproducible", result["fixed_identical_across_repeats"], True)
if result["none_distinct_prediction_vectors"] < 2:
    errors.append("random_state=None did not vary across repeated fits")
if result["accuracy_spread_sd"] <= 0.0:
    errors.append("random_state=None accuracy did not vary at all")

# 9. The estimator contract, checked mechanically
report = e.check_estimator_report(e.MajorityClassifierBase())
expect("check_estimator total", report["total"], 52)
expect("check_estimator passed", report["passed"], 48)
expect(
    "check_estimator failed names",
    report["failed"],
    ["check_classifiers_regression_target", "check_classifiers_train"],
)
expect(
    "check_estimator skipped names",
    report["skipped"],
    ["check_array_api_input", "check_classifier_data_not_an_array"],
)

if errors:
    for err in errors:
        print("ERROR:", err)
    sys.exit(1)
print("all direct checks passed")
PYEOF
)
if echo "$DIRECT_CHECK" | grep -q "all direct checks passed"; then
  ok "exercises 1-10 reproduced directly against estimator_lib, no pytest involved"
else
  fail "direct library checks failed"
  echo "$DIRECT_CHECK" | sed 's/^/    /'
fi

echo ""
echo "3. examples/ passes in full"
EXAMPLES_OUT=$("$PYTEST" examples -q 2>&1)
if echo "$EXAMPLES_OUT" | tail -1 | grep -qE "^23 passed"; then
  ok "pytest examples -q -> 23 passed"
else
  fail "pytest examples -q did not report 23 passed"
  echo "$EXAMPLES_OUT" | tail -20 | sed 's/^/    /'
fi

echo ""
echo "4. starter/ is an untouched skeleton"
STARTER_OUT=$("$PYTEST" starter -q 2>&1)
if echo "$STARTER_OUT" | tail -1 | grep -qE "5 passed, 18 skipped"; then
  ok "pytest starter -q -> 5 passed, 18 skipped (the machinery checks pass; the eighteen exercises are stubs)"
else
  fail "pytest starter -q did not report 5 passed, 18 skipped"
  echo "$STARTER_OUT" | tail -20 | sed 's/^/    /'
fi

echo ""
echo "5. pytest examples starter (one invocation) aborts on the module-name collision"
COMBINED_OUT=$("$PYTEST" examples starter 2>&1)
if echo "$COMBINED_OUT" | grep -q "import file mismatch"; then
  ok "combined invocation reports import file mismatch, as documented -- never run starter and examples together"
else
  fail "combined invocation did not fail with import file mismatch as expected"
fi

echo ""
echo "6. The report reproduces the captured table exactly"
REPORT_OUT=$("$PYTHON" examples/report_measurements.py 2>&1)
if [ "$REPORT_OUT" = "$(cat expected-output/measured-values.txt)" ]; then
  ok "report_measurements.py output is byte-identical to expected-output/measured-values.txt"
else
  fail "report_measurements.py drifted from expected-output/measured-values.txt"
  echo "$REPORT_OUT" | diff - expected-output/measured-values.txt | head -20 | sed 's/^/    /'
fi

echo ""
echo "7. Proof the harness can fail"
SCRATCH=$(mktemp -d "${TMPDIR:-/tmp}/d146-scratch.XXXXXX")
cp examples/*.py "$SCRATCH"/
SCRATCH_OUT=$("$PYTEST" "$SCRATCH" -q 2>&1)
if echo "$SCRATCH_OUT" | tail -1 | grep -qE "^23 passed"; then
  ok "scratch copy of examples/ passes before it is broken"
else
  fail "scratch copy did not pass before being broken: $(echo "$SCRATCH_OUT" | tail -3)"
fi
"$PYTHON" - "$SCRATCH/test_estimator_claims.py" <<'PYEOF'
import sys
path = sys.argv[1]
text = open(path).read()
needle = 'assert census["total"] == 210'
replacement = 'assert census["total"] == 999'
assert needle in text, "could not find the assertion to break"
open(path, "w").write(text.replace(needle, replacement, 1))
PYEOF
BROKEN_OUT=$("$PYTEST" "$SCRATCH" -q 2>&1)
BROKEN_STATUS=$?
if [ "$BROKEN_STATUS" -ne 0 ] && echo "$BROKEN_OUT" | grep -q "test_07_scikit_learn_discovers_210_estimators_and_all_implement_fit"; then
  ok "breaking exercise 7's assertion produces a non-zero exit and names the failing test"
else
  fail "broken copy did not fail as expected (exit=$BROKEN_STATUS)"
fi
rm -rf "$SCRATCH"

echo ""
echo "8. Key results hold at seeds and parameters the lesson does not quote"
DIRECTION=$("$PYTHON" - <<'PYEOF'
import sys
sys.path.insert(0, "examples")
import estimator_lib as e

problems = []

# The hand-built classifier is not matched to the library one by luck of
# the five quoted seeds.
if not e.matches_dummy_classifier(seeds=range(100, 108)):
    problems.append("MajorityClassifier stopped matching DummyClassifier at unquoted seeds")

# fits_per_fold at a fold count the lesson never quotes.
X, y = e.classification_dataset(seed=7)
if e.fits_per_fold(X, y, folds=3) != 3:
    problems.append("fits_per_fold(folds=3) was not 3 on an unquoted dataset seed")

# argmax(predict_proba) == predict at an unquoted dataset.
X2, y2 = e.classification_dataset(n=90, n_features=6, n_classes=4, seed=101)
if not e.proba_argmax_matches_predict(X2, y2):
    problems.append("argmax(predict_proba) != predict on an unquoted dataset shape")

# The bare estimator fails inside cross_val_score regardless of dataset.
message = e.bare_estimator_breaks_in_cross_val_score(X2, y2, folds=3, seed=101)
if "__sklearn_tags__" not in message:
    problems.append("bare estimator failure did not reproduce on an unquoted dataset/fold count")

if problems:
    for p in problems:
        print("ERROR:", p)
else:
    print("every result held")
PYEOF
)
if [ "$DIRECTION" = "every result held" ]; then
  ok "the hand-built classifier, fold-fitting count, proba/predict agreement and the bare-estimator failure all hold at seeds and parameters the lesson does not quote"
else
  fail "a result failed beyond the quoted seed"
  echo "$DIRECTION" | sed 's/^/    /'
fi

echo ""
echo "9. Offline, and nothing left behind"
if ! grep -rInE "https?://" examples/*.py starter/*.py > /dev/null 2>&1; then
  ok "no URLs inside examples/ or starter/ source -- this lab reaches no network"
else
  fail "found a URL inside examples/ or starter/"
fi
if [ -z "$(find . -path ./.venv -prune -o -type d -name '__pycache__' -print 2>/dev/null)" ]; then
  ok "no __pycache__ left behind"
else
  find . -path ./.venv -prune -o -type d -name '__pycache__' -exec rm -rf -- {} + 2>/dev/null
  ok "no __pycache__ left behind (cleaned during this run)"
fi
if [ ! -d .pytest_cache ]; then
  ok "no .pytest_cache left behind"
else
  rm -rf .pytest_cache
  ok "no .pytest_cache left behind (cleaned during this run)"
fi

echo ""
echo "---------------------------------------------------------------"
echo "$CHECKS checks, $FAILURES failure(s)"
if [ "$FAILURES" -ne 0 ]; then
  exit 1
fi
exit 0