Hands-on lab — Day 125: Cleaning Messy Data

"""Shared fixtures. pytest finds this file by itself -- nothing imports it.

Each fixture returns a FRESH copy of its table, so one test's mutation can
never leak into the next test.
"""

import pytest

from data import (
    build_clean_customers,
    build_coerce_frame,
    build_contract_violating_customers,
    build_country_frame,
    build_dropna_frame,
    build_duplicates_frame,
    build_income_spending,
    build_sensor_timeseries,
    build_temperature_readings,
    shuffle_rows,
)


@pytest.fixture
def income_spending():
    return build_income_spending()


@pytest.fixture
def temperature_readings():
    return build_temperature_readings()


@pytest.fixture
def dropna_frame():
    return build_dropna_frame()


@pytest.fixture
def sensor_timeseries():
    return build_sensor_timeseries()


@pytest.fixture
def shuffled_sensor_timeseries():
    return shuffle_rows(build_sensor_timeseries())


@pytest.fixture
def coerce_frame():
    return build_coerce_frame()


@pytest.fixture
def country_frame():
    return build_country_frame()


@pytest.fixture
def duplicates_frame():
    return build_duplicates_frame()


@pytest.fixture
def clean_customers():
    return build_clean_customers()


@pytest.fixture
def contract_violating_customers():
    return build_contract_violating_customers()

"""A tiny cleaning contract: name the post-conditions cleaning is supposed
to guarantee, and check them mechanically instead of trusting that the
cleaning steps above did what they were meant to do.

This is deliberately small -- three checks, one function -- because the
point of exercise 9 is the SHAPE of the idea (assert what you intend, fail
loudly when it does not hold), not a general-purpose validation framework.
Day 126 builds a real reusable pipeline around this idea; this is the
one-function version that proves the shape works.
"""

from __future__ import annotations

import pandas as pd


class ContractViolation(AssertionError):
    """Raised when cleaned data fails one of the contract's post-conditions."""


def assert_cleaning_contract(
    df: pd.DataFrame,
    *,
    key_columns: list[str],
    dtypes: dict[str, str],
    min_rows: int,
    max_rows: int,
) -> None:
    """Check three post-conditions a cleaning step is supposed to guarantee.

    Raises `ContractViolation` naming the FIRST check that fails, so a
    caller sees exactly what broke rather than a generic assertion error.
    """
    for column in key_columns:
        n_null = df[column].isna().sum()
        if n_null > 0:
            raise ContractViolation(
                f"key column {column!r} has {n_null} null value(s); "
                "key columns must be fully populated after cleaning"
            )

    for column, expected_dtype in dtypes.items():
        actual_dtype = str(df[column].dtype)
        if actual_dtype != expected_dtype:
            raise ContractViolation(
                f"column {column!r} has dtype {actual_dtype!r}, expected {expected_dtype!r}"
            )

    n_rows = len(df)
    if not (min_rows <= n_rows <= max_rows):
        raise ContractViolation(
            f"row count {n_rows} is outside the expected range [{min_rows}, {max_rows}]"
        )

"""The tables every exercise in this lab is built from.

Nothing here downloads anything. Two tables use a seeded NumPy generator so
a re-run always produces the same rows (`build_income_spending`,
`build_large_income_spending`); everything else is a small hand-written
literal table, chosen so every exercise's expected values can be checked
exactly rather than approximately.

`build_income_spending` carries the day's opening failure on purpose: some
rows have no recorded income at all, and `spending` is genuinely, linearly
related to `income` plus noise -- so a real correlation exists to distort.
"""

from __future__ import annotations

import numpy as np
import pandas as pd

# --------------------------------------------------------------------------
# Exercises 1 and 2 -- mean imputation and fillna(0), and the correlation
# claim in exercise 1. Seeded so the numbers are reproducible exactly.
# --------------------------------------------------------------------------


def build_income_spending(seed: int = 20250825, n: int = 40, n_missing: int = 10) -> pd.DataFrame:
    """`income` and `spending`, genuinely correlated, with `income` missing
    at `n_missing` rows chosen completely at random (MCAR)."""
    rng = np.random.default_rng(seed)
    income = rng.normal(52_000, 11_000, n)
    spending = 0.42 * income + rng.normal(0, 3_500, n)
    df = pd.DataFrame(
        {
            "customer_id": np.arange(1001, 1001 + n),
            "income": income,
            "spending": spending,
        }
    )
    missing_idx = rng.choice(n, size=n_missing, replace=False)
    df.loc[missing_idx, "income"] = np.nan
    return df


def build_temperature_readings() -> pd.DataFrame:
    """A sensor log where a missing reading and a genuine 0.0C reading must
    stay distinguishable -- exercise 2's fillna(0) trap."""
    return pd.DataFrame(
        {
            "station": ["A", "A", "A", "B", "B", "B", "C", "C", "C", "C"],
            "reading_c": [18.2, np.nan, 17.5, 22.0, 21.4, np.nan, -3.0, np.nan, -1.5, 0.0],
        }
    )


# --------------------------------------------------------------------------
# Exercise 3 -- dropna with how, thresh, subset. Eight rows, three columns,
# a deliberately mixed missingness pattern.
# --------------------------------------------------------------------------


def build_dropna_frame() -> pd.DataFrame:
    return pd.DataFrame(
        {
            "customer_id": [1, 2, 3, 4, 5, 6, 7, 8],
            "email": ["a@x.com", None, "c@x.com", None, "e@x.com", None, "g@x.com", None],
            "phone": ["555-1", "555-2", None, None, "555-5", None, None, None],
            "signup_date": [
                "2026-01-02",
                "2026-01-03",
                "2026-01-04",
                None,
                "2026-01-06",
                None,
                "2026-01-08",
                None,
            ],
        }
    )


# --------------------------------------------------------------------------
# Exercise 4 -- ffill on unsorted data. A daily reading with two gaps,
# written here in TRUE chronological order; the exercise shuffles it.
# --------------------------------------------------------------------------


def build_sensor_timeseries() -> pd.DataFrame:
    return pd.DataFrame(
        {
            "day": [1, 2, 3, 4, 5, 6, 7, 8],
            "reading": [10.0, np.nan, np.nan, 13.0, 14.0, np.nan, 16.0, 17.0],
        }
    )


def shuffle_rows(df: pd.DataFrame, seed: int = 7) -> pd.DataFrame:
    rng = np.random.default_rng(seed)
    order = rng.permutation(len(df))
    return df.iloc[order].reset_index(drop=True)


# --------------------------------------------------------------------------
# Exercise 5 -- the missing indicator. Reuses build_temperature_readings.
# --------------------------------------------------------------------------

# (no separate builder needed -- exercise 5 works directly on
# build_temperature_readings())


# --------------------------------------------------------------------------
# Exercise 6 -- to_numeric(errors="coerce"). A column that is mostly clean
# numbers with a KNOWN number of unparseable strings planted in it.
# --------------------------------------------------------------------------


def build_coerce_frame() -> pd.DataFrame:
    return pd.DataFrame(
        {
            "order_id": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
            "quantity_raw": ["12", "7", "N/A", "3", "unknown", "9", "5", "--", "20", "4"],
        }
    )


# --------------------------------------------------------------------------
# Exercise 7 -- string normalisation. One true country, four spellings.
# --------------------------------------------------------------------------


def build_country_frame() -> pd.DataFrame:
    return pd.DataFrame(
        {
            "customer_id": list(range(1, 13)),
            "country_raw": [
                "USA",
                "U.S.A.",
                " usa ",
                "Usa",
                "USA",
                "U.S.A.",
                "Canada",
                "canada ",
                " Canada",
                "USA",
                "CANADA",
                "U.S.A.",
            ],
            "amount": [
                120.0,
                85.0,
                60.0,
                200.0,
                75.0,
                150.0,
                40.0,
                90.0,
                30.0,
                110.0,
                55.0,
                65.0,
            ],
        }
    )


# --------------------------------------------------------------------------
# Exercise 8 -- duplicates. Row 5 is an exact duplicate of row 1. Row 6
# shares (customer_id, item) with row 2 but has a different price -- a
# subset duplicate that is not an exact duplicate.
# --------------------------------------------------------------------------


def build_duplicates_frame() -> pd.DataFrame:
    return pd.DataFrame(
        {
            "customer_id": [1, 2, 3, 4, 1, 2, 5],
            "item": ["pen", "mug", "pen", "bag", "pen", "mug", "hat"],
            "price": [1.5, 8.0, 1.5, 20.0, 1.5, 8.0, 12.0],
            "order_ts": [
                "2026-01-01T09:00",
                "2026-01-01T09:05",
                "2026-01-01T09:10",
                "2026-01-01T09:15",
                "2026-01-01T09:00",
                "2026-01-02T14:30",
                "2026-01-01T09:20",
            ],
        }
    )


# --------------------------------------------------------------------------
# Exercise 9 -- the cleaning contract. A frame that passes, and one that
# is built to fail each post-condition in turn.
# --------------------------------------------------------------------------


def build_clean_customers() -> pd.DataFrame:
    return pd.DataFrame(
        {
            "customer_id": [1, 2, 3, 4, 5],
            "country": ["USA", "Canada", "USA", "Canada", "USA"],
            "income": [52_000.0, 61_000.0, 48_500.0, 73_200.0, 55_000.0],
        }
    )


def build_contract_violating_customers() -> pd.DataFrame:
    """Violates the contract three ways: a null in a key column, the wrong
    dtype on `income` (object, because one entry is a stray string), and a
    row count far outside the expected range."""
    return pd.DataFrame(
        {
            "customer_id": [1, 2, None],
            "country": ["USA", "Canada", "USA"],
            "income": [52_000.0, "unknown", 48_500.0],
        }
    )

"""The worked reference suite for Day 125 -- "Cleaning With Receipts".

Nine exercises, each proving one real pandas 3.0.5 cleaning behaviour by
running code and reading real values -- never by reading source. Run it:

    pytest examples

Every table these tests use comes from `data.py`, imported through the
fixtures in `conftest.py`. Read `starter/00_brief.md` for the exercise-by-
exercise explanation; this file is the answer key.
"""

import numpy as np
import pandas as pd
import pytest

from contract import ContractViolation, assert_cleaning_contract

# --------------------------------------------------------------------------
# Exercise 1 -- mean imputation distorts. The mean survives untouched; the
# standard deviation strictly shrinks; and -- contrary to the intuitive
# guess that a "pile of average points" inflates a correlation -- the
# correlation with a genuinely related column strictly ATTENUATES (moves
# toward zero), never grows. See this lab's README and the lesson's "Why
# this matters" section for why that direction is mathematically forced,
# not a coincidence of this one dataset.
# --------------------------------------------------------------------------


def test_1_mean_survives_imputation_unchanged(income_spending):
    before_mean = income_spending["income"].mean()
    imputed = income_spending["income"].fillna(before_mean)
    after_mean = imputed.mean()

    assert before_mean == pytest.approx(52_666.679, abs=0.01)
    assert after_mean == pytest.approx(before_mean, abs=1e-6)


def test_1_std_strictly_shrinks(income_spending):
    before_std = income_spending["income"].std()
    imputed = income_spending["income"].fillna(income_spending["income"].mean())
    after_std = imputed.std()

    assert before_std == pytest.approx(10_663.947, abs=0.01)
    assert after_std == pytest.approx(9_195.697, abs=0.01)
    assert after_std < before_std, "imputation must strictly shrink the standard deviation"


def test_1_correlation_strictly_attenuates_not_inflates(income_spending):
    before_corr = income_spending[["income", "spending"]].corr().iloc[0, 1]

    after = income_spending.copy()
    after["income"] = after["income"].fillna(after["income"].mean())
    after_corr = after[["income", "spending"]].corr().iloc[0, 1]

    assert before_corr == pytest.approx(0.745156, abs=1e-4)
    assert after_corr == pytest.approx(0.606148, abs=1e-4)
    assert abs(after_corr) < abs(before_corr), (
        "mean imputation must strictly shrink the correlation's magnitude, not grow it -- "
        "an imputed point sits exactly at the column mean, so it contributes zero to the "
        "covariance and zero to its own column's variance term, and can only dilute an "
        "existing relationship, never strengthen one"
    )


def test_1_n_missing_matches_the_planted_count(income_spending):
    assert income_spending["income"].isna().sum() == 10


# --------------------------------------------------------------------------
# Exercise 2 -- fillna(0) on a measurement column. Zero is a value, not an
# absence: a station's real 0.0C reading and a MISSING reading must not be
# collapsed into the same number.
# --------------------------------------------------------------------------


def test_2_fillna_zero_moves_the_mean_by_the_exact_amount(temperature_readings):
    before_mean = temperature_readings["reading_c"].mean()
    after_mean = temperature_readings["reading_c"].fillna(0.0).mean()

    assert before_mean == pytest.approx(10.657143, abs=1e-4)
    assert after_mean == pytest.approx(7.46, abs=1e-4)

    shift = after_mean - before_mean
    assert shift == pytest.approx(-3.197143, abs=1e-4)
    assert shift != 0.0, "filling with 0 must move the mean -- 0 is a real value, not a no-op"


def test_2_fillna_zero_confuses_missing_with_a_real_zero_reading(temperature_readings):
    # Station C's row 9 (index 9) is a GENUINE 0.0C reading, already present
    # before any fill. After fillna(0), it is indistinguishable from the
    # three rows that were actually missing.
    genuine_zero_before = temperature_readings.loc[9, "reading_c"]
    assert genuine_zero_before == 0.0
    assert not pd.isna(genuine_zero_before)

    filled = temperature_readings["reading_c"].fillna(0.0)
    was_missing = temperature_readings["reading_c"].isna()
    now_all_read_as_zero_or_real = (filled[was_missing] == 0.0).all()
    assert now_all_read_as_zero_or_real
    # The three imputed rows and the one genuine zero are now bit-for-bit
    # identical in the data -- nothing downstream can tell them apart.
    assert (filled == 0.0).sum() == was_missing.sum() + 1


# --------------------------------------------------------------------------
# Exercise 3 -- dropna with how, thresh and subset give four different row
# counts on the same eight-row frame.
# --------------------------------------------------------------------------


def test_3_dropna_row_counts(dropna_frame):
    assert dropna_frame.shape == (8, 4)

    how_any = dropna_frame.dropna(how="any")
    how_all = dropna_frame.dropna(how="all")
    thresh_2 = dropna_frame.dropna(thresh=2)
    subset_email = dropna_frame.dropna(subset=["email"])

    assert how_any.shape[0] == 2, "how='any' drops any row missing even one field -- the strictest cut"
    assert how_all.shape[0] == 8, "how='all' only drops a row missing on EVERY field -- none here are"
    assert thresh_2.shape[0] == 5, "thresh=2 keeps rows with at least 2 non-null fields"
    assert subset_email.shape[0] == 4, "subset=['email'] only checks the named column"


# --------------------------------------------------------------------------
# Exercise 4 -- ffill on unsorted data is a real bug. The wrong answer
# appears on the unsorted frame; sorting first fixes it.
# --------------------------------------------------------------------------


def test_4_ffill_on_unsorted_data_is_wrong(sensor_timeseries):
    from data import shuffle_rows

    shuffled = shuffle_rows(sensor_timeseries, seed=7)
    wrong = shuffled.copy()
    wrong["reading"] = wrong["reading"].ffill()

    # Re-sort by day only to COMPARE against the correct answer -- the
    # ffill computation itself already happened on the unsorted frame.
    wrong_by_day = wrong.sort_values("day").reset_index(drop=True)

    correct = sensor_timeseries.sort_values("day").reset_index(drop=True)
    correct["reading"] = correct["reading"].ffill()

    assert wrong_by_day.loc[wrong_by_day["day"] == 2, "reading"].item() == 14.0
    assert correct.loc[correct["day"] == 2, "reading"].item() == 10.0
    assert wrong_by_day.loc[wrong_by_day["day"] == 3, "reading"].item() == 17.0
    assert correct.loc[correct["day"] == 3, "reading"].item() == 10.0

    differing_days = wrong_by_day.loc[wrong_by_day["reading"] != correct["reading"], "day"].tolist()
    assert differing_days == [2, 3]


def test_4_ffill_after_sorting_is_correct(sensor_timeseries):
    from data import shuffle_rows

    shuffled = shuffle_rows(sensor_timeseries, seed=7)
    fixed = shuffled.sort_values("day").reset_index(drop=True)
    fixed["reading"] = fixed["reading"].ffill()

    expected = [10.0, 10.0, 10.0, 13.0, 14.0, 14.0, 16.0, 17.0]
    assert fixed["reading"].tolist() == expected


# --------------------------------------------------------------------------
# Exercise 5 -- the missing indicator. Record WHICH values were imputed
# before the imputation erases the evidence, and confirm the flag column
# is an exact record of the original isna() mask.
# --------------------------------------------------------------------------


def test_5_missing_indicator_exactly_matches_original_mask(temperature_readings):
    reading_was_missing = temperature_readings["reading_c"].isna()

    cleaned = temperature_readings.copy()
    cleaned["reading_c_was_missing"] = reading_was_missing
    cleaned["reading_c"] = cleaned["reading_c"].fillna(cleaned["reading_c"].mean())

    # After imputation, isna() alone can no longer tell you anything --
    # every value looks "present" now.
    assert cleaned["reading_c"].isna().sum() == 0

    # But the indicator column still says exactly what isna() said before
    # the fill erased the evidence.
    assert cleaned["reading_c_was_missing"].sum() == 3
    assert (cleaned["reading_c_was_missing"] == reading_was_missing).all()


# --------------------------------------------------------------------------
# Exercise 6 -- to_numeric(errors="coerce") converts every unparseable
# string into a missing value, silently. Count them; never coerce blind.
# --------------------------------------------------------------------------


def test_6_coerce_count_matches_the_planted_garbage(coerce_frame):
    coerced = pd.to_numeric(coerce_frame["quantity_raw"], errors="coerce")

    n_coerced = coerced.isna().sum()
    planted_garbage = coerce_frame["quantity_raw"].isin(["N/A", "unknown", "--"]).sum()

    assert n_coerced == 3
    assert planted_garbage == 3
    assert n_coerced == planted_garbage, "every coerced NaN must trace back to a planted garbage string"

    # The clean values must survive the coercion exactly, as floats.
    clean_values = coerced.dropna().tolist()
    assert clean_values == [12.0, 7.0, 3.0, 9.0, 5.0, 20.0, 4.0]


# --------------------------------------------------------------------------
# Exercise 7 -- string normalisation. One country, four raw spellings; a
# raw groupby silently produces more groups than the truth.
# --------------------------------------------------------------------------


def test_7_nunique_before_and_after_normalisation(country_frame):
    before = country_frame["country_raw"].nunique()
    assert before == 8

    normalised = (
        country_frame["country_raw"]
        .str.strip()
        .str.lower()
        .str.replace(".", "", regex=False)
        .replace({"usa": "USA", "canada": "Canada"})
    )
    after = normalised.nunique()
    assert after == 2
    assert set(normalised.unique()) == {"USA", "Canada"}


def test_7_raw_groupby_splits_one_true_country_into_several(country_frame):
    raw_groups = country_frame.groupby("country_raw")["amount"].sum()
    assert len(raw_groups) == 8  # the true count is 2

    normalised = (
        country_frame["country_raw"]
        .str.strip()
        .str.lower()
        .str.replace(".", "", regex=False)
        .replace({"usa": "USA", "canada": "Canada"})
    )
    true_groups = country_frame.assign(country=normalised).groupby("country")["amount"].sum()
    assert len(true_groups) == 2
    assert true_groups.loc["USA"] == pytest.approx(865.0)
    assert true_groups.loc["Canada"] == pytest.approx(215.0)


# --------------------------------------------------------------------------
# Exercise 8 -- duplicates. "Duplicate" means whatever subset you named.
# --------------------------------------------------------------------------


def test_8_exact_and_subset_duplicate_counts_differ(duplicates_frame):
    exact_dupes = duplicates_frame.duplicated().sum()
    subset_dupes = duplicates_frame.duplicated(subset=["customer_id", "item"]).sum()

    assert exact_dupes == 1, "row 4 exactly repeats row 0 (customer, item, price AND timestamp)"
    assert subset_dupes == 2, (
        "row 4 repeats row 0's (customer_id, item), and row 5 repeats row 1's "
        "(customer_id, item) even though row 5's timestamp differs"
    )
    assert subset_dupes > exact_dupes, (
        "'duplicate on a subset' is a strictly looser question than 'duplicate on every "
        "column', so it can only find as many or more rows"
    )


def test_8_which_definition_is_right_depends_on_the_question(duplicates_frame):
    # Question: "did this exact order get logged twice?" -- exact duplicates
    # is right, because a genuine re-order at the same second with the same
    # price is (at these row counts) indistinguishable from a duplicate log
    # entry.
    exact = duplicates_frame[duplicates_frame.duplicated()]
    assert exact["customer_id"].tolist() == [1]

    # Question: "did this customer buy this item more than once?" -- subset
    # duplicates on (customer_id, item) is right, because customer 2's two
    # mug purchases on different days are two real events, not one row
    # logged twice, and only the subset definition catches both repeats.
    subset = duplicates_frame[duplicates_frame.duplicated(subset=["customer_id", "item"])]
    assert sorted(subset["customer_id"].tolist()) == [1, 2]


# --------------------------------------------------------------------------
# Exercise 9 -- the cleaning contract. Post-conditions hold on cleaned
# data, and the contract genuinely RAISES on data that violates it.
# --------------------------------------------------------------------------


def test_9_contract_passes_on_clean_data(clean_customers):
    # Must not raise.
    assert_cleaning_contract(
        clean_customers,
        key_columns=["customer_id", "country"],
        dtypes={"income": "float64"},
        min_rows=3,
        max_rows=10,
    )


def test_9_contract_raises_on_a_null_key_column(contract_violating_customers):
    with pytest.raises(ContractViolation, match="customer_id"):
        assert_cleaning_contract(
            contract_violating_customers,
            key_columns=["customer_id", "country"],
            dtypes={"income": "float64"},
            min_rows=3,
            max_rows=10,
        )


def test_9_contract_raises_on_a_wrong_dtype():
    from data import build_clean_customers

    wrong_dtype = build_clean_customers()
    wrong_dtype["income"] = wrong_dtype["income"].astype(str)

    with pytest.raises(ContractViolation, match="income"):
        assert_cleaning_contract(
            wrong_dtype,
            key_columns=["customer_id", "country"],
            dtypes={"income": "float64"},
            min_rows=3,
            max_rows=10,
        )


def test_9_contract_raises_on_a_row_count_outside_the_range():
    from data import build_clean_customers

    too_few_rows = build_clean_customers().iloc[:1]

    with pytest.raises(ContractViolation, match="row count"):
        assert_cleaning_contract(
            too_few_rows,
            key_columns=["customer_id", "country"],
            dtypes={"income": "float64"},
            min_rows=3,
            max_rows=10,
        )

lesson_id: D125
day: 125
kind: guided-build
languages: [python, bash]
setup_commands:
  - cd labs/sections/math-statistics-and-data/day-125-cleaning-messy-data
  - python3 -m venv .venv
  - .venv/bin/pip install -r requirements/requirements.txt
  - .venv/bin/python3 -c "import pandas; print(pandas.__version__)"
run_commands:
  - .venv/bin/pytest examples
  - .venv/bin/pytest starter
test_commands:
  - bash tests/run_tests.sh
cleanup_commands:
  - "find . -path ./.venv -prune -o -type d -name '__pycache__' -print -exec rm -rf -- {} +"
  - rm -rf .pytest_cache
  - 'rm -rf .venv  # optional: removes the lab virtual environment'
  - 'git checkout -- starter/  # optional: reset your work'
requires_network: true
requires_api_key: false
estimated_minutes: 40
last_executed: '2026-08-19'
executed_on: 'macOS 26.5.2 (Apple Silicon, arm64), Python 3.14.0, pandas 3.0.5, pyarrow 25.0.1, numpy 2.5.2, pytest 9.1.1, bash 3.2.57 -- bash tests/run_tests.sh -> 13 checks, 0 failure(s), exit 0. pytest examples -> 19 passed. pytest starter -> 19 skipped (untouched checkout). Section 5 of the harness solves every exercise in a scratch copy (19 passed), deliberately breaks the exercise-3 thresh=2 assertion (5 -> 999), confirms the run exits non-zero with a printed FAIL, restores the file, and confirms 19 passed again -- so the suite is demonstrated to be capable of failing rather than merely claimed to be. Separately, the coordinator ran `pytest examples starter` together (not through the harness) and confirmed it errors out with "import file mismatch" on this pytest version, rather than silently shadowing -- both directories define a module named test_cleaning.py, and troubleshooting.md documents the danger. Everything was run through a real lab-local .venv created by the documented setup commands. Two honesty notes from this run. FIRST, and the most important call in this lab: the day brief specified that mean-imputing a column would show its correlation with a genuinely related second column strictly INCREASE ("inflated"). Direct measurement on this lab''s income_spending table shows the opposite: correlation strictly ATTENUATES, from 0.745156 to 0.606148, every time. This is not a one-machine anomaly -- it is a provable algebraic consequence of the Pearson correlation formula (an imputed value sits exactly at the column mean, so its deviation from the mean is exactly zero, and a zero-deviation term contributes exactly zero to the covariance sum, so it can only dilute an existing relationship, never strengthen one), and it was independently confirmed empirically across 200 reseeded trials, including a second scheme that imputes two columns independently at disjoint missing rows, none of which produced an increase. The lesson and lab both assert and teach the measured direction (attenuation), not the brief''s original guess, and say so explicitly. SECOND: matplotlib, scipy and scikit-learn are not installed in this environment; the lesson''s Tools section describes scikit-learn''s SimpleImputer, pyjanitor, and Great Expectations / pandera from their public documentation only, and says so plainly -- no output attributed to any of them is reproduced anywhere in this lab.'

# What is installed, why, and what it costs

Four packages, all free and open source, installed into a lab-local
virtual environment that `rm -rf .venv` completely undoes.

| Package | Version pinned | Licence | What this lab uses it for |
| --- | --- | --- | --- |
| `pandas` | 3.0.5 | BSD 3-Clause | Every `fillna`, `dropna`, `ffill`, `to_numeric`, `.str`, `duplicated` and `groupby` call in this lab. |
| `pyarrow` | 25.0.1 | Apache 2.0 | pandas 3.0's default backend for the `str` dtype; installed for parity with Days 120-124 even though this lab's own assertions do not inspect it directly. |
| `numpy` | 2.5.2 | BSD 3-Clause | `np.nan`, and the seeded `np.random.default_rng` that builds `income_spending` and shuffles the sensor time series. |
| `pytest` | 9.1.1 | MIT | The test harness every exercise is written against. |

There is no paid tier of anything in this lab, no account, no key and no
signup, personally or commercially.

## The one time the network is needed

```bash
.venv/bin/pip install -r requirements/requirements.txt
```

That is the only command in the lab that opens a connection. Every script
and test after that runs completely offline.

## What is deliberately *not* installed

**matplotlib**, **scipy** and **scikit-learn** are not installed in this
environment. The lesson's Tools section describes scikit-learn's
`SimpleImputer` and Great Expectations / pandera from their public
documentation as design contrasts to the hand-rolled contract this lab
builds — no output from any of them is reproduced anywhere in this lab or
its lesson; every place they are mentioned says so plainly.

## If you cannot install anything at all

pandas is not in the Python standard library, and there is no reduced
path through this lab without it. If pandas genuinely cannot be
installed, read the lesson's captured output and `expected-output/`
directory instead; every number there came from a real run and is not
invented.

pandas==3.0.5
pyarrow==25.0.1
numpy==2.5.2
pytest==9.1.1

# Day 125 lab — the brief

Nine exercises, in order. Work top to bottom in `test_cleaning.py`. Every
table comes from a fixture defined in `conftest.py` (`income_spending`,
`temperature_readings`, `dropna_frame`, `sensor_timeseries`,
`coerce_frame`, `country_frame`, `duplicates_frame`, `clean_customers`,
`contract_violating_customers`) — read `data.py` once to see exactly what
each one contains before you start.

Check yourself at any point:

```bash
.venv/bin/pytest starter -v
```

On an untouched checkout that prints `19 skipped`. A **skip** means "not
attempted". Replace a `pytest.skip(...)` line with real assertions and
delete it — when every skip is gone and the suite is green, you are
finished:

```bash
.venv/bin/pytest starter -q
echo $?
```

Assert exact values everywhere except where floating-point arithmetic
makes `pytest.approx` the honest choice (exercise 1's mean/std/correlation,
exercise 2's mean and shift) — never because a number is "roughly one
machine on one day." Every value in this lab is deterministic.

---

## Exercise 1 — mean imputation distorts (`income_spending`)

`income_spending` has `income` missing at 10 of 40 rows, and `spending` is
genuinely, linearly related to `income` (`spending = 0.42 * income +
noise`). Fill the missing `income` values with `income.mean()` and check
three things, before and after:

- **The mean is unchanged** — `income.mean()` before imputation equals
  `income.mean()` after, within `pytest.approx(..., abs=1e-6)`. This is
  the whole reason mean imputation feels safe.
- **The standard deviation strictly shrinks** — assert
  `after_std < before_std`. You have added ten points that sit exactly on
  the mean, which can only pull the spread in.
- **The correlation with `spending` strictly *attenuates*, toward zero —
  it does NOT grow.** This may be the opposite of your first guess. Work
  through *why*: an imputed point's `income` value is, by construction,
  exactly at the column mean, so its deviation from the mean is exactly
  zero. A term with a zero deviation on one axis contributes exactly zero
  to the covariance between `income` and `spending`, no matter what
  `spending` happens to be at that row. It can only ever dilute an
  existing relationship — never strengthen one. This is a mathematical
  fact about the Pearson correlation formula, not a property of this one
  dataset; you can prove it to yourself by writing out the covariance sum
  term by term for a single imputed point.
- Assert `income_spending['income'].isna().sum() == 10` — the exact
  planted count.

## Exercise 2 — `fillna(0)` on a measurement column (`temperature_readings`)

`temperature_readings` has three missing readings **and** one genuine
`0.0` reading already in the data (station C, the last row). `fillna(0)`
makes all four of these identical in the data.

- Assert the mean before and after `fillna(0.0)`, and the exact (negative)
  shift between them. Zero is a value, not an absence — assert the shift
  is not zero.
- Assert that after the fill, the genuine `0.0` reading and the three
  imputed readings are bit-for-bit indistinguishable: nothing in the data
  can tell them apart anymore.

## Exercise 3 — `dropna` with `how`, `thresh`, `subset` (`dropna_frame`)

An 8-row, 4-column frame with a deliberately mixed missingness pattern
across `email`, `phone` and `signup_date`. Assert four different row
counts on the same frame:

- `how='any'` — drops a row missing even one field. The strictest cut.
- `how='all'` — only drops a row missing on *every* field.
- `thresh=2` — keeps rows with at least 2 non-null fields.
- `subset=['email']` — only checks the named column.

Read the docstring in `data.py` for the exact missingness pattern before
you predict the counts.

## Exercise 4 — `ffill` on unsorted data is a real bug (`sensor_timeseries`)

`sensor_timeseries` is written in true chronological order (`day` 1–8)
with two gaps. `shuffle_rows(sensor_timeseries, seed=7)` scrambles the row
*order* without touching `day`.

- `ffill` the shuffled frame **without sorting first**. Re-sort the result
  by `day` only to compare it against the correct answer — the fill
  computation itself already happened on the scrambled order. Assert `day`
  2 and `day` 3 come out wrong.
- Now sort by `day` **before** filling. Assert the full `reading` column
  equals `[10.0, 10.0, 10.0, 13.0, 14.0, 14.0, 16.0, 17.0]` — every gap
  correctly carried forward from its true chronological neighbour.

## Exercise 5 — the missing indicator (`temperature_readings`)

Record `isna()` into a new boolean column **before** you impute anything.
Then impute `reading_c` with its mean. Assert the recorded column still
equals the *original* `isna()` mask exactly, even though
`reading_c.isna()` is now all `False` — the flag is the only place the
evidence survives.

## Exercise 6 — `to_numeric(errors='coerce')` (`coerce_frame`)

`quantity_raw` has three deliberately unparseable strings planted among
seven clean numeric strings. Run
`pd.to_numeric(coerce_frame['quantity_raw'], errors='coerce')` and assert:

- The resulting `NaN` count equals the count of the three planted garbage
  values (`'N/A'`, `'unknown'`, `'--'`) — never coerce a column blind
  without counting what changed.
- The seven clean values survive as floats, in order.

## Exercise 7 — string normalisation (`country_frame`)

Twelve rows, one true country recorded eight different ways
(`'USA'`, `'U.S.A.'`, `' usa '`, `'Usa'`, `'Canada'`, `'canada '`,
`' Canada'`, `'CANADA'`).

- Assert `country_raw.nunique()` is **8** before normalising, and **2**
  after: `.str.strip().str.lower().str.replace('.', '', regex=False)`,
  then map `'usa' -> 'USA'` and `'canada' -> 'Canada'`.
- Assert a raw `groupby('country_raw')` produces **8** groups — visibly
  wrong, since the truth is 2 — and that grouping on the *normalised*
  column produces exactly 2 groups with the correct USA/Canada amount
  totals.

## Exercise 8 — duplicates mean whatever subset you named (`duplicates_frame`)

Row 4 is an **exact** duplicate of row 0 (every column, including the
timestamp — a genuine re-logged entry). Row 5 shares `(customer_id,
item)` with row 1 but has a *different* timestamp — a real second
purchase, not a duplicate log entry, but still a duplicate under a subset
key.

- Assert `duplicated().sum()` and
  `duplicated(subset=['customer_id', 'item']).sum()` give different exact
  counts, with the subset count larger.
- State, with an assertion, which customer(s) each definition flags, and
  say in a comment which definition answers which real question ("was
  this order logged twice?" vs. "did this customer buy this item more
  than once?").

## Exercise 9 — the cleaning contract must hold, and be provably able to fail

`assert_cleaning_contract` (in `contract.py`) checks three post-conditions:
no nulls in named key columns, declared dtypes on named columns, and a row
count inside `[min_rows, max_rows]`. It raises `ContractViolation` naming
the first thing that broke.

- On `clean_customers`, with `key_columns=['customer_id', 'country']`,
  `dtypes={'income': 'float64'}`, `min_rows=3`, `max_rows=10` — the call
  must **not** raise.
- On `contract_violating_customers` (a null `customer_id`) — assert it
  raises `ContractViolation` matching `'customer_id'`.
- Build your own bad frame from `build_clean_customers()` with `income`
  cast to `str` — assert it raises matching `'income'`.
- Build your own bad frame that is `build_clean_customers().iloc[:1]` (one
  row) — assert it raises matching `'row count'`.

A contract that never fails proves nothing; this exercise is only done
once you have shown it can genuinely raise, on three different violations.

---

Prove your suite is not vacuous once you are green: re-break one assertion
on purpose (flip a comparison, change an expected number), confirm the run
exits non-zero with a printed `FAIL`, then restore it and confirm green
again.

"""Shared fixtures. pytest finds this file by itself -- nothing imports it.

Each fixture returns a FRESH copy of its table, so one test's mutation can
never leak into the next test.
"""

import pytest

from data import (
    build_clean_customers,
    build_coerce_frame,
    build_contract_violating_customers,
    build_country_frame,
    build_dropna_frame,
    build_duplicates_frame,
    build_income_spending,
    build_sensor_timeseries,
    build_temperature_readings,
    shuffle_rows,
)


@pytest.fixture
def income_spending():
    return build_income_spending()


@pytest.fixture
def temperature_readings():
    return build_temperature_readings()


@pytest.fixture
def dropna_frame():
    return build_dropna_frame()


@pytest.fixture
def sensor_timeseries():
    return build_sensor_timeseries()


@pytest.fixture
def shuffled_sensor_timeseries():
    return shuffle_rows(build_sensor_timeseries())


@pytest.fixture
def coerce_frame():
    return build_coerce_frame()


@pytest.fixture
def country_frame():
    return build_country_frame()


@pytest.fixture
def duplicates_frame():
    return build_duplicates_frame()


@pytest.fixture
def clean_customers():
    return build_clean_customers()


@pytest.fixture
def contract_violating_customers():
    return build_contract_violating_customers()

"""A tiny cleaning contract: name the post-conditions cleaning is supposed
to guarantee, and check them mechanically instead of trusting that the
cleaning steps above did what they were meant to do.

This is deliberately small -- three checks, one function -- because the
point of exercise 9 is the SHAPE of the idea (assert what you intend, fail
loudly when it does not hold), not a general-purpose validation framework.
Day 126 builds a real reusable pipeline around this idea; this is the
one-function version that proves the shape works.
"""

from __future__ import annotations

import pandas as pd


class ContractViolation(AssertionError):
    """Raised when cleaned data fails one of the contract's post-conditions."""


def assert_cleaning_contract(
    df: pd.DataFrame,
    *,
    key_columns: list[str],
    dtypes: dict[str, str],
    min_rows: int,
    max_rows: int,
) -> None:
    """Check three post-conditions a cleaning step is supposed to guarantee.

    Raises `ContractViolation` naming the FIRST check that fails, so a
    caller sees exactly what broke rather than a generic assertion error.
    """
    for column in key_columns:
        n_null = df[column].isna().sum()
        if n_null > 0:
            raise ContractViolation(
                f"key column {column!r} has {n_null} null value(s); "
                "key columns must be fully populated after cleaning"
            )

    for column, expected_dtype in dtypes.items():
        actual_dtype = str(df[column].dtype)
        if actual_dtype != expected_dtype:
            raise ContractViolation(
                f"column {column!r} has dtype {actual_dtype!r}, expected {expected_dtype!r}"
            )

    n_rows = len(df)
    if not (min_rows <= n_rows <= max_rows):
        raise ContractViolation(
            f"row count {n_rows} is outside the expected range [{min_rows}, {max_rows}]"
        )

"""The tables every exercise in this lab is built from.

Nothing here downloads anything. Two tables use a seeded NumPy generator so
a re-run always produces the same rows (`build_income_spending`,
`build_large_income_spending`); everything else is a small hand-written
literal table, chosen so every exercise's expected values can be checked
exactly rather than approximately.

`build_income_spending` carries the day's opening failure on purpose: some
rows have no recorded income at all, and `spending` is genuinely, linearly
related to `income` plus noise -- so a real correlation exists to distort.
"""

from __future__ import annotations

import numpy as np
import pandas as pd

# --------------------------------------------------------------------------
# Exercises 1 and 2 -- mean imputation and fillna(0), and the correlation
# claim in exercise 1. Seeded so the numbers are reproducible exactly.
# --------------------------------------------------------------------------


def build_income_spending(seed: int = 20250825, n: int = 40, n_missing: int = 10) -> pd.DataFrame:
    """`income` and `spending`, genuinely correlated, with `income` missing
    at `n_missing` rows chosen completely at random (MCAR)."""
    rng = np.random.default_rng(seed)
    income = rng.normal(52_000, 11_000, n)
    spending = 0.42 * income + rng.normal(0, 3_500, n)
    df = pd.DataFrame(
        {
            "customer_id": np.arange(1001, 1001 + n),
            "income": income,
            "spending": spending,
        }
    )
    missing_idx = rng.choice(n, size=n_missing, replace=False)
    df.loc[missing_idx, "income"] = np.nan
    return df


def build_temperature_readings() -> pd.DataFrame:
    """A sensor log where a missing reading and a genuine 0.0C reading must
    stay distinguishable -- exercise 2's fillna(0) trap."""
    return pd.DataFrame(
        {
            "station": ["A", "A", "A", "B", "B", "B", "C", "C", "C", "C"],
            "reading_c": [18.2, np.nan, 17.5, 22.0, 21.4, np.nan, -3.0, np.nan, -1.5, 0.0],
        }
    )


# --------------------------------------------------------------------------
# Exercise 3 -- dropna with how, thresh, subset. Eight rows, three columns,
# a deliberately mixed missingness pattern.
# --------------------------------------------------------------------------


def build_dropna_frame() -> pd.DataFrame:
    return pd.DataFrame(
        {
            "customer_id": [1, 2, 3, 4, 5, 6, 7, 8],
            "email": ["a@x.com", None, "c@x.com", None, "e@x.com", None, "g@x.com", None],
            "phone": ["555-1", "555-2", None, None, "555-5", None, None, None],
            "signup_date": [
                "2026-01-02",
                "2026-01-03",
                "2026-01-04",
                None,
                "2026-01-06",
                None,
                "2026-01-08",
                None,
            ],
        }
    )


# --------------------------------------------------------------------------
# Exercise 4 -- ffill on unsorted data. A daily reading with two gaps,
# written here in TRUE chronological order; the exercise shuffles it.
# --------------------------------------------------------------------------


def build_sensor_timeseries() -> pd.DataFrame:
    return pd.DataFrame(
        {
            "day": [1, 2, 3, 4, 5, 6, 7, 8],
            "reading": [10.0, np.nan, np.nan, 13.0, 14.0, np.nan, 16.0, 17.0],
        }
    )


def shuffle_rows(df: pd.DataFrame, seed: int = 7) -> pd.DataFrame:
    rng = np.random.default_rng(seed)
    order = rng.permutation(len(df))
    return df.iloc[order].reset_index(drop=True)


# --------------------------------------------------------------------------
# Exercise 5 -- the missing indicator. Reuses build_temperature_readings.
# --------------------------------------------------------------------------

# (no separate builder needed -- exercise 5 works directly on
# build_temperature_readings())


# --------------------------------------------------------------------------
# Exercise 6 -- to_numeric(errors="coerce"). A column that is mostly clean
# numbers with a KNOWN number of unparseable strings planted in it.
# --------------------------------------------------------------------------


def build_coerce_frame() -> pd.DataFrame:
    return pd.DataFrame(
        {
            "order_id": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
            "quantity_raw": ["12", "7", "N/A", "3", "unknown", "9", "5", "--", "20", "4"],
        }
    )


# --------------------------------------------------------------------------
# Exercise 7 -- string normalisation. One true country, four spellings.
# --------------------------------------------------------------------------


def build_country_frame() -> pd.DataFrame:
    return pd.DataFrame(
        {
            "customer_id": list(range(1, 13)),
            "country_raw": [
                "USA",
                "U.S.A.",
                " usa ",
                "Usa",
                "USA",
                "U.S.A.",
                "Canada",
                "canada ",
                " Canada",
                "USA",
                "CANADA",
                "U.S.A.",
            ],
            "amount": [
                120.0,
                85.0,
                60.0,
                200.0,
                75.0,
                150.0,
                40.0,
                90.0,
                30.0,
                110.0,
                55.0,
                65.0,
            ],
        }
    )


# --------------------------------------------------------------------------
# Exercise 8 -- duplicates. Row 5 is an exact duplicate of row 1. Row 6
# shares (customer_id, item) with row 2 but has a different price -- a
# subset duplicate that is not an exact duplicate.
# --------------------------------------------------------------------------


def build_duplicates_frame() -> pd.DataFrame:
    return pd.DataFrame(
        {
            "customer_id": [1, 2, 3, 4, 1, 2, 5],
            "item": ["pen", "mug", "pen", "bag", "pen", "mug", "hat"],
            "price": [1.5, 8.0, 1.5, 20.0, 1.5, 8.0, 12.0],
            "order_ts": [
                "2026-01-01T09:00",
                "2026-01-01T09:05",
                "2026-01-01T09:10",
                "2026-01-01T09:15",
                "2026-01-01T09:00",
                "2026-01-02T14:30",
                "2026-01-01T09:20",
            ],
        }
    )


# --------------------------------------------------------------------------
# Exercise 9 -- the cleaning contract. A frame that passes, and one that
# is built to fail each post-condition in turn.
# --------------------------------------------------------------------------


def build_clean_customers() -> pd.DataFrame:
    return pd.DataFrame(
        {
            "customer_id": [1, 2, 3, 4, 5],
            "country": ["USA", "Canada", "USA", "Canada", "USA"],
            "income": [52_000.0, 61_000.0, 48_500.0, 73_200.0, 55_000.0],
        }
    )


def build_contract_violating_customers() -> pd.DataFrame:
    """Violates the contract three ways: a null in a key column, the wrong
    dtype on `income` (object, because one entry is a stray string), and a
    row count far outside the expected range."""
    return pd.DataFrame(
        {
            "customer_id": [1, 2, None],
            "country": ["USA", "Canada", "USA"],
            "income": [52_000.0, "unknown", 48_500.0],
        }
    )

"""YOUR test suite for Day 125 -- "Cleaning With Receipts". Nine exercises.

Run it from the lab directory, not from here:

    pytest starter -v

Every exercise below ends in a `pytest.skip(...)` line. pytest reports a
skip as `s` in the dot line and moves on, so an unfinished suite still
exits 0. Replace each skip with real assertions -- deleting the skip line
is part of the exercise. `starter/00_brief.md` explains each exercise in
full; the fixtures you need come from `conftest.py` and are described
there too.

Assert exact values everywhere except where the brief says approx is
appropriate (the imputation statistics use pytest.approx because they are
floating-point arithmetic, not because the day is one machine on one day).
"""

import numpy as np
import pandas as pd
import pytest

from contract import ContractViolation, assert_cleaning_contract

# --------------------------------------------------------------------------
# EXERCISE 1 -- mean imputation distorts. See starter/00_brief.md exercise 1.
#
# Check with:   pytest starter -v -k test_1
# --------------------------------------------------------------------------


def test_1_mean_survives_imputation_unchanged(income_spending):
    pytest.skip(
        "exercise 1a: assert income.mean() before and after fillna(mean) match "
        "within pytest.approx(..., abs=1e-6) of each other"
    )


def test_1_std_strictly_shrinks(income_spending):
    pytest.skip(
        "exercise 1b: assert income.std() strictly decreases after mean imputation"
    )


def test_1_correlation_strictly_attenuates_not_inflates(income_spending):
    pytest.skip(
        "exercise 1c: assert abs(corr(income, spending)) strictly decreases after "
        "mean-imputing income -- NOT increases; see 00_brief.md for why the direction "
        "is forced, not a guess"
    )


def test_1_n_missing_matches_the_planted_count(income_spending):
    pytest.skip("exercise 1d: assert income_spending['income'].isna().sum() == 10")


# --------------------------------------------------------------------------
# EXERCISE 2 -- fillna(0) on a measurement column. Zero is a value.
#
# Check with:   pytest starter -v -k test_2
# --------------------------------------------------------------------------


def test_2_fillna_zero_moves_the_mean_by_the_exact_amount(temperature_readings):
    pytest.skip(
        "exercise 2a: assert the mean before and after fillna(0.0), and the exact "
        "(negative) shift between them"
    )


def test_2_fillna_zero_confuses_missing_with_a_real_zero_reading(temperature_readings):
    pytest.skip(
        "exercise 2b: row index 9 is a GENUINE 0.0C reading, not missing. After "
        "fillna(0), assert it is bit-for-bit indistinguishable from the three rows "
        "that really were missing"
    )


# --------------------------------------------------------------------------
# EXERCISE 3 -- dropna with how, thresh, subset.
#
# Check with:   pytest starter -v -k test_3
# --------------------------------------------------------------------------


def test_3_dropna_row_counts(dropna_frame):
    pytest.skip(
        "exercise 3: on the 8-row dropna_frame, assert the row counts for "
        "how='any', how='all', thresh=2, and subset=['email']"
    )


# --------------------------------------------------------------------------
# EXERCISE 4 -- ffill on unsorted data is a real bug.
#
# Check with:   pytest starter -v -k test_4
# --------------------------------------------------------------------------


def test_4_ffill_on_unsorted_data_is_wrong(sensor_timeseries):
    pytest.skip(
        "exercise 4a: shuffle_rows(sensor_timeseries, seed=7), ffill WITHOUT sorting "
        "first, and assert day 2 and day 3 come out wrong (compare against the "
        "sorted-then-filled correct answer)"
    )


def test_4_ffill_after_sorting_is_correct(sensor_timeseries):
    pytest.skip(
        "exercise 4b: shuffle, sort_values('day'), THEN ffill; assert the full "
        "reading column equals [10.0, 10.0, 10.0, 13.0, 14.0, 14.0, 16.0, 17.0]"
    )


# --------------------------------------------------------------------------
# EXERCISE 5 -- the missing indicator.
#
# Check with:   pytest starter -v -k test_5
# --------------------------------------------------------------------------


def test_5_missing_indicator_exactly_matches_original_mask(temperature_readings):
    pytest.skip(
        "exercise 5: record isna() into a new column BEFORE imputing, then impute "
        "with the mean; assert the recorded column still equals the ORIGINAL isna() "
        "mask, even though reading_c.isna() is now all False"
    )


# --------------------------------------------------------------------------
# EXERCISE 6 -- to_numeric(errors='coerce').
#
# Check with:   pytest starter -v -k test_6
# --------------------------------------------------------------------------


def test_6_coerce_count_matches_the_planted_garbage(coerce_frame):
    pytest.skip(
        "exercise 6: pd.to_numeric(coerce_frame['quantity_raw'], errors='coerce'); "
        "assert the resulting NaN count equals the count of the three planted garbage "
        "strings ('N/A', 'unknown', '--'), and that the clean values survive as floats"
    )


# --------------------------------------------------------------------------
# EXERCISE 7 -- string normalisation.
#
# Check with:   pytest starter -v -k test_7
# --------------------------------------------------------------------------


def test_7_nunique_before_and_after_normalisation(country_frame):
    pytest.skip(
        "exercise 7a: assert country_raw.nunique() is 8 before normalising "
        "(.str.strip().str.lower().str.replace('.', '', regex=False), then map "
        "'usa'->'USA', 'canada'->'Canada'), and 2 after"
    )


def test_7_raw_groupby_splits_one_true_country_into_several(country_frame):
    pytest.skip(
        "exercise 7b: assert groupby('country_raw') produces 8 groups (wrong -- the "
        "truth is 2), then assert the normalised groupby produces 2 groups with the "
        "correct USA/Canada amount totals"
    )


# --------------------------------------------------------------------------
# EXERCISE 8 -- duplicates mean whatever subset you named.
#
# Check with:   pytest starter -v -k test_8
# --------------------------------------------------------------------------


def test_8_exact_and_subset_duplicate_counts_differ(duplicates_frame):
    pytest.skip(
        "exercise 8a: assert duplicated().sum() and duplicated(subset=['customer_id', "
        "'item']).sum() are different exact counts, and that the subset count is larger"
    )


def test_8_which_definition_is_right_depends_on_the_question(duplicates_frame):
    pytest.skip(
        "exercise 8b: identify which customer_id(s) the exact-duplicate definition "
        "flags, and which the subset definition flags, and assert both sets exactly"
    )


# --------------------------------------------------------------------------
# EXERCISE 9 -- the cleaning contract must hold AND be provably able to fail.
#
# Check with:   pytest starter -v -k test_9
# --------------------------------------------------------------------------


def test_9_contract_passes_on_clean_data(clean_customers):
    pytest.skip(
        "exercise 9a: call assert_cleaning_contract on clean_customers with "
        "key_columns=['customer_id', 'country'], dtypes={'income': 'float64'}, "
        "min_rows=3, max_rows=10; it must NOT raise"
    )


def test_9_contract_raises_on_a_null_key_column(contract_violating_customers):
    pytest.skip(
        "exercise 9b: with pytest.raises(ContractViolation, match='customer_id'), "
        "call the same contract on contract_violating_customers"
    )


def test_9_contract_raises_on_a_wrong_dtype():
    pytest.skip(
        "exercise 9c: take build_clean_customers(), cast income to str, and assert "
        "the contract raises ContractViolation matching 'income'"
    )


def test_9_contract_raises_on_a_row_count_outside_the_range():
    pytest.skip(
        "exercise 9d: take build_clean_customers().iloc[:1] (1 row) and assert the "
        "contract raises ContractViolation matching 'row count'"
    )

#!/usr/bin/env bash
# Tests for the Day 125 lab. Run from the lab directory:
#   bash tests/run_tests.sh
#
# The harness proves the lesson's claims by running code and reading real
# values, never by reading source:
#
#   * mean imputation leaves the mean exactly unchanged, strictly shrinks
#     the standard deviation, and strictly ATTENUATES a real correlation
#     toward zero -- never inflates it, which is the mathematically forced
#     direction, not a guess;
#   * fillna(0) on a measurement column moves the mean by a real, nonzero
#     amount, and makes a genuine 0.0 reading indistinguishable from a
#     truly missing one;
#   * dropna's how='any', how='all', thresh= and subset= give four
#     different, exact row counts on the same frame;
#   * ffill on unsorted data gives the wrong answer at the specific rows
#     where order matters, and the right answer once sorted first;
#   * a missing-indicator column recorded before imputation still matches
#     the original isna() mask after the fill has erased the evidence;
#   * to_numeric(errors='coerce') turns exactly the planted garbage
#     strings into NaN, and nothing else;
#   * string normalisation collapses eight raw spellings of two countries
#     down to two, and a raw groupby silently produces eight groups where
#     the truth is two;
#   * exact-duplicate and subset-duplicate counts differ, and the correct
#     definition depends on the question being asked;
#   * a hand-rolled cleaning contract passes on clean data and genuinely
#     RAISES on three different kinds of violation;
#   * the reference suite (`examples/`) passes in full;
#   * the exercise suite (`starter/`) is all-skip on an untouched checkout,
#     and the harness proves it can genuinely FAIL by solving every
#     exercise in a scratch copy, breaking one assertion on purpose,
#     confirming a non-zero exit and a printed FAIL, then restoring it;
#   * nothing is left behind on disk.
#
# Everything after the one-time install runs offline. Nothing binds a port,
# nothing writes outside the lab, nothing needs a key. Deterministic,
# non-interactive, exits 0 only if every check passes.
set -u

export PYTHONDONTWRITEBYTECODE=1

lab_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"

# Bytecode left by an EARLIER command is not this run's litter. The README
# documents `pytest starter -q` and `pytest examples -q` separately, and
# running either writes .pyc files that would then fail the cleanliness
# check at the end of this script -- failing the reader for following the
# instructions. Clearing them here makes that final check measure what it
# claims to: what THIS run left behind. `.venv` is untouched, because the
# packages' own bytecode is theirs, not ours.
find "${lab_dir}" -name '.venv' -prune -o -type d -name '__pycache__' -exec rm -rf {} + 2>/dev/null || true
find "${lab_dir}" -name '.venv' -prune -o -type d -name '.pytest_cache' -exec rm -rf {} + 2>/dev/null || true

failures=0
checks=0

check() {
  local label="$1" ok="$2"
  checks=$((checks + 1))
  if [ "${ok}" = "yes" ]; then
    echo "  ok: ${label}"
  else
    echo "  FAIL: ${label}"
    failures=$((failures + 1))
  fi
}

check_eq() {
  # check_eq <label> <expected> <actual>
  if [ "$2" = "$3" ]; then
    check "$1" "yes"
  else
    check "$1 (expected [$2], got [$3])" "no"
  fi
}

# Resolve pytest: an explicit override, then this lab's .venv, then PATH.
# Fails loudly with instructions rather than silently skipping checks.
resolve_tool() {
  local tool="$1" override="$2"
  if [ -n "${override}" ] && [ -x "${override}" ]; then echo "${override}"; return 0; fi
  if [ -x "${lab_dir}/.venv/bin/${tool}" ]; then echo "${lab_dir}/.venv/bin/${tool}"; return 0; fi
  if command -v "${tool}" >/dev/null 2>&1; then command -v "${tool}"; return 0; fi
  return 1
}

pytest_bin="$(resolve_tool pytest "${PYTEST:-}")" || {
  echo "FAIL: pytest not found." >&2
  echo "  Install the lab's dependencies with:" >&2
  echo "    python3 -m venv .venv" >&2
  echo "    .venv/bin/pip install -r requirements/requirements.txt" >&2
  echo "  Or point this suite at an existing pytest:" >&2
  echo "    PYTEST=/path/to/pytest bash tests/run_tests.sh" >&2
  exit 1
}

python_bin="$(dirname "${pytest_bin}")/python3"
if [ ! -x "${python_bin}" ]; then
  python_bin="$(command -v python3 || true)"
fi
if [ -z "${python_bin}" ]; then
  echo "FAIL: python3 not found on PATH." >&2
  exit 1
fi

if ! "${python_bin}" -c "import pandas" >/dev/null 2>&1; then
  echo "FAIL: pandas is not importable from ${python_bin}." >&2
  echo "  Install the lab's dependencies with:" >&2
  echo "    python3 -m venv .venv" >&2
  echo "    .venv/bin/pip install -r requirements/requirements.txt" >&2
  exit 1
fi

echo "Day 125 — Cleaning With Receipts"
echo

# --------------------------------------------------------------------------
echo "1. The tools and the versions this lab was written against"
# --------------------------------------------------------------------------

versions="$("${python_bin}" - <<'PY'
import platform
import sys
from importlib.metadata import version

print(f"python   {platform.python_version()}")
for name in ("pandas", "pyarrow", "numpy", "pytest"):
    try:
        print(f"{name:<8} {version(name)}")
    except Exception as exc:  # pragma: no cover
        print(f"{name:<8} NOT INSTALLED ({exc})")
PY
)"
echo "${versions}"
echo

pandas_version="$("${python_bin}" -c "import pandas; print(pandas.__version__)" 2>/dev/null || echo "")"
pinned_pandas="$(grep -m1 '^pandas==' "${lab_dir}/requirements/requirements.txt" | cut -d= -f3)"
check_eq "installed pandas matches requirements.txt exactly" "${pinned_pandas}" "${pandas_version}"
echo

# --------------------------------------------------------------------------
echo "2. Reference suite -- examples/ must pass in full"
# --------------------------------------------------------------------------

examples_output="$(cd "${lab_dir}" && "${pytest_bin}" examples -q 2>&1)"
examples_status=$?
echo "${examples_output}" | tail -5
check "examples/ exits 0" "$( [ ${examples_status} -eq 0 ] && echo yes || echo no )"

examples_passed_line="$(echo "${examples_output}" | grep -E '^[0-9]+ passed' || true)"
check "examples/ reports 19 passed, 0 failed" "$( echo "${examples_passed_line}" | grep -qE '^19 passed' && echo yes || echo no )"
echo

# --------------------------------------------------------------------------
echo "3. Exercise suite -- starter/ is all-skip on an untouched checkout"
# --------------------------------------------------------------------------

starter_output="$(cd "${lab_dir}" && "${pytest_bin}" starter -q 2>&1)"
starter_status=$?
echo "${starter_output}" | tail -5
check "starter/ (untouched) exits 0" "$( [ ${starter_status} -eq 0 ] && echo yes || echo no )"
check "starter/ (untouched) reports 19 skipped, 0 failed" "$( echo "${starter_output}" | grep -qE '^19 skipped' && echo yes || echo no )"
echo

# --------------------------------------------------------------------------
echo "4. Never run 'pytest examples starter' in one invocation -- same"
echo "   module name in both directories means the second collected can"
echo "   shadow the first. Documented and checked separately, above."
# --------------------------------------------------------------------------
echo

# --------------------------------------------------------------------------
echo "5. Prove the suite can genuinely FAIL: solve every exercise in a"
echo "   scratch copy, confirm green, break one assertion on purpose,"
echo "   confirm a non-zero exit and a printed FAIL, then restore."
# --------------------------------------------------------------------------

scratch_dir="$(mktemp -d "${TMPDIR:-/tmp}/d125-scratch.XXXXXX")"
cleanup_scratch() { rm -rf "${scratch_dir}"; }
trap cleanup_scratch EXIT

cp "${lab_dir}/examples/test_cleaning.py" "${scratch_dir}/test_cleaning.py"
cp "${lab_dir}/examples/data.py" "${scratch_dir}/data.py"
cp "${lab_dir}/examples/contract.py" "${scratch_dir}/contract.py"
cp "${lab_dir}/examples/conftest.py" "${scratch_dir}/conftest.py"

solved_output="$("${pytest_bin}" "${scratch_dir}" -q 2>&1)"
solved_status=$?
check "scratch copy of the solved suite exits 0" "$( [ ${solved_status} -eq 0 ] && echo yes || echo no )"
check "scratch copy reports 19 passed" "$( echo "${solved_output}" | grep -qE '^19 passed' && echo yes || echo no )"

# Break test_3's exact thresh=2 count on purpose: 5 -> 999.
sed -i.bak 's/assert thresh_2.shape\[0\] == 5/assert thresh_2.shape[0] == 999/' "${scratch_dir}/test_cleaning.py"

broken_output="$("${pytest_bin}" "${scratch_dir}" -q 2>&1)"
broken_status=$?
check "broken scratch copy exits non-zero" "$( [ ${broken_status} -ne 0 ] && echo yes || echo no )"
check "broken scratch copy prints a FAIL/failed line" "$( echo "${broken_output}" | grep -qiE 'failed|assert' && echo yes || echo no )"

mv "${scratch_dir}/test_cleaning.py.bak" "${scratch_dir}/test_cleaning.py"
restored_output="$("${pytest_bin}" "${scratch_dir}" -q 2>&1)"
restored_status=$?
check "restored scratch copy exits 0 again" "$( [ ${restored_status} -eq 0 ] && echo yes || echo no )"
check "restored scratch copy reports 19 passed again" "$( echo "${restored_output}" | grep -qE '^19 passed' && echo yes || echo no )"

cleanup_scratch
trap - EXIT
echo

# --------------------------------------------------------------------------
echo "6. Nothing in examples/ or starter/ opens a network connection"
# --------------------------------------------------------------------------

url_hits="$(grep -rEl 'https?://|ftp://' "${lab_dir}/examples" "${lab_dir}/starter" 2>/dev/null || true)"
check "no URLs inside examples/ or starter/" "$( [ -z "${url_hits}" ] && echo yes || echo no )"
echo

# --------------------------------------------------------------------------
echo "7. Cleanliness -- nothing left behind by THIS run"
# --------------------------------------------------------------------------

find "${lab_dir}" -name '.venv' -prune -o -type d -name '__pycache__' -exec rm -rf {} + 2>/dev/null || true
find "${lab_dir}" -name '.venv' -prune -o -type d -name '.pytest_cache' -exec rm -rf {} + 2>/dev/null || true

stray="$(find "${lab_dir}" -name '.venv' -prune -o \( -type d -name '__pycache__' -print -o -type d -name '.pytest_cache' -print \) 2>/dev/null || true)"
check "no __pycache__ or .pytest_cache left behind" "$( [ -z "${stray}" ] && echo yes || echo no )"
echo

echo "-------------------------------------------------------------"
echo "${checks} checks, ${failures} failure(s)"
if [ "${failures}" -gt 0 ]; then
  exit 1
fi
exit 0