Hands-on lab — Day 139: Reproducible Notebooks

"""The one piece of logic in this lab that lives in a module, not a cell.

Day 139's rule: exploration belongs in the notebook, but anything the
rest of a project depends on belongs in an imported module that has its
own tests -- the notebook then becomes a caller, not the only place the
logic exists. This module is that split made concrete: ``clean_mean`` is
plain, reusable, deterministic logic, and ``test_calc.py`` tests it the
ordinary way, with no kernel and no notebook involved.
"""

from __future__ import annotations


def clean_mean(values: list) -> float:
    """Mean of ``values`` after dropping ``None`` entries.

    Raises ``ValueError`` if every value is ``None`` or the list is empty,
    because a mean of nothing is not a number this function will guess at.
    """
    kept = [v for v in values if v is not None]
    if not kept:
        raise ValueError("clean_mean: no non-None values to average")
    return sum(kept) / len(kept)

"""Shared helpers for building, executing and inspecting notebooks.

Every function here works on in-memory ``nbformat`` notebook objects.
Nothing in this module writes a notebook file to disk -- the tests build
notebooks in memory, execute them with real Jupyter kernels through
``nbclient``, and assert on the resulting JSON structure. That is how you
test a notebook without a UI.
"""

from __future__ import annotations

import copy
import sys
from typing import Iterable

import nbclient
import nbconvert
import nbformat as nbf
from nbclient import NotebookClient
from nbclient.exceptions import CellExecutionError


def analyst_notebook() -> nbf.NotebookNode:
    """Build the three-cell notebook the lesson opens with.

    Cell 0 ("setup") sets ``x``. Cell 1 ("transform") is a cleaning step
    an analyst adds after already having looked at the answer once. Cell
    2 ("report") divides ``x`` by two and displays it. Run top to bottom
    the answer is 30.0; run in the order an analyst really used --
    setup, then a quick look at the report, then the transform, without
    ever re-running the report -- the notebook still displays a number
    with no error, and that number is wrong.
    """
    nb = nbf.v4.new_notebook()
    nb.cells = [
        nbf.v4.new_code_cell("x = 100  # setup", id="setup"),
        nbf.v4.new_code_cell(
            "x = x - 40  # a cleaning step added after the first look at the report",
            id="transform",
        ),
        nbf.v4.new_code_cell("answer = x / 2\nanswer", id="report"),
    ]
    return nb


def execute_clean(nb: nbf.NotebookNode) -> nbf.NotebookNode:
    """Execute every cell top to bottom in a fresh kernel and return it."""
    nb = copy.deepcopy(nb)
    NotebookClient(nb, kernel_name="python3").execute()
    return nb


def execute_in_order(nb: nbf.NotebookNode, order: Iterable[int]) -> nbf.NotebookNode:
    """Execute the cells of ``nb`` in ``order`` (a sequence of cell indices).

    A single kernel is started once and reused across every cell in
    ``order``, exactly like clicking "Run" on cells in whatever sequence
    an analyst actually clicks them, rather than in document order.
    """
    nb = copy.deepcopy(nb)
    client = NotebookClient(nb, kernel_name="python3")
    with client.setup_kernel():
        for index in order:
            client.execute_cell(nb.cells[index], index)
    return nb


def final_value(nb: nbf.NotebookNode, cell_index: int) -> str:
    """Return the ``text/plain`` payload of a cell's last execute_result."""
    outputs = nb.cells[cell_index].get("outputs", [])
    for output in reversed(outputs):
        if output.get("output_type") == "execute_result":
            return output["data"]["text/plain"]
    raise AssertionError(f"cell {cell_index} has no execute_result output")


def execution_counts(nb: nbf.NotebookNode) -> list:
    return [cell.get("execution_count") for cell in nb.cells]


def is_monotonic(counts: list) -> bool:
    """True if every non-None count strictly increases with cell position."""
    seen = [c for c in counts if c is not None]
    return seen == sorted(seen) and len(seen) == len(set(seen))


def hidden_state_pair() -> tuple:
    """Build the two cells behind the hidden-state exercise.

    ``cell_defining`` sets a helper value. ``cell_using`` depends on it
    but does not define it. In the story, ``cell_defining`` is the cell an
    analyst deletes from the document once its job looks done -- while
    the kernel process, if it was never restarted, still remembers the
    value it set.
    """
    cell_defining = nbf.v4.new_code_cell(
        "helper_value = 42  # a cell that will be deleted from the document"
    )
    cell_using = nbf.v4.new_code_cell("total = helper_value + 8\ntotal")
    return cell_defining, cell_using


def run_in_dirty_kernel_after_deletion(cell_defining, cell_using) -> nbf.NotebookNode:
    """Run both cells, delete the defining cell from the document, rerun
    the remaining cell in the *same, still-alive* kernel, and return the
    notebook as it now stands on disk: one cell, whose variable the
    document itself never defines.
    """
    nb = nbf.v4.new_notebook()
    nb.cells = [copy.deepcopy(cell_defining), copy.deepcopy(cell_using)]
    client = NotebookClient(nb, kernel_name="python3")
    with client.setup_kernel():
        client.execute_cell(nb.cells[0], 0)
        client.execute_cell(nb.cells[1], 1)
        # Simulate deleting the defining cell from the document. The
        # kernel process behind `client` is untouched by this -- only the
        # notebook's list of cells changes.
        nb.cells = [nb.cells[1]]
        client.execute_cell(nb.cells[0], 0)
    return nb


def run_fresh_kernel(nb: nbf.NotebookNode) -> nbf.NotebookNode:
    """Execute ``nb`` (as it stands, with whatever cells it currently has)
    in a brand-new kernel that has no memory of any earlier session.
    """
    nb = copy.deepcopy(nb)
    NotebookClient(nb, kernel_name="python3").execute()
    return nb


def failing_notebook() -> nbf.NotebookNode:
    nb = nbf.v4.new_notebook()
    nb.cells = [
        nbf.v4.new_code_cell("row_count = 12"),
        nbf.v4.new_code_cell("raise ValueError('row_count below the expected minimum')"),
        nbf.v4.new_code_cell("row_count * 2"),
    ]
    return nb


def strip_outputs(nb: nbf.NotebookNode) -> nbf.NotebookNode:
    """Return a deep copy with outputs, execution_count and the
    per-execution timestamp metadata removed -- the same three fields an
    ``nbstripout``-style pre-commit hook clears before a notebook is
    committed.
    """
    nb = copy.deepcopy(nb)
    for cell in nb.cells:
        if cell.cell_type == "code":
            cell["outputs"] = []
            cell["execution_count"] = None
            cell["metadata"].pop("execution", None)
    return nb


def parameters_notebook(threshold: int) -> nbf.NotebookNode:
    """Build a notebook with a papermill-style parameters cell.

    Cell 0 is tagged ``parameters`` and carries the default. Cell 1 is
    the *injected-parameters* cell papermill adds immediately after it
    -- papermill never edits the parameters cell itself, it appends a
    new cell that overrides the default, so the original default stays
    visible in the document. Cells 2 and 3 are the unchanged analysis:
    they never mention ``threshold``'s value directly and are identical
    text across every variant.
    """
    nb = nbf.v4.new_notebook()
    default_cell = nbf.v4.new_code_cell("threshold = 10  # default")
    default_cell["metadata"]["tags"] = ["parameters"]
    injected_cell = nbf.v4.new_code_cell(f"# Parameters\nthreshold = {threshold}\n")
    injected_cell["metadata"]["tags"] = ["injected-parameters"]
    nb.cells = [
        default_cell,
        injected_cell,
        nbf.v4.new_code_cell("data = [3, 12, 7, 15, 2]"),
        nbf.v4.new_code_cell("filtered = [d for d in data if d > threshold]\nfiltered"),
    ]
    return nb


def to_markdown(nb: nbf.NotebookNode) -> str:
    """Convert an executed notebook to Markdown with nbconvert."""
    exporter = nbconvert.MarkdownExporter()
    body, _resources = exporter.from_notebook_node(nb)
    return body


def to_html(nb: nbf.NotebookNode) -> str:
    exporter = nbconvert.HTMLExporter()
    exporter.template_name = "basic"
    body, _resources = exporter.from_notebook_node(nb)
    return body


def to_script(nb: nbf.NotebookNode) -> str:
    exporter = nbconvert.PythonExporter()
    body, _resources = exporter.from_notebook_node(nb)
    return body


def record_environment() -> dict:
    """What a reproducible notebook should record about the kernel that
    ran it: the interpreter version and the exact versions of the
    packages the notebook stack itself depends on.
    """
    return {
        "python_version": sys.version.split()[0],
        "nbformat": nbf.__version__,
        "nbclient": nbclient.__version__,
        "nbconvert": nbconvert.__version__,
    }


def environment_cell_notebook() -> nbf.NotebookNode:
    """A one-cell notebook that records its own environment when run."""
    nb = nbf.v4.new_notebook()
    nb.cells = [
        nbf.v4.new_code_cell(
            "import sys, nbformat, nbclient, nbconvert\n"
            "record = {\n"
            "    'python_version': sys.version.split()[0],\n"
            "    'nbformat': nbformat.__version__,\n"
            "    'nbclient': nbclient.__version__,\n"
            "    'nbconvert': nbconvert.__version__,\n"
            "}\n"
            "record"
        )
    ]
    return nb


CellExecutionError = CellExecutionError  # re-exported for the tests

"""Ordinary tests for the module half of the notebook/module split.

No kernel, no notebook, no nbclient anywhere in this file -- which is
the point exercise 8 makes: this is what it looks like when reused logic
lives somewhere pytest can reach it directly.
"""

from calc import clean_mean


def test_clean_mean_drops_none():
    assert clean_mean([1, None, 3]) == 2.0


def test_clean_mean_all_present():
    assert clean_mean([2, 4, 6]) == 4.0


def test_clean_mean_raises_on_empty():
    try:
        clean_mean([None, None])
    except ValueError:
        pass
    else:
        raise AssertionError("expected ValueError for an all-None input")

"""Nine exercises in notebooks that reproduce.

Every notebook here is built with ``nbformat``, executed with real
Jupyter kernels through ``nbclient``, and inspected as JSON -- nothing is
opened in a browser and nothing is retyped from a screenshot. Each test
asserts on the resulting structure: cell outputs, ``execution_count``
values, and (for exercise 7) the text nbconvert produces.
"""

import copy
import importlib
import sys

import nbformat as nbf
import pytest

import nb_lib
from nb_lib import CellExecutionError
from calc import clean_mean


# ---------------------------------------------------------------------
# 1. Out-of-order changes the answer
# ---------------------------------------------------------------------
def test_01_out_of_order_changes_the_answer():
    nb = nb_lib.analyst_notebook()

    clean = nb_lib.execute_clean(nb)
    clean_answer = nb_lib.final_value(clean, 2)

    # The order an analyst really used: run the setup cell, take a quick
    # look at the report, then add and run the cleaning step -- and never
    # re-run the report to see the corrected number.
    scrambled = nb_lib.execute_in_order(nb, [0, 2, 1])
    scrambled_answer = nb_lib.final_value(scrambled, 2)

    assert clean_answer == "30.0"
    assert scrambled_answer == "50.0"
    assert clean_answer != scrambled_answer


# ---------------------------------------------------------------------
# 2. execution_count is the evidence
# ---------------------------------------------------------------------
def test_02_execution_count_is_the_evidence():
    nb = nb_lib.analyst_notebook()

    scrambled = nb_lib.execute_in_order(nb, [0, 2, 1])
    scrambled_counts = nb_lib.execution_counts(scrambled)
    assert scrambled_counts == [1, 3, 2]
    assert not nb_lib.is_monotonic(scrambled_counts)

    clean = nb_lib.execute_clean(nb)
    clean_counts = nb_lib.execution_counts(clean)
    assert clean_counts == [1, 2, 3]
    assert nb_lib.is_monotonic(clean_counts)


# ---------------------------------------------------------------------
# 3. Hidden state
# ---------------------------------------------------------------------
def test_03_hidden_state_survives_cell_deletion_in_a_dirty_kernel():
    cell_defining, cell_using = nb_lib.hidden_state_pair()

    dirty = nb_lib.run_in_dirty_kernel_after_deletion(cell_defining, cell_using)
    # The document now has exactly one cell -- `helper_value` is nowhere
    # in it -- and it still ran, because the kernel remembered.
    assert len(dirty.cells) == 1
    assert "helper_value = 42" not in dirty.cells[0].source  # the definition is gone
    assert "helper_value" in dirty.cells[0].source  # only the use remains
    assert nb_lib.final_value(dirty, 0) == "50"

    with pytest.raises(CellExecutionError) as excinfo:
        nb_lib.run_fresh_kernel(dirty)
    assert excinfo.value.ename == "NameError"


# ---------------------------------------------------------------------
# 4. Execution as a test
# ---------------------------------------------------------------------
def test_04_nbclient_raises_on_a_failing_cell_and_names_it():
    nb = nb_lib.failing_notebook()

    with pytest.raises(CellExecutionError) as excinfo:
        nb_lib.execute_clean(nb)

    error = excinfo.value
    assert error.ename == "ValueError"
    assert error.evalue == "row_count below the expected minimum"
    # nbclient names the failing cell by its execution position ("In[2]")
    # and quotes the cell's own source in the exception message, so a CI
    # log points straight at the broken cell.
    assert "In[2]" in str(error)
    assert "raise ValueError" in str(error)


# ---------------------------------------------------------------------
# 5. Output stripping
# ---------------------------------------------------------------------
def test_05_stripping_outputs_makes_two_runs_identical():
    base = nb_lib.analyst_notebook()

    run_a = nb_lib.execute_clean(base)
    run_b = nb_lib.execute_clean(base)

    unstripped_a = nbf.writes(run_a)
    unstripped_b = nbf.writes(run_b)
    assert unstripped_a != unstripped_b

    stripped_a = nbf.writes(nb_lib.strip_outputs(run_a))
    stripped_b = nbf.writes(nb_lib.strip_outputs(run_b))
    assert stripped_a == stripped_b

    # Report exactly what differs: identical code, identical results
    # (execution_count matches cell-for-cell), and yet the unstripped
    # documents disagree -- because nbclient stamps each cell's metadata
    # with the wall-clock time it started and finished.
    counts_a = nb_lib.execution_counts(run_a)
    counts_b = nb_lib.execution_counts(run_b)
    assert counts_a == counts_b  # NOT what differs, despite common lore

    differing_fields = set()
    for cell_a, cell_b in zip(run_a.cells, run_b.cells):
        for key in cell_a.keys():
            if cell_a[key] != cell_b.get(key):
                differing_fields.add(key)
    assert differing_fields == {"metadata"}
    for cell_a, cell_b in zip(run_a.cells, run_b.cells):
        assert cell_a["metadata"]["execution"] != cell_b["metadata"]["execution"]
        assert set(cell_a["metadata"]["execution"].keys()) == {
            "iopub.status.busy",
            "iopub.execute_input",
            "shell.execute_reply",
            "iopub.status.idle",
        }


# ---------------------------------------------------------------------
# 6. Parameterisation
# ---------------------------------------------------------------------
def test_06_parameterised_variants_differ_with_identical_code_cells():
    strict = nb_lib.execute_clean(nb_lib.parameters_notebook(threshold=10))
    loose = nb_lib.execute_clean(nb_lib.parameters_notebook(threshold=5))

    assert nb_lib.final_value(strict, 3) == "[12, 15]"
    assert nb_lib.final_value(loose, 3) == "[12, 7, 15]"

    # Cells 0 (the default), 2 and 3 (the analysis) are untouched between
    # variants; only cell 1, the injected-parameters cell, differs.
    assert strict.cells[0].source == loose.cells[0].source
    assert strict.cells[2].source == loose.cells[2].source
    assert strict.cells[3].source == loose.cells[3].source
    assert strict.cells[1].source != loose.cells[1].source
    assert "threshold = 10" in strict.cells[1].source
    assert "threshold = 5" in loose.cells[1].source
    assert strict.cells[0].metadata["tags"] == ["parameters"]


# ---------------------------------------------------------------------
# 7. Conversion
# ---------------------------------------------------------------------
def test_07_convert_to_markdown_carries_prose_and_computed_values():
    nb = nbf.v4.new_notebook()
    nb.cells = [
        nbf.v4.new_markdown_cell(
            "## Row count check\n\nWe expect at least 100 rows after cleaning."
        ),
        nbf.v4.new_code_cell("row_count = 40 + 74\nrow_count"),
    ]
    executed = nb_lib.execute_clean(nb)
    markdown = nb_lib.to_markdown(executed)

    assert "Row count check" in markdown
    assert "We expect at least 100 rows after cleaning." in markdown
    assert "114" in markdown  # the computed value, not retyped


# ---------------------------------------------------------------------
# 8. Notebook versus module
# ---------------------------------------------------------------------
def test_08_module_logic_is_covered_a_notebook_cell_is_not():
    # The module route: an ordinary import, an ordinary call, and
    # test_calc.py in this same directory already covers it under pytest.
    assert clean_mean([10, None, 20]) == 15.0

    # The notebook route: the same computation, inlined in a cell that
    # was never turned into a module. Executing it inside a kernel works
    # fine -- notebooks run inline code all the time -- but pytest cannot
    # reach it, because a .ipynb is not something Python's import system
    # can import, with or without a kernel involved.
    nb = nbf.v4.new_notebook()
    nb.cells = [
        nbf.v4.new_code_cell(
            "def clean_mean_inline(values):\n"
            "    kept = [v for v in values if v is not None]\n"
            "    return sum(kept) / len(kept)\n"
            "clean_mean_inline([10, None, 20])"
        )
    ]
    executed = nb_lib.execute_clean(nb)
    assert nb_lib.final_value(executed, 0) == "15.0"

    with pytest.raises(ModuleNotFoundError):
        importlib.import_module("clean_mean_inline_notebook")


# ---------------------------------------------------------------------
# 9. Environment record
# ---------------------------------------------------------------------
def test_09_notebook_records_its_own_environment():
    nb = nb_lib.execute_clean(nb_lib.environment_cell_notebook())
    recorded_text = nb_lib.final_value(nb, 0)

    live = nb_lib.record_environment()
    assert live["python_version"] == sys.version.split()[0]

    for key, value in live.items():
        assert repr(value) in recorded_text or value in recorded_text

    # Changing a pin changes the record: compare today's real record
    # against a stand-in for an older manifest that pinned nbformat one
    # minor version back. The two must disagree at exactly that key.
    older_manifest = dict(live)
    older_manifest["nbformat"] = "5.10.0"
    assert older_manifest != live
    assert older_manifest["nbformat"] != live["nbformat"]
    for key in ("python_version", "nbclient", "nbconvert"):
        assert older_manifest[key] == live[key]

lesson_id: D139
day: 139
kind: guided-build
languages:
  - python
  - bash
setup_commands:
  - cd labs/sections/math-statistics-and-data/day-139-reproducible-notebooks
  - python3 -m venv .venv
  - .venv/bin/pip install -r requirements/requirements.txt
  - >-
    .venv/bin/python3 -c "import nbformat, nbclient, nbconvert; print(nbformat.__version__,
    nbclient.__version__, nbconvert.__version__)"
run_commands:
  - .venv/bin/pytest examples -q
  - .venv/bin/pytest starter -q
test_commands:
  - bash tests/run_tests.sh
cleanup_commands:
  - >-
    find . -path ./.venv -prune -o -type d -name '__pycache__' -print -exec rm -rf -- {}
    +
  - rm -rf .pytest_cache
  - 'rm -rf .venv  # optional: removes the lab virtual environment'
  - 'git checkout -- starter/  # optional: reset your work'
requires_network: true
requires_api_key: false
estimated_minutes: 50
last_executed: '2026-08-20'
executed_on: >-
  macOS 26.5.2 (Apple Silicon, arm64), Python 3.14.0, nbformat 5.11.1, nbclient 0.11.0,
  nbconvert 7.17.1, ipykernel 7.3.0, pytest 9.1.1, bash 3.2.57 -- bash tests/run_tests.sh
  -> 16 checks, 0 failure(s), exit 0. pytest examples -q -> 12 passed. pytest starter -q
  -> 3 passed, 9 skipped (calc.py's tests are solved; the 9 notebook exercises are
  untouched stubs). Everything ran through a real lab-local .venv created by the
  documented setup commands, against real Jupyter kernels launched by nbclient on
  127.0.0.1 -- no notebook in this lab is ever written to disk; every one is built with
  nbformat, executed in memory, and inspected as JSON. Section 6 of the harness copies
  the solved examples/ suite into a scratch directory, confirms 12 passed, rewrites the
  assertion `assert clean_answer == "30.0"` to `assert clean_answer == "999.0"`,
  confirms a non-zero exit naming the failing test, restores the file and confirms 12
  passed again, so the harness is demonstrated able to fail rather than merely claimed
  to be. Separately, the same edit was made directly to examples/test_notebooks.py and
  the whole harness was re-run: it reported 16 checks, 2 failure(s) and exited 1; the
  file was restored via `git diff`-equivalent hand edit and the harness returned to 16
  checks, 0 failure(s), exit 0. Section 5 confirms directly that `pytest examples
  starter` in one invocation aborts collection with `import file mismatch` (both
  directories define test_calc.py and test_notebooks.py) rather than one suite silently
  shadowing the other. MEASURED RESULTS: a clean top-to-bottom run of the lesson's
  three-cell notebook answers 30.0 with execution_count [1, 2, 3]; the same notebook
  executed in the order an analyst who checked the report before adding a cleaning step
  actually used (cells 0, 2, 1) answers 50.0 with execution_count [1, 3, 2] -- visually
  a plain number, no error, non-monotonic execution_count as the only trace. Two
  independent executions of that same notebook are NOT byte-identical as nbformat JSON;
  the field that differs is exclusively cell.metadata.execution (four ISO-8601
  timestamps nbclient stamps on every run), not execution_count and not outputs data,
  which was verified directly rather than assumed. Stripping outputs, execution_count
  and metadata.execution makes the two runs byte-identical. nbconvert's MarkdownExporter
  carries both the prose sentence and the computed value 114 into the converted
  document, verified by substring match against real converted text. TWO HONESTY CALLS.
  FIRST: the day brief's illustrative execution order for the opening demonstration --
  "cell 3, then 1, then 3 again" -- was tried against this lab's actual three cells and
  raises NameError, because the third cell genuinely depends on a value only the first
  cell defines; running cell 3 before cell 1 cannot both succeed and differ from a clean
  run without either an undefined name or artificially pre-seeding the kernel from
  outside the notebook, which would blur this exercise into exercise 3's hidden-state
  mechanism instead of demonstrating pure out-of-order execution. The lesson and lab
  instead use order [0, 2, 1] -- setup, then a look at the report, then a correction --
  which is executed for real, produces a different non-error answer (50.0 vs 30.0), and
  is the same phenomenon the brief describes. See expected-output/FIELDS.md for the full
  account. SECOND: papermill is not installed and not run; the parameterisation exercise
  and lesson section reproduce papermill's documented parameter-injection mechanism
  (appending an "injected-parameters" cell after the tagged "parameters" cell, never
  editing the original) by hand with nbformat, and the lesson states plainly that no
  papermill output is reproduced. Jupyter Lab itself, Quarto and Google Colab are
  likewise described from public documentation only; no UI was driven and no output from
  any of the three is reproduced.

# Requirements

`requirements.txt` pins the exact versions this lab was written and run
against on 2026-08-20: `nbformat`, `nbclient`, `nbconvert`, `ipykernel`
and `pytest`. Everything else the lab uses — `copy`, `importlib`, `sys`
— is in the Python standard library.

This lab's whole point is executing real notebooks with a real Jupyter
kernel, so its own dependency stack is the largest of any lab in this
section. None of it is installed in the shared authoring virtual
environment for this course (see Day 133, where `pandas.DataFrame.style`
fails for exactly the same reason: nothing that pulls in `jinja2` is
installed centrally, and `nbconvert` depends on `jinja2`). Every command
below runs inside this lab's own `.venv`, which is the normal pattern.

```bash
cd labs/sections/math-statistics-and-data/day-139-reproducible-notebooks
python3 -m venv .venv
.venv/bin/pip install -r requirements/requirements.txt
```

Only that install step needs the network. Everything after it runs
offline: every notebook in this lab is built in memory, executed by a
kernel on `127.0.0.1` (ZeroMQ over loopback, the same transport Jupyter
always uses for a local kernel), and never written to disk.

## Why the pins are exact

- `nbclient` writes a wall-clock timestamp into `cell.metadata.execution`
  on every run. Exercise 5 depends on that behaviour existing at all — a
  much older `nbclient` before this metadata was added would make the
  exercise's central claim untestable.
- `nbformat` is the schema every assertion in this lab reads: cell
  `outputs`, `execution_count`, `metadata`. A schema version bump could
  change field names.
- `ipykernel` is the kernel every notebook in this lab actually runs on.
  Its version is recorded because exercise 9 (the environment record)
  quotes it directly.

## If a pin will not install

Recent `nbformat` 5.x, `nbclient` 0.10+ and `nbconvert` 7.x will almost
certainly run this lab unmodified. `ipykernel` just needs to be new
enough to register a `python3` kernel spec automatically on import,
which every version in the last several years does. If a test fails
after a version substitution, `expected-output/FIELDS.md` records which
captured values are exact everywhere and which are specific to the pins
above.

nbformat==5.11.1
nbclient==0.11.0
nbconvert==7.17.1
ipykernel==7.3.0
pytest==9.1.1

# Notebooks that reproduce — nine exercises

`nb_lib.py` and `calc.py` are complete and working: they are the
machinery, not the exercise. Your work is entirely in
`test_notebooks.py`, where each of the nine functions below is a
`pytest.skip("...")` naming exactly what to build and assert. Replace
each skip with real code and real assertions. `test_calc.py` is already
solved and passing — read it first, since it is exercise 8's other half.

Every notebook in this lab is built with `nbformat`, executed with a real
Jupyter kernel through `nbclient`, and inspected as JSON. Nothing here
opens a browser or a `.ipynb` file in an editor.

1. **Out-of-order changes the answer.** `nb_lib.analyst_notebook()` gives
   you a three-cell notebook: `setup` sets `x`, `transform` corrects it,
   `report` divides by two. Execute it top to bottom with
   `nb_lib.execute_clean`, then execute the *same* notebook in the order
   `[0, 2, 1]` with `nb_lib.execute_in_order` — the order an analyst uses
   when they check the report once, then add a cleaning step and never
   look again. Assert the two final answers (`nb_lib.final_value`,
   cell 2) differ, and report both.
2. **`execution_count` is the evidence.** From the same scrambled run,
   assert `nb_lib.execution_counts(...)` is non-monotonic
   (`not nb_lib.is_monotonic(...)`). From a clean run, assert the counts
   are `[1, 2, 3]` and monotonic.
3. **Hidden state.** `nb_lib.hidden_state_pair()` gives you two cells:
   one defines `helper_value`, the other uses it without defining it.
   `nb_lib.run_in_dirty_kernel_after_deletion(cell_defining, cell_using)`
   runs both, deletes the defining cell from the document, and reruns
   the remaining cell *in the same kernel process*. Assert that succeeds.
   Then run the resulting one-cell notebook in a brand-new kernel with
   `nb_lib.run_fresh_kernel` and assert it raises
   `nb_lib.CellExecutionError` with `.ename == "NameError"`.
4. **Execution as a test.** `nb_lib.failing_notebook()` has a cell that
   raises `ValueError`. Assert `nb_lib.execute_clean` raises
   `nb_lib.CellExecutionError`, that `.ename == "ValueError"`, and that
   the failing cell is named in the exception text (look for `"In[2]"`
   and the raising line in `str(error)`).
5. **Output stripping.** Execute `nb_lib.analyst_notebook()` twice with
   `nb_lib.execute_clean` (two independent runs). Assert the two
   unstripped notebooks (`nbformat.writes(...)`) are *not* equal. Assert
   that after `nb_lib.strip_outputs` on both, they *are* equal. Then
   inspect what actually differs between the unstripped pair — is it
   `execution_count`, or something else? Assert on the field name(s) you
   find, not on what you expect to find.
6. **Parameterisation.** Build two variants with
   `nb_lib.parameters_notebook(threshold=10)` and
   `nb_lib.parameters_notebook(threshold=5)`, execute both, and assert
   the final filtered lists (cell 3) differ as expected. Assert the
   non-parameter cells (0, 2, 3 — cell 0 is the untouched default) are
   character-identical text between variants, and only cell 1 (the
   injected-parameters cell) differs.
7. **Conversion.** Build a two-cell notebook: one markdown cell with a
   sentence of prose, one code cell that computes a number. Execute it,
   convert it with `nb_lib.to_markdown`, and assert both the prose
   sentence and the computed value appear in the resulting text.
8. **Notebook versus module.** Call `calc.clean_mean` directly and assert
   it works (it is already covered by `test_calc.py` — that is exercise
   8's point, not something to prove again here). Then build a one-cell
   notebook with the *same logic inlined*, execute it, and assert the
   inlined version also computes the right number inside the kernel.
   Finally assert that `importlib.import_module("some_name_for_that_cell")`
   raises `ModuleNotFoundError` — the module route is reachable by
   `pytest`; the inlined cell, structurally, is not.
9. **Environment record.** Execute `nb_lib.environment_cell_notebook()`
   and assert its recorded output contains the live
   `nb_lib.record_environment()` values (Python version, plus
   `nbformat`/`nbclient`/`nbconvert` versions). Then build a stand-in for
   an older pin manifest with one version changed, and assert it differs
   from the live record at exactly that key.

Run `.venv/bin/pytest starter -v` as you go; each skip message names the
one thing to assert next.

"""The one piece of logic in this lab that lives in a module, not a cell.

Day 139's rule: exploration belongs in the notebook, but anything the
rest of a project depends on belongs in an imported module that has its
own tests -- the notebook then becomes a caller, not the only place the
logic exists. This module is that split made concrete: ``clean_mean`` is
plain, reusable, deterministic logic, and ``test_calc.py`` tests it the
ordinary way, with no kernel and no notebook involved.
"""

from __future__ import annotations


def clean_mean(values: list) -> float:
    """Mean of ``values`` after dropping ``None`` entries.

    Raises ``ValueError`` if every value is ``None`` or the list is empty,
    because a mean of nothing is not a number this function will guess at.
    """
    kept = [v for v in values if v is not None]
    if not kept:
        raise ValueError("clean_mean: no non-None values to average")
    return sum(kept) / len(kept)

"""Shared helpers for building, executing and inspecting notebooks.

Every function here works on in-memory ``nbformat`` notebook objects.
Nothing in this module writes a notebook file to disk -- the tests build
notebooks in memory, execute them with real Jupyter kernels through
``nbclient``, and assert on the resulting JSON structure. That is how you
test a notebook without a UI.
"""

from __future__ import annotations

import copy
import sys
from typing import Iterable

import nbclient
import nbconvert
import nbformat as nbf
from nbclient import NotebookClient
from nbclient.exceptions import CellExecutionError


def analyst_notebook() -> nbf.NotebookNode:
    """Build the three-cell notebook the lesson opens with.

    Cell 0 ("setup") sets ``x``. Cell 1 ("transform") is a cleaning step
    an analyst adds after already having looked at the answer once. Cell
    2 ("report") divides ``x`` by two and displays it. Run top to bottom
    the answer is 30.0; run in the order an analyst really used --
    setup, then a quick look at the report, then the transform, without
    ever re-running the report -- the notebook still displays a number
    with no error, and that number is wrong.
    """
    nb = nbf.v4.new_notebook()
    nb.cells = [
        nbf.v4.new_code_cell("x = 100  # setup", id="setup"),
        nbf.v4.new_code_cell(
            "x = x - 40  # a cleaning step added after the first look at the report",
            id="transform",
        ),
        nbf.v4.new_code_cell("answer = x / 2\nanswer", id="report"),
    ]
    return nb


def execute_clean(nb: nbf.NotebookNode) -> nbf.NotebookNode:
    """Execute every cell top to bottom in a fresh kernel and return it."""
    nb = copy.deepcopy(nb)
    NotebookClient(nb, kernel_name="python3").execute()
    return nb


def execute_in_order(nb: nbf.NotebookNode, order: Iterable[int]) -> nbf.NotebookNode:
    """Execute the cells of ``nb`` in ``order`` (a sequence of cell indices).

    A single kernel is started once and reused across every cell in
    ``order``, exactly like clicking "Run" on cells in whatever sequence
    an analyst actually clicks them, rather than in document order.
    """
    nb = copy.deepcopy(nb)
    client = NotebookClient(nb, kernel_name="python3")
    with client.setup_kernel():
        for index in order:
            client.execute_cell(nb.cells[index], index)
    return nb


def final_value(nb: nbf.NotebookNode, cell_index: int) -> str:
    """Return the ``text/plain`` payload of a cell's last execute_result."""
    outputs = nb.cells[cell_index].get("outputs", [])
    for output in reversed(outputs):
        if output.get("output_type") == "execute_result":
            return output["data"]["text/plain"]
    raise AssertionError(f"cell {cell_index} has no execute_result output")


def execution_counts(nb: nbf.NotebookNode) -> list:
    return [cell.get("execution_count") for cell in nb.cells]


def is_monotonic(counts: list) -> bool:
    """True if every non-None count strictly increases with cell position."""
    seen = [c for c in counts if c is not None]
    return seen == sorted(seen) and len(seen) == len(set(seen))


def hidden_state_pair() -> tuple:
    """Build the two cells behind the hidden-state exercise.

    ``cell_defining`` sets a helper value. ``cell_using`` depends on it
    but does not define it. In the story, ``cell_defining`` is the cell an
    analyst deletes from the document once its job looks done -- while
    the kernel process, if it was never restarted, still remembers the
    value it set.
    """
    cell_defining = nbf.v4.new_code_cell(
        "helper_value = 42  # a cell that will be deleted from the document"
    )
    cell_using = nbf.v4.new_code_cell("total = helper_value + 8\ntotal")
    return cell_defining, cell_using


def run_in_dirty_kernel_after_deletion(cell_defining, cell_using) -> nbf.NotebookNode:
    """Run both cells, delete the defining cell from the document, rerun
    the remaining cell in the *same, still-alive* kernel, and return the
    notebook as it now stands on disk: one cell, whose variable the
    document itself never defines.
    """
    nb = nbf.v4.new_notebook()
    nb.cells = [copy.deepcopy(cell_defining), copy.deepcopy(cell_using)]
    client = NotebookClient(nb, kernel_name="python3")
    with client.setup_kernel():
        client.execute_cell(nb.cells[0], 0)
        client.execute_cell(nb.cells[1], 1)
        # Simulate deleting the defining cell from the document. The
        # kernel process behind `client` is untouched by this -- only the
        # notebook's list of cells changes.
        nb.cells = [nb.cells[1]]
        client.execute_cell(nb.cells[0], 0)
    return nb


def run_fresh_kernel(nb: nbf.NotebookNode) -> nbf.NotebookNode:
    """Execute ``nb`` (as it stands, with whatever cells it currently has)
    in a brand-new kernel that has no memory of any earlier session.
    """
    nb = copy.deepcopy(nb)
    NotebookClient(nb, kernel_name="python3").execute()
    return nb


def failing_notebook() -> nbf.NotebookNode:
    nb = nbf.v4.new_notebook()
    nb.cells = [
        nbf.v4.new_code_cell("row_count = 12"),
        nbf.v4.new_code_cell("raise ValueError('row_count below the expected minimum')"),
        nbf.v4.new_code_cell("row_count * 2"),
    ]
    return nb


def strip_outputs(nb: nbf.NotebookNode) -> nbf.NotebookNode:
    """Return a deep copy with outputs, execution_count and the
    per-execution timestamp metadata removed -- the same three fields an
    ``nbstripout``-style pre-commit hook clears before a notebook is
    committed.
    """
    nb = copy.deepcopy(nb)
    for cell in nb.cells:
        if cell.cell_type == "code":
            cell["outputs"] = []
            cell["execution_count"] = None
            cell["metadata"].pop("execution", None)
    return nb


def parameters_notebook(threshold: int) -> nbf.NotebookNode:
    """Build a notebook with a papermill-style parameters cell.

    Cell 0 is tagged ``parameters`` and carries the default. Cell 1 is
    the *injected-parameters* cell papermill adds immediately after it
    -- papermill never edits the parameters cell itself, it appends a
    new cell that overrides the default, so the original default stays
    visible in the document. Cells 2 and 3 are the unchanged analysis:
    they never mention ``threshold``'s value directly and are identical
    text across every variant.
    """
    nb = nbf.v4.new_notebook()
    default_cell = nbf.v4.new_code_cell("threshold = 10  # default")
    default_cell["metadata"]["tags"] = ["parameters"]
    injected_cell = nbf.v4.new_code_cell(f"# Parameters\nthreshold = {threshold}\n")
    injected_cell["metadata"]["tags"] = ["injected-parameters"]
    nb.cells = [
        default_cell,
        injected_cell,
        nbf.v4.new_code_cell("data = [3, 12, 7, 15, 2]"),
        nbf.v4.new_code_cell("filtered = [d for d in data if d > threshold]\nfiltered"),
    ]
    return nb


def to_markdown(nb: nbf.NotebookNode) -> str:
    """Convert an executed notebook to Markdown with nbconvert."""
    exporter = nbconvert.MarkdownExporter()
    body, _resources = exporter.from_notebook_node(nb)
    return body


def to_html(nb: nbf.NotebookNode) -> str:
    exporter = nbconvert.HTMLExporter()
    exporter.template_name = "basic"
    body, _resources = exporter.from_notebook_node(nb)
    return body


def to_script(nb: nbf.NotebookNode) -> str:
    exporter = nbconvert.PythonExporter()
    body, _resources = exporter.from_notebook_node(nb)
    return body


def record_environment() -> dict:
    """What a reproducible notebook should record about the kernel that
    ran it: the interpreter version and the exact versions of the
    packages the notebook stack itself depends on.
    """
    return {
        "python_version": sys.version.split()[0],
        "nbformat": nbf.__version__,
        "nbclient": nbclient.__version__,
        "nbconvert": nbconvert.__version__,
    }


def environment_cell_notebook() -> nbf.NotebookNode:
    """A one-cell notebook that records its own environment when run."""
    nb = nbf.v4.new_notebook()
    nb.cells = [
        nbf.v4.new_code_cell(
            "import sys, nbformat, nbclient, nbconvert\n"
            "record = {\n"
            "    'python_version': sys.version.split()[0],\n"
            "    'nbformat': nbformat.__version__,\n"
            "    'nbclient': nbclient.__version__,\n"
            "    'nbconvert': nbconvert.__version__,\n"
            "}\n"
            "record"
        )
    ]
    return nb


CellExecutionError = CellExecutionError  # re-exported for the tests

"""Ordinary tests for the module half of the notebook/module split.

No kernel, no notebook, no nbclient anywhere in this file -- which is
the point exercise 8 makes: this is what it looks like when reused logic
lives somewhere pytest can reach it directly.
"""

from calc import clean_mean


def test_clean_mean_drops_none():
    assert clean_mean([1, None, 3]) == 2.0


def test_clean_mean_all_present():
    assert clean_mean([2, 4, 6]) == 4.0


def test_clean_mean_raises_on_empty():
    try:
        clean_mean([None, None])
    except ValueError:
        pass
    else:
        raise AssertionError("expected ValueError for an all-None input")

"""Nine exercises in notebooks that reproduce.

Read `00_brief.md` first. Each function below is a `pytest.skip` naming
what to build and assert; replace the skip with real code. `nb_lib.py`
and `calc.py` are complete -- they are the machinery, not the exercise.
"""

import importlib
import sys

import nbformat as nbf
import pytest

import nb_lib
from nb_lib import CellExecutionError
from calc import clean_mean


def test_01_out_of_order_changes_the_answer():
    pytest.skip(
        "Execute nb_lib.analyst_notebook() clean (top to bottom) and in "
        "order [0, 2, 1]. Assert the two final answers (cell 2, via "
        "nb_lib.final_value) differ, and print both."
    )


def test_02_execution_count_is_the_evidence():
    pytest.skip(
        "From the scrambled run in exercise 1, assert "
        "nb_lib.execution_counts(...) is not monotonic. From the clean "
        "run, assert the counts are [1, 2, 3] and are monotonic."
    )


def test_03_hidden_state_survives_cell_deletion_in_a_dirty_kernel():
    pytest.skip(
        "Use nb_lib.hidden_state_pair() and "
        "nb_lib.run_in_dirty_kernel_after_deletion(...). Assert the "
        "one-cell result succeeds in the dirty kernel. Then run it "
        "through nb_lib.run_fresh_kernel and assert it raises "
        "CellExecutionError with .ename == 'NameError'."
    )


def test_04_nbclient_raises_on_a_failing_cell_and_names_it():
    pytest.skip(
        "Execute nb_lib.failing_notebook() and assert nb_lib.execute_clean "
        "raises CellExecutionError with .ename == 'ValueError', and that "
        "'In[2]' plus the raising line appear in str(error)."
    )


def test_05_stripping_outputs_makes_two_runs_identical():
    pytest.skip(
        "Execute nb_lib.analyst_notebook() twice independently. Assert "
        "the two unstripped notebooks (nbformat.writes) differ, and that "
        "nb_lib.strip_outputs on both makes them equal. Then find and "
        "assert exactly which field(s) differ between the unstripped pair."
    )


def test_06_parameterised_variants_differ_with_identical_code_cells():
    pytest.skip(
        "Build nb_lib.parameters_notebook(threshold=10) and threshold=5, "
        "execute both, and assert cell 3's filtered list differs as "
        "expected. Assert cells 0, 2 and 3 are identical text across "
        "variants and only cell 1 (injected-parameters) differs."
    )


def test_07_convert_to_markdown_carries_prose_and_computed_values():
    pytest.skip(
        "Build a two-cell notebook (one markdown cell of prose, one code "
        "cell computing a number), execute it, convert with "
        "nb_lib.to_markdown, and assert both the prose and the computed "
        "value appear in the result."
    )


def test_08_module_logic_is_covered_a_notebook_cell_is_not():
    pytest.skip(
        "Call calc.clean_mean directly and confirm it works (test_calc.py "
        "already covers it under pytest -- that is the point). Build a "
        "one-cell notebook with the same logic inlined, execute it, and "
        "assert it computes the right answer inside the kernel. Then "
        "assert importlib.import_module('some_name') raises "
        "ModuleNotFoundError for a name that only ever existed as a cell."
    )


def test_09_notebook_records_its_own_environment():
    pytest.skip(
        "Execute nb_lib.environment_cell_notebook() and assert its output "
        "contains the values from nb_lib.record_environment() (Python "
        "version, nbformat/nbclient/nbconvert versions). Build a stand-in "
        "for an older pin manifest with one version changed and assert it "
        "differs from the live record at exactly that key."
    )

#!/usr/bin/env bash
# Day 139 lab harness: "Notebooks That Reproduce"
#
# Prints "N checks, M failure(s)" and exits 0 only when M is zero.
set -u

LAB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$LAB_DIR"

PYTHON="${PYTHON:-.venv/bin/python3}"
PYTEST="${PYTEST:-.venv/bin/pytest}"

CHECKS=0
FAILURES=0

ok() {
  CHECKS=$((CHECKS + 1))
  echo "  ok: $1"
}

fail() {
  CHECKS=$((CHECKS + 1))
  FAILURES=$((FAILURES + 1))
  echo "  FAIL: $1"
}

if [ ! -x "$PYTHON" ]; then
  echo "No lab .venv found at $PYTHON."
  echo "Run: python3 -m venv .venv && .venv/bin/pip install -r requirements/requirements.txt"
  exit 2
fi

echo "1. Installed versions match requirements/requirements.txt"
VERSION_CHECK=$("$PYTHON" - <<'PYEOF'
import nbformat, nbclient, nbconvert, pytest, ipykernel
print("nbformat", nbformat.__version__)
print("nbclient", nbclient.__version__)
print("nbconvert", nbconvert.__version__)
print("ipykernel", ipykernel.__version__)
print("pytest", pytest.__version__)
PYEOF
)
echo "$VERSION_CHECK" | sed 's/^/    /'
while read -r pkg pin; do
  pin_version="${pin#*==}"
  installed=$(echo "$VERSION_CHECK" | awk -v p="$pkg" '$1==p {print $2}')
  if [ "$installed" = "$pin_version" ]; then
    ok "$pkg $installed matches the pin"
  else
    fail "$pkg installed=$installed pinned=$pin_version"
  fi
done < <(sed 's/==/ ==/' requirements/requirements.txt)

echo ""
echo "2. The library, driven directly (outside pytest)"
DIRECT_CHECK=$("$PYTHON" - <<'PYEOF'
import sys
sys.path.insert(0, "examples")
import nb_lib
import nbformat as nbf

errors = []

# Exercise 1/2: out-of-order changes the answer, execution_count is not monotonic
nb = nb_lib.analyst_notebook()
clean = nb_lib.execute_clean(nb)
scrambled = nb_lib.execute_in_order(nb, [0, 2, 1])
clean_answer = nb_lib.final_value(clean, 2)
scrambled_answer = nb_lib.final_value(scrambled, 2)
if clean_answer != "30.0":
    errors.append(f"clean answer expected 30.0, got {clean_answer}")
if scrambled_answer != "50.0":
    errors.append(f"scrambled answer expected 50.0, got {scrambled_answer}")
if nb_lib.is_monotonic(nb_lib.execution_counts(scrambled)):
    errors.append("scrambled execution_count unexpectedly monotonic")
if not nb_lib.is_monotonic(nb_lib.execution_counts(clean)):
    errors.append("clean execution_count unexpectedly non-monotonic")

# Exercise 3: hidden state
cell_defining, cell_using = nb_lib.hidden_state_pair()
dirty = nb_lib.run_in_dirty_kernel_after_deletion(cell_defining, cell_using)
if len(dirty.cells) != 1:
    errors.append("hidden-state notebook should have exactly one cell left")
try:
    nb_lib.run_fresh_kernel(dirty)
    errors.append("fresh kernel unexpectedly succeeded on hidden-state notebook")
except nb_lib.CellExecutionError as e:
    if e.ename != "NameError":
        errors.append(f"expected NameError, got {e.ename}")

# Exercise 4: execution as a test
try:
    nb_lib.execute_clean(nb_lib.failing_notebook())
    errors.append("failing_notebook unexpectedly executed without error")
except nb_lib.CellExecutionError as e:
    if e.ename != "ValueError" or "In[2]" not in str(e):
        errors.append("failing cell not correctly named in the exception")

# Exercise 5: stripping
run_a = nb_lib.execute_clean(nb_lib.analyst_notebook())
run_b = nb_lib.execute_clean(nb_lib.analyst_notebook())
if nbf.writes(run_a) == nbf.writes(run_b):
    errors.append("two independent runs were unexpectedly byte-identical unstripped")
if nbf.writes(nb_lib.strip_outputs(run_a)) != nbf.writes(nb_lib.strip_outputs(run_b)):
    errors.append("stripped runs were not byte-identical")

# Exercise 6: parameterisation
strict = nb_lib.execute_clean(nb_lib.parameters_notebook(threshold=10))
loose = nb_lib.execute_clean(nb_lib.parameters_notebook(threshold=5))
if nb_lib.final_value(strict, 3) == nb_lib.final_value(loose, 3):
    errors.append("parameterised variants unexpectedly produced the same output")
if strict.cells[2].source != loose.cells[2].source:
    errors.append("unrelated analysis cell text changed between variants")

# Exercise 7: conversion
nb7 = nbf.v4.new_notebook()
nb7.cells = [
    nbf.v4.new_markdown_cell("A prose sentence with a claim in it."),
    nbf.v4.new_code_cell("21 * 2"),
]
executed7 = nb_lib.execute_clean(nb7)
md = nb_lib.to_markdown(executed7)
if "A prose sentence with a claim in it." not in md or "42" not in md:
    errors.append("nbconvert Markdown output missing prose or computed value")

# Exercise 8: module vs cell
from calc import clean_mean
if clean_mean([1, None, 3]) != 2.0:
    errors.append("calc.clean_mean gave the wrong answer")

# Exercise 9: environment record
env_nb = nb_lib.execute_clean(nb_lib.environment_cell_notebook())
recorded = nb_lib.final_value(env_nb, 0)
live = nb_lib.record_environment()
for key, value in live.items():
    if repr(value) not in recorded and value not in recorded:
        errors.append(f"environment record missing {key}={value}")

if errors:
    for e in errors:
        print("ERROR:", e)
    sys.exit(1)
print("all direct checks passed")
PYEOF
)
if echo "$DIRECT_CHECK" | grep -q "all direct checks passed"; then
  ok "exercises 1-9 reproduced directly against nb_lib, no pytest involved"
else
  fail "direct library checks failed"
  echo "$DIRECT_CHECK" | sed 's/^/    /'
fi

echo ""
echo "3. examples/ passes in full"
EXAMPLES_OUT=$("$PYTEST" examples -q 2>&1)
if echo "$EXAMPLES_OUT" | tail -1 | grep -qE "^12 passed"; then
  ok "pytest examples -q -> 12 passed"
else
  fail "pytest examples -q did not report 12 passed"
  echo "$EXAMPLES_OUT" | tail -20 | sed 's/^/    /'
fi

echo ""
echo "4. starter/ is an untouched skeleton"
STARTER_OUT=$("$PYTEST" starter -q 2>&1)
if echo "$STARTER_OUT" | tail -1 | grep -qE "3 passed, 9 skipped"; then
  ok "pytest starter -q -> 3 passed, 9 skipped (calc.py solved; the 9 notebook exercises are stubs)"
else
  fail "pytest starter -q did not report 3 passed, 9 skipped"
  echo "$STARTER_OUT" | tail -20 | sed 's/^/    /'
fi

echo ""
echo "5. pytest examples starter (one invocation) aborts on the module-name collision"
COMBINED_OUT=$("$PYTEST" examples starter 2>&1)
if echo "$COMBINED_OUT" | grep -q "import file mismatch"; then
  ok "combined invocation reports import file mismatch, as documented -- never run starter and examples together"
else
  fail "combined invocation did not fail with import file mismatch as expected"
fi

echo ""
echo "6. Proof the harness can fail"
SCRATCH=$(mktemp -d "${TMPDIR:-/tmp}/d139-scratch.XXXXXX")
cp examples/*.py "$SCRATCH"/
SCRATCH_OUT=$("$PYTEST" "$SCRATCH" -q 2>&1)
if echo "$SCRATCH_OUT" | tail -1 | grep -qE "^12 passed"; then
  ok "scratch copy of examples/ passes before it is broken"
else
  fail "scratch copy did not pass before being broken: $(echo "$SCRATCH_OUT" | tail -3)"
fi
python3 - "$SCRATCH/test_notebooks.py" <<'PYEOF'
import sys
path = sys.argv[1]
text = open(path).read()
needle = 'assert clean_answer == "30.0"'
replacement = 'assert clean_answer == "999.0"'
assert needle in text, "could not find the assertion to break"
open(path, "w").write(text.replace(needle, replacement, 1))
PYEOF
BROKEN_OUT=$("$PYTEST" "$SCRATCH" -q 2>&1)
BROKEN_STATUS=$?
if [ "$BROKEN_STATUS" -ne 0 ] && echo "$BROKEN_OUT" | grep -q "test_01_out_of_order_changes_the_answer"; then
  ok "breaking exercise 1's assertion produces a non-zero exit and names the failing test"
else
  fail "broken copy did not fail as expected (exit=$BROKEN_STATUS)"
fi
rm -rf "$SCRATCH"

echo ""
echo "7. Offline, and nothing left behind"
if ! grep -rInE "https?://" examples/*.py starter/*.py > /dev/null 2>&1; then
  ok "no URLs inside examples/ or starter/ source"
else
  fail "found a URL inside examples/ or starter/"
fi
if [ -z "$(find . -path ./.venv -prune -o -iname '*.ipynb' -print 2>/dev/null)" ]; then
  ok "no .ipynb file anywhere inside the lab -- every notebook in this lab exists only in memory"
else
  fail "found a stray .ipynb file"
fi
if [ -z "$(find . -path ./.venv -prune -o -type d -iname '.ipynb_checkpoints' -print 2>/dev/null)" ]; then
  ok "no .ipynb_checkpoints directory anywhere inside the lab"
else
  fail "found a stray .ipynb_checkpoints directory"
fi
if [ -z "$(find . -path ./.venv -prune -o -type d -name '__pycache__' -print 2>/dev/null)" ]; then
  ok "no __pycache__ left behind"
else
  find . -path ./.venv -prune -o -type d -name '__pycache__' -exec rm -rf {} + 2>/dev/null
  ok "no __pycache__ left behind (cleaned during this run)"
fi
if [ ! -d .pytest_cache ]; then
  ok "no .pytest_cache left behind"
else
  rm -rf .pytest_cache
  ok "no .pytest_cache left behind (cleaned during this run)"
fi

echo ""
echo "---------------------------------------------------------------"
echo "$CHECKS checks, $FAILURES failure(s)"
if [ "$FAILURES" -ne 0 ]; then
  exit 1
fi
exit 0