Hands-on lab — Day 120: pandas: Series and DataFrames

"""Exercise 1 -- build a Series and a DataFrame three different ways.

Run: python3 01_three_ways_to_build.py

A Series is values plus an index. A DataFrame is a set of Series that share
one index. This script builds both from a dict, from a list of records, and
from a bare NumPy array with an explicit index supplied separately -- and
checks, rather than assumes, what the index and the dtypes become in each
case.
"""

import numpy as np
import pandas as pd

checks = 0
failures = 0


def check(label, condition):
    global checks, failures
    checks += 1
    if condition:
        print(f"  ok: {label}")
    else:
        print(f"  FAIL: {label}")
        failures += 1


print(f"pandas {pd.__version__}, numpy {np.__version__}")
print()

# -- Series from a dict: the dict's keys become the index, in insertion order
print("-- Series from a dict --")
s_dict = pd.Series({"a": 10, "b": 20, "c": 30})
print(s_dict)
check("index becomes the dict's keys", list(s_dict.index) == ["a", "b", "c"])
check("dtype is inferred as int64 from all-int values", s_dict.dtype == np.dtype("int64"))

# -- DataFrame from a dict of lists: keys become column names, index is the
#    default RangeIndex 0..n-1 because nothing said otherwise
print("\n-- DataFrame from a dict of lists --")
df_dict = pd.DataFrame({"x": [1, 2, 3], "y": [4.0, 5.0, 6.0]})
print(df_dict)
print(df_dict.dtypes)
check("columns are the dict's keys, in order", list(df_dict.columns) == ["x", "y"])
check("index defaults to RangeIndex(0, 3)", list(df_dict.index) == [0, 1, 2])
check("column x is int64 (all ints)", df_dict["x"].dtype == np.dtype("int64"))
check("column y is float64 (all floats)", df_dict["y"].dtype == np.dtype("float64"))

# -- DataFrame from a list of records (one dict per row): same result as
#    above, because a "record" is just a row-oriented way of writing the
#    same table -- pandas transposes it for you
print("\n-- DataFrame from a list of records --")
records = [{"x": 1, "y": 4.0}, {"x": 2, "y": 5.0}, {"x": 3, "y": 6.0}]
df_records = pd.DataFrame(records)
print(df_records)
check("records give the same columns", list(df_records.columns) == ["x", "y"])
check("records give the same index", list(df_records.index) == [0, 1, 2])
check("records and dict-of-lists produce an identical frame", df_records.equals(df_dict))

# -- DataFrame from a bare NumPy array: the array carries NO labels at all --
#    you must supply both the index and the columns yourself, or pandas
#    falls back to the same default RangeIndex on both axes
print("\n-- DataFrame from a NumPy array, explicit index --")
arr = np.array([[1, 2], [3, 4], [5, 6]])
df_arr = pd.DataFrame(arr, index=["p", "q", "r"], columns=["c1", "c2"])
print(df_arr)
print(df_arr.dtypes)
check("explicit index is used verbatim", list(df_arr.index) == ["p", "q", "r"])
check("explicit columns are used verbatim", list(df_arr.columns) == ["c1", "c2"])
check("a plain int array gives int64 columns", (df_arr.dtypes == np.dtype("int64")).all())

df_arr_default = pd.DataFrame(arr)
check(
    "without an explicit index, a NumPy array also falls back to RangeIndex",
    list(df_arr_default.index) == [0, 1, 2] and list(df_arr_default.columns) == [0, 1],
)

print(f"\n{checks} checks, {failures} failure(s).")
if failures:
    raise SystemExit(1)
print("01_three_ways_to_build.py: every assertion held.")

"""Exercise 2 -- index alignment, the opening failure of this whole day.

Run: python3 02_alignment.py

Adding two Series does NOT add them position by position. pandas lines the
two indexes up by LABEL first, and any label that exists on only one side
produces NaN. This is the single most common source of a silently corrupted
feature column in a real pipeline: two Series built from different filters,
joins or sorts end up with different row orders, and '+' still "succeeds" --
it just answers a different question than the one you meant to ask.
"""

import numpy as np
import pandas as pd

checks = 0
failures = 0


def check(label, condition):
    global checks, failures
    checks += 1
    if condition:
        print(f"  ok: {label}")
    else:
        print(f"  FAIL: {label}")
        failures += 1


x = pd.Series([1, 2, 3], index=["a", "b", "c"])
y = pd.Series([10, 20, 30], index=["b", "c", "d"])
print("x:")
print(x)
print("y:")
print(y)

z = x + y
print("\nx + y (label-aligned):")
print(z)

expected_nan_labels = {"a", "d"}
actual_nan_labels = set(z.index[z.isna()])
check("labels present on only one side become NaN: a and d", actual_nan_labels == expected_nan_labels)
check("label b, present on both sides, sums to 2 + 10 = 12", z["b"] == 12.0)
check("label c, present on both sides, sums to 3 + 20 = 23", z["c"] == 23.0)
check("alignment promotes the result to float64 (NaN is a float)", z.dtype == np.dtype("float64"))

# Opting out: .to_numpy() (or .values) drops the labels and adds by position.
z_positional = x.to_numpy() + y.to_numpy()
print("\nx.to_numpy() + y.to_numpy() (positional, labels discarded):")
print(z_positional)
check(
    "positional addition on the raw arrays gives 1+10, 2+20, 3+30",
    list(z_positional) == [11, 22, 33],
)
check(
    "the positional answer and the aligned answer disagree at every position",
    not np.array_equal(z_positional, z.dropna().to_numpy()),
)

# reset_index(drop=True) is the DataFrame-shaped version of the same opt-out:
# it discards the current labels and replaces them with a fresh RangeIndex,
# so alignment then happens positionally because the labels now agree by
# construction.
x_reset = x.reset_index(drop=True)
y_reset = y.reset_index(drop=True)
z_reset = x_reset + y_reset
print("\nx.reset_index(drop=True) + y.reset_index(drop=True):")
print(z_reset)
check(
    "reset_index(drop=True) reproduces the positional sum with no NaN",
    list(z_reset) == [11, 22, 33] and not z_reset.isna().any(),
)

print(f"\n{checks} checks, {failures} failure(s).")
if failures:
    raise SystemExit(1)
print("02_alignment.py: every assertion held.")

"""Exercise 3 -- the moment a NaN enters an int64 column.

Run: python3 03_dtype_promotion.py

NumPy's int64 has no bit pattern reserved for "missing". The instant a
missing value enters an int64 Series, pandas silently promotes the WHOLE
column to float64, because float64 has NaN available. That is how an ID
column -- something you never intended to do arithmetic on -- quietly loses
exact-integer precision the moment one row is missing. The nullable Int64
dtype (capital I) exists specifically to avoid this: it carries its own
missing marker (pd.NA) without leaving the integer family.
"""

import numpy as np
import pandas as pd

checks = 0
failures = 0


def check(label, condition):
    global checks, failures
    checks += 1
    if condition:
        print(f"  ok: {label}")
    else:
        print(f"  FAIL: {label}")
        failures += 1


ids = pd.Series([1001, 1002, 1003], dtype="int64")
print("a clean int64 column of IDs:")
print(ids)
check("a clean column of whole numbers is int64", ids.dtype == np.dtype("int64"))

# reindex() is what a real pipeline does when a join fails to find a match
# for one label: it asks the Series for a row that was not there, and
# pandas fills the gap with NaN. NumPy's int64 array has no bit pattern
# reserved for "missing", so the ENTIRE column is silently rebuilt as
# float64 -- not just the new row -- so that NaN has somewhere to live.
ids_with_gap = ids.reindex([0, 1, 2, 3])
print("\nafter reindexing onto a label that was never there (a join miss):")
print(ids_with_gap)
check(
    "the ENTIRE column is promoted to float64, not just the missing row",
    ids_with_gap.dtype == np.dtype("float64"),
)
check("the missing row reads as NaN", pd.isna(ids_with_gap.iloc[3]))

# The precision loss this enables: an ID large enough to exceed float64's
# 53-bit exact-integer mantissa silently rounds once it is forced to share a
# column with a float.
big_id = 2**53 + 1  # the first integer float64 cannot represent exactly
mixed = pd.Series([big_id], dtype="int64").reindex([0, 1])
recovered = int(mixed.iloc[0])
print(f"\noriginal big ID:  {big_id}")
print(f"after promotion:  {recovered}")
check(
    "an ID past 2**53 loses its exact value once promoted to float64",
    recovered != big_id,
)

# The nullable Int64 dtype avoids all of this: it stays in the integer
# family and represents the missing entry as pd.NA instead of NaN, even
# after the same reindex onto a label that was never there.
ids_nullable = pd.Series([1001, 1002, 1003], dtype="Int64").reindex([0, 1, 2, 3])
print("\nthe same reindex, but the column is declared nullable Int64:")
print(ids_nullable)
check("a nullable Int64 column stays Int64 after the same reindex", ids_nullable.dtype == "Int64")
check("the missing entry is pd.NA, not NaN", ids_nullable.iloc[3] is pd.NA)
check(
    "the surviving values keep exact integer precision under Int64",
    int(ids_nullable.iloc[0]) == 1001,
)

big_id_nullable = pd.Series([big_id], dtype="Int64").reindex([0, 1])
check(
    "even a value past 2**53 keeps its exact integer value under Int64",
    int(big_id_nullable.iloc[0]) == big_id,
)

print(f"\n{checks} checks, {failures} failure(s).")
if failures:
    raise SystemExit(1)
print("03_dtype_promotion.py: every assertion held.")

"""Exercise 4 -- Copy-on-Write and chained assignment. This is the day's
most important check.

Run: python3 04_copy_on_write.py

pandas 3.0 has Copy-on-Write ALWAYS on; it can no longer be switched off.
One direct consequence: chained assignment -- indexing twice in one
statement, `df[mask]['col'] = value` -- silently does nothing. The first
`df[mask]` produces a temporary DataFrame; the second `['col'] = value`
assigns into that temporary, which is then discarded. The original `df` is
completely unchanged, and on 3.0.5 pandas raises a ChainedAssignmentError
*warning* (not an exception -- the statement still "succeeds" and moves on)
telling you exactly this. Reader beware: that warning is easy to miss if
warnings are filtered or redirected, which is exactly the situation every
tutorial written for pandas < 2.0 describes as silent and undetectable.
The fix is a single `.loc` call that does the selection and the assignment
in one step, which is the only form Copy-on-Write actually allows to work.
"""

import warnings

import pandas as pd

checks = 0
failures = 0


def check(label, condition):
    global checks, failures
    checks += 1
    if condition:
        print(f"  ok: {label}")
    else:
        print(f"  FAIL: {label}")
        failures += 1


print(f"pandas {pd.__version__}")

df = pd.DataFrame({"a": [1, 2, 3], "b": [10, 20, 30]})
original_b = df["b"].tolist()
print("\noriginal frame:")
print(df)
print(f"df['b'] before: {original_b}")

# The chained-assignment form: two lookups in one statement. Capture the
# warning pandas 3.0.5 raises about it, rather than letting it print to
# stderr, so this script's own output stays clean -- but the READER should
# see that warning by default; it is not suppressed anywhere else in this
# lab.
with warnings.catch_warnings(record=True) as caught:
    warnings.simplefilter("always")
    df[df["a"] > 1]["b"] = 0  # chained assignment -- looks like it should work
    chained_warning_names = [w.category.__name__ for w in caught]

after_chained = df["b"].tolist()
print(f"\ndf['b'] after chained assignment `df[df['a'] > 1]['b'] = 0`: {after_chained}")
print(f"warning(s) raised by that statement: {chained_warning_names}")

check(
    "chained assignment leaves the original frame COMPLETELY unchanged",
    after_chained == original_b == [10, 20, 30],
)
check(
    "pandas 3.0.5 warns about it with ChainedAssignmentError (a Warning, not raised as an exception)",
    "ChainedAssignmentError" in chained_warning_names,
)

# The fix: one .loc call carrying both the row selector and the column
# selector, so there is only ever one object involved -- no temporary to
# lose the write to.
df.loc[df["a"] > 1, "b"] = 0
after_loc = df["b"].tolist()
print(f"\ndf['b'] after `.loc[df['a'] > 1, 'b'] = 0`: {after_loc}")
check("the .loc form DOES change the original frame", after_loc == [10, 0, 0])
check("the .loc form does not equal the untouched original", after_loc != original_b)

# The deprecated switch: pandas 3.0 removed the ability to turn Copy-on-Write
# off. Setting the old option is now a no-op that only warns.
print("\nsetting the old pd.options.mode.copy_on_write switch:")
with warnings.catch_warnings(record=True) as caught2:
    warnings.simplefilter("always")
    pd.options.mode.copy_on_write = False
    option_warning_messages = [str(w.message) for w in caught2]
for msg in option_warning_messages:
    print(f"  warning: {msg}")
check(
    "setting mode.copy_on_write now only emits a deprecation warning and has no effect",
    any("no impact" in msg or "no longer be disabled" in msg for msg in option_warning_messages),
)

print(f"\n{checks} checks, {failures} failure(s).")
if failures:
    raise SystemExit(1)
print("04_copy_on_write.py: every assertion held.")

"""Exercise 5 -- .loc is label-based and INCLUSIVE of the stop; .iloc is
positional and EXCLUSIVE of the stop.

Run: python3 05_loc_vs_iloc.py

Putting `.loc['b':'d']` and `.iloc[1:4]` side by side on a 5-row frame
indexed a..e looks, at first glance, like the two behave the same way --
both return b, c, d. That similarity is what makes the asymmetry dangerous:
it hides until you write the stop value the "obvious" way and get a
different answer. The sharp version of the rule: to get the SAME rows out
of .loc and .iloc you must write a DIFFERENT stop value, because .loc's
stop is the label to include and .iloc's stop is the position to stop
before. Change only the number -- .iloc[1:3] instead of .iloc[1:4] -- and
one row silently disappears, even though 3 is the position of the label
'd' that .loc happily included.
"""

import pandas as pd

checks = 0
failures = 0


def check(label, condition):
    global checks, failures
    checks += 1
    if condition:
        print(f"  ok: {label}")
    else:
        print(f"  FAIL: {label}")
        failures += 1


df = pd.DataFrame({"val": [10, 20, 30, 40, 50]}, index=["a", "b", "c", "d", "e"])
print("frame, index a..e:")
print(df)

by_label = df.loc["b":"d"]
print("\ndf.loc['b':'d']  (label-based, stop 'd' INCLUDED):")
print(by_label)

by_position_4 = df.iloc[1:4]
print("\ndf.iloc[1:4]  (positional, stop position 4 EXCLUDED -- but position 4 is 'e', so 'd' still shows):")
print(by_position_4)

check(
    "with a DIFFERENT stop value (1:4, not 1:3), .loc['b':'d'] and .iloc[1:4] return the same three rows",
    by_label.equals(by_position_4),
)
check("both forms include the label 'd'", "d" in by_label.index and "d" in by_position_4.index)

# The sharp version of the rule: 'd' sits at position 3. Writing .iloc[1:3]
# -- the "matching number" a reader expects after seeing .loc['b':'d'] --
# EXCLUDES position 3, so 'd' is silently dropped and only two rows survive.
by_position_3 = df.iloc[1:3]
print("\ndf.iloc[1:3]  (stop position 3 EXCLUDED -- position 3 IS 'd', so 'd' is now dropped):")
print(by_position_3)

check(
    ".iloc[1:3] is SHORTER than .loc['b':'d'] -- one row missing -- even though 3 is 'd''s own position",
    len(by_position_3) == len(by_label) - 1,
)
check("'d' is present in the .loc result", "d" in by_label.index)
check("'d' is ABSENT from .iloc[1:3], which stops before position 3", "d" not in by_position_3.index)
check(
    "the row count differs: 3 labels from .loc, 2 rows from .iloc[1:3]",
    (len(by_label), len(by_position_3)) == (3, 2),
)

print(f"\n{checks} checks, {failures} failure(s).")
if failures:
    raise SystemExit(1)
print("05_loc_vs_iloc.py: every assertion held.")

"""Exercise 6 -- NaN is not equal to anything, including itself.

Run: python3 06_nan_semantics.py

NaN ("not a number") follows IEEE 754: by definition, NaN != NaN, and every
comparison against NaN using ==, <, >, <=, >= is False -- never an error,
never True. That is exactly why pandas gives you .isna() as a dedicated
test rather than expecting `series == float('nan')` to work: the second
form always returns an all-False mask, silently finding nothing, no matter
how many missing values are actually present.
"""

import numpy as np
import pandas as pd

checks = 0
failures = 0


def check(label, condition):
    global checks, failures
    checks += 1
    if condition:
        print(f"  ok: {label}")
    else:
        print(f"  FAIL: {label}")
        failures += 1


print(f"pandas {pd.__version__}, numpy {np.__version__}")

# The raw Python/IEEE-754 fact underneath everything else in this exercise.
raw_nan = float("nan")
print(f"\nfloat('nan') != float('nan'): {raw_nan != raw_nan}")
check("a bare NaN is never equal to itself", raw_nan != raw_nan)
check("a bare NaN is also never LESS than itself (not just !=)", not (raw_nan < raw_nan))

s = pd.Series([1.0, np.nan, 3.0])
print("\nSeries with a NaN:")
print(s)

isna_mask = s.isna()
print(f"s.isna(): {isna_mask.tolist()}")
check(".isna() correctly finds the NaN at position 1", isna_mask.tolist() == [False, True, False])

eq_nan_mask = s == np.nan
print(f"s == np.nan: {eq_nan_mask.tolist()}")
check(
    "comparing == np.nan is USELESS for finding missing values -- always all False",
    eq_nan_mask.tolist() == [False, False, False],
)
check(
    "== np.nan never finds the NaN, even at the position where it actually is",
    eq_nan_mask.iloc[1] == False,
)

# None behaves differently depending on the surrounding dtype: in a numeric
# column it is converted to NaN on entry; in a string column (the new pandas
# 3.0 default `str` dtype) it is also reported as missing by isna(), even
# though the underlying stored value differs from a numeric column's NaN.
numeric_with_none = pd.Series([1, None, 3])
print(f"\nnumeric column built with None: dtype={numeric_with_none.dtype}, values={numeric_with_none.tolist()}")
check("None inserted into a numeric Series becomes float64 NaN", numeric_with_none.dtype == np.dtype("float64"))
check("isna() finds it there too", numeric_with_none.isna().tolist() == [False, True, False])

string_with_none = pd.Series(["a", None, "c"])
print(f"string column built with None: dtype={string_with_none.dtype}, isna={string_with_none.isna().tolist()}")
check("isna() finds None in a string-dtype column just as reliably", string_with_none.isna().tolist() == [False, True, False])

print(f"\n{checks} checks, {failures} failure(s).")
if failures:
    raise SystemExit(1)
print("06_nan_semantics.py: every assertion held.")

"""Exercise 7 -- vectorised arithmetic against .apply with a lambda, on the
same column.

Run: python3 07_vectorized_vs_apply.py

`.apply(lambda x: ...)` calls a real Python function once per row, from
Python. `series * 1.08` never leaves compiled code (Day 104's lesson on
NumPy). This script measures both on an identical column and reports the
result as a RATIO and a SHAPE -- "at least N times faster, on M rows" --
never a millisecond figure, because a millisecond figure is a property of
this one machine on this one day and would be misleading reported any more
precisely than that.
"""

import time

import numpy as np
import pandas as pd

checks = 0
failures = 0


def check(label, condition):
    global checks, failures
    checks += 1
    if condition:
        print(f"  ok: {label}")
    else:
        print(f"  FAIL: {label}")
        failures += 1


print(f"pandas {pd.__version__}, numpy {np.__version__}")

rng = np.random.default_rng(42)
n = 200_000
df = pd.DataFrame({"price": rng.uniform(1, 1000, n)})
print(f"\ncolumn shape: {df.shape}")


def vectorized():
    return df["price"] * 1.08


def apply_lambda():
    return df["price"].apply(lambda x: x * 1.08)


# One untimed call each to warm up caches before the timed runs.
vectorized()
apply_lambda()

reps = 5
t0 = time.perf_counter()
for _ in range(reps):
    result_vectorized = vectorized()
vectorized_time = (time.perf_counter() - t0) / reps

t0 = time.perf_counter()
for _ in range(reps):
    result_apply = apply_lambda()
apply_time = (time.perf_counter() - t0) / reps

ratio = apply_time / vectorized_time
print(f"apply / vectorized time ratio, averaged over {reps} runs: {ratio:.1f}x")
print("(one machine, one day -- this ratio is a shape, not a promise)")

check(
    "the two approaches compute the identical result",
    np.allclose(result_vectorized.to_numpy(), result_apply.to_numpy()),
)
check(
    f"vectorised arithmetic is at least 20x faster than .apply on {n:,} rows (measured {ratio:.1f}x)",
    ratio >= 20,
)

print(f"\n{checks} checks, {failures} failure(s).")
if failures:
    raise SystemExit(1)
print("07_vectorized_vs_apply.py: every assertion held.")

"""Exercise 8 -- the pandas 3.0 string dtype default.

Run: python3 08_string_dtype.py

Every pre-3.0 pandas tutorial says a Series of strings has dtype `object`
-- a column of pointers to arbitrary Python objects, no faster and no more
memory-efficient than a Python list. As of pandas 3.0, built on PyArrow,
the DEFAULT dtype for string data is `str`, a dedicated string dtype backed
by PyArrow's contiguous string arrays. `object` still exists and is still
what you get for genuinely mixed-type columns, but a column that is
actually just text no longer pays the `object` tax by default.
"""

import pandas as pd

checks = 0
failures = 0


def check(label, condition):
    global checks, failures
    checks += 1
    if condition:
        print(f"  ok: {label}")
    else:
        print(f"  FAIL: {label}")
        failures += 1


print(f"pandas {pd.__version__}")

s = pd.Series(["a", "b"])
print(f"\npd.Series(['a', 'b']).dtype  ->  {s.dtype}")
print("(pandas < 2.x and most tutorials say this is `object` -- it is not, here)")

check("the default dtype for a plain string Series is 'str' on pandas 3.0.5", str(s.dtype) == "str")
check("it is explicitly NOT the old object dtype", str(s.dtype) != "object")

# object is still available -- and still what mixed-type data gets.
mixed = pd.Series(["a", 1, 3.5])
print(f"\npd.Series(['a', 1, 3.5]).dtype  ->  {mixed.dtype}  (genuinely mixed types)")
check("a genuinely mixed-type column still falls back to object", mixed.dtype == "object")

# You can still ask for object explicitly if you need the old behaviour.
forced_object = pd.Series(["a", "b"], dtype="object")
print(f"pd.Series(['a', 'b'], dtype='object').dtype  ->  {forced_object.dtype}")
check("object remains available on request, it is just no longer the default", forced_object.dtype == "object")

print(f"\n{checks} checks, {failures} failure(s).")
if failures:
    raise SystemExit(1)
print("08_string_dtype.py: every assertion held.")

"""Exercise 9 -- .describe(), .info(), .head() and memory_usage(deep=True)
on a column with hand-computable values, tying back to Day 116.

Run: python3 09_describe_known_column.py

These four commands are what you run on any frame you have not met before.
`.describe()` computes exactly the summary statistics Day 116 taught by
hand -- count, mean, standard deviation, min, the quartiles, max -- so this
script checks its output against arithmetic done independently of pandas,
the same discipline Day 116 insisted on: never trust a reported number
without knowing what produced it.
"""

import pandas as pd

checks = 0
failures = 0


def check(label, condition):
    global checks, failures
    checks += 1
    if condition:
        print(f"  ok: {label}")
    else:
        print(f"  FAIL: {label}")
        failures += 1


print(f"pandas {pd.__version__}")

values = [2, 4, 4, 4, 5, 5, 7, 9]
scores = pd.Series(values, name="score")
print(f"\nscores: {values}")

desc = scores.describe()
print("\n.describe():")
print(desc)

hand_count = len(values)
hand_mean = sum(values) / len(values)
hand_min = min(values)
hand_max = max(values)

print(f"\nhand-computed count={hand_count}, mean={hand_mean}, min={hand_min}, max={hand_max}")

check("describe()'s count matches len(values) exactly", desc["count"] == hand_count)
check("describe()'s mean matches sum(values)/len(values) exactly", desc["mean"] == hand_mean)
check("describe()'s min matches min(values) exactly", desc["min"] == hand_min)
check("describe()'s max matches max(values) exactly", desc["max"] == hand_max)

# Day 116's Bessel-corrected sample standard deviation: divide the sum of
# squared deviations by n - 1, not n, then take the square root.
mean = hand_mean
sq_dev = sum((v - mean) ** 2 for v in values)
hand_std = (sq_dev / (hand_count - 1)) ** 0.5
print(f"hand-computed sample std (n-1 denominator, Day 116's Bessel correction): {hand_std:.6f}")
check("describe()'s std uses the same n-1 (Bessel-corrected) denominator as Day 116", abs(desc["std"] - hand_std) < 1e-9)

# A small DataFrame to exercise .head(), .info() and memory_usage(deep=True).
df = pd.DataFrame({"score": values, "grade": ["F", "D", "D", "D", "C", "C", "B", "A"]})
print("\n.head(3):")
print(df.head(3))
check(".head(3) returns exactly 3 rows", len(df.head(3)) == 3)
check(".head(3) returns the FIRST 3 rows, in order", df.head(3)["score"].tolist() == values[:3])

print("\n.info():")
df.info()

usage_shallow = df.memory_usage(deep=False)
usage_deep = df.memory_usage(deep=True)
print("\n.memory_usage(deep=False):")
print(usage_shallow)
print(".memory_usage(deep=True):")
print(usage_deep)
check("memory_usage(deep=True) reports a positive byte count for the string column", usage_deep["grade"] > 0)

# A version-specific surprise worth recording: because the pandas-3.0 `str`
# dtype already stores its bytes contiguously (PyArrow-backed) rather than
# as pointers to scattered Python objects, deep=True and deep=False report
# the SAME number for a str column -- there is no hidden pointer indirection
# left for "deep" to go and discover. That is new in 3.0; the object dtype,
# still reachable on request, is the one where "deep" used to matter.
check(
    "for the pandas-3.0 str dtype, deep=True and deep=False report the SAME byte count (no pointer indirection left to find)",
    usage_deep["grade"] == usage_shallow["grade"],
)

df_legacy_object = df.astype({"grade": "object"})
usage_legacy_shallow = df_legacy_object.memory_usage(deep=False)
usage_legacy_deep = df_legacy_object.memory_usage(deep=True)
print("\nthe SAME column forced back to the legacy object dtype:")
print(f"  deep=False: {usage_legacy_shallow['grade']} bytes   deep=True: {usage_legacy_deep['grade']} bytes")
check(
    "on the legacy object dtype, deep=True reports far MORE bytes than deep=False -- the old surprise is still there if you ask for object",
    usage_legacy_deep["grade"] > usage_legacy_shallow["grade"],
)

print(f"\n{checks} checks, {failures} failure(s).")
if failures:
    raise SystemExit(1)
print("09_describe_known_column.py: every assertion held.")

lesson_id: D120
day: 120
kind: guided-build
languages: [python, bash]
setup_commands:
  - cd labs/sections/math-statistics-and-data/day-120-pandas-series-and-dataframes
  - python3 -m venv .venv
  - .venv/bin/pip install -r requirements/requirements.txt
  - .venv/bin/python3 -c "import pandas; print(pandas.__version__)"
run_commands:
  - 'cd examples && ../.venv/bin/python3 01_three_ways_to_build.py && cd ..'
  - 'cd examples && ../.venv/bin/python3 02_alignment.py && cd ..'
  - 'cd examples && ../.venv/bin/python3 03_dtype_promotion.py && cd ..'
  - 'cd examples && ../.venv/bin/python3 04_copy_on_write.py && cd ..'
  - 'cd examples && ../.venv/bin/python3 05_loc_vs_iloc.py && cd ..'
  - 'cd examples && ../.venv/bin/python3 06_nan_semantics.py && cd ..'
  - 'cd examples && ../.venv/bin/python3 07_vectorized_vs_apply.py && cd ..'
  - 'cd examples && ../.venv/bin/python3 08_string_dtype.py && cd ..'
  - 'cd examples && ../.venv/bin/python3 09_describe_known_column.py && cd ..'
  - .venv/bin/python3 starter/check_progress.py
test_commands:
  - bash tests/run_tests.sh
cleanup_commands:
  - "find . -path ./.venv -prune -o -type d -name '__pycache__' -print -exec rm -rf -- {} +"
  - rm -rf .pytest_cache
  - 'rm -rf .venv  # optional: removes the lab virtual environment'
  - 'git checkout -- starter/  # optional: reset your work'
requires_network: true
requires_api_key: false
estimated_minutes: 35
last_executed: '2026-08-19'
executed_on: 'macOS 26.5.2 (Apple Silicon, arm64), Python 3.14.0, pandas 3.0.5, pyarrow 25.0.1, numpy 2.5.2, bash 3.2.57 -- bash tests/run_tests.sh -> 41 checks, 0 failure(s), exit 0. All nine reference scripts in examples/ exit 0 with every internal assertion holding, run individually and via the harness. starter/check_progress.py reports 0 of 9 exercises complete on the untouched checkout (exit 1), and the harness separately solves every blank in a scratch copy and confirms 9 of 9 complete with exit 0, without modifying the real starter/exercises.py on disk. Section 5 of the harness deliberately breaks the assertion inside 08_string_dtype.py (expects "object" instead of "str"), confirms the run exits non-zero with a printed FAIL line, restores the file, and confirms it passes again -- so the suite is demonstrated to be capable of failing rather than merely claimed to be. Everything was run through a real lab-local .venv created by the documented setup commands. Three honesty notes from this run. FIRST: the coordinator independently measured the same pandas 3.0.5 behaviours (chained assignment leaving [10, 20, 30] unchanged, .loc giving [10, 0, 0], the str dtype default, the alignment result on x/y, an int64-plus-None promoting to float64 versus Int64 staying Int64) immediately before this lab was authored; every one of those was re-measured independently in this session rather than copied, and both runs agree. SECOND: pandas 3.0.5 does NOT silently swallow chained assignment the way the original brief assumed -- it raises a real ChainedAssignmentError warning (a Warning subclass, not a raised exception, so the statement still "succeeds" and execution continues) naming the exact fix. This is reported as measured rather than assumed, and the lesson corrects the brief''s framing rather than repeating it. THIRD: exercise 5''s original brief framing (comparing .loc[''b'':''d''] against .iloc[1:4] side by side) was corrected before writing, per the coordinator''s note, to assert .iloc[1:3] as the row that differs -- the version that actually demonstrates the inclusive/exclusive asymmetry rather than hiding it behind two slices that happen to agree. matplotlib, scipy and polars are not installed in this environment; polars is described from its public documentation in the lesson''s Tools section as a design contrast (no implicit index) and no output attributed to it is reproduced anywhere.'

# What is installed, why, and what it costs

Three packages, all free and open source, installed into a lab-local
virtual environment that `rm -rf .venv` completely undoes.

| Package | Version pinned | Licence | What this lab uses it for |
| --- | --- | --- | --- |
| `pandas` | 3.0.5 | BSD 3-Clause | Every Series and DataFrame in this lab. Pinned exactly because this day's captured output is version-specific — see below. |
| `pyarrow` | 25.0.1 | Apache 2.0 | The storage backend behind pandas 3.0's default `str` dtype (exercise 8) and the `Int64` nullable-integer arrays (exercise 3). |
| `numpy` | 2.5.2 | BSD 3-Clause | `np.nan`, `np.dtype`, and the underlying arrays a Series wraps — Day 104's ndarray, one layer down. |

There is no paid tier of anything in this lab, no account, no key and no
signup, personally or commercially.

## Why the versions are pinned exactly, not just floored

Every other lab in this course pins a *minimum* version. This one pins the
exact version, because **the lesson's central claims are pandas-3.0-specific
and would print different, equally-correct values on pandas 2.x**:

- `pd.Series(['a', 'b']).dtype` is `str` here; it is `object` on every
  pandas release before 3.0.
- Chained assignment (`df[mask]['col'] = value`) leaves the frame
  completely unchanged here, with a `ChainedAssignmentError` warning
  explaining why, because Copy-on-Write is unconditional starting in 3.0.
  On earlier pandas the same statement's behaviour depends on internal
  memory layout that is not part of any stable API contract.
- `pd.options.mode.copy_on_write = False` is a no-op with a deprecation
  warning here; on 2.x it actually did something.

Running this lab's suite against a different pandas major version will
produce test failures that are not bugs — they are the exact behavioural
difference the lesson exists to teach. `expected-output/FIELDS.md` states
precisely which values are version-specific.

## The one time the network is needed

```bash
.venv/bin/pip install -r requirements/requirements.txt
```

That is the only command in the lab that opens a connection. Step 6 of
`tests/run_tests.sh` greps every source file in `examples/` and `starter/`
to prove that nothing else does.

## What is deliberately *not* installed

**matplotlib**, **scipy** and **polars** are not installed in this
environment. The lesson's Tools section describes polars from its public
documentation as a design contrast to pandas — specifically, that polars
has no implicit row index at all, which is the single fact that sharpens
what this whole day is about: an index is not free, and it is not
decorative. **No output from polars, scipy or matplotlib is reproduced
anywhere** in this lab or its lesson; every place they are mentioned says
so plainly.

**scikit-learn** is not installed either. The lesson's AI-thread paragraph
references how a DataFrame index feeds into a training pipeline, but does
not run one.

## If you cannot install anything at all

pandas is not in the Python standard library, and there is no reduced path
through this lab without it — the whole point is pandas 3.0's specific
behaviour, which nothing else on your system will reproduce. If pandas
genuinely cannot be installed, read the lesson's captured output and
`expected-output/` directory instead; every number there came from a real
run and is not invented.

pandas==3.0.5
pyarrow==25.0.1
numpy==2.5.2

"""Run from this directory (or `python3 starter/check_progress.py` from the
lab root): reports how many of the nine exercises in exercises.py are
complete and correct, against the same expected values the reference
examples/ scripts assert.

Exit code is 0 only when all nine are correct, matching the convention the
rest of this lab's tests use.
"""

import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))

import exercises as ex

passed = 0
total = 9


def report(number, description, fn, expected_check):
    """Run one exercise function, catching the NameError an unfilled blank
    raises, and check its return value against expected_check(value)."""
    global passed
    try:
        result = fn()
    except NameError as exc:
        print(f"  {number}. {description}: NOT YET COMPLETE ({exc})")
        return
    except Exception as exc:  # noqa: BLE001 -- report any other mistake too
        print(f"  {number}. {description}: ERROR -- {exc!r}")
        return
    ok = expected_check(result)
    if ok:
        passed += 1
        print(f"  {number}. {description}: correct  (got {result!r})")
    else:
        print(f"  {number}. {description}: WRONG    (got {result!r})")


report(1, "build three ways", ex.ex01_build_three_ways, lambda r: r[1] == ["a", "b", "c"])
report(2, "alignment", ex.ex02_alignment, lambda r: set(r) == {"a", "d"})
report(3, "dtype promotion", ex.ex03_dtype_promotion, lambda r: r == "float64")
report(
    4,
    "copy-on-write",
    ex.ex04_copy_on_write,
    lambda r: r[0] == [10, 20, 30] and r[1] == [10, 20, 30] and r[2] == [10, 0, 0],
)
report(5, "loc vs iloc", ex.ex05_loc_vs_iloc, lambda r: r == (3, 2))
report(6, "nan semantics", ex.ex06_nan_semantics, lambda r: r is False)
report(7, "vectorized vs apply", ex.ex07_vectorized_vs_apply, lambda r: r == [108.0, 216.0, 324.0])
report(8, "string dtype", ex.ex08_string_dtype, lambda r: r == "str")
report(9, "describe", ex.ex09_describe, lambda r: tuple(r) == (8.0, 5.0, 2.0, 9.0))

print(f"\n{passed} of {total} exercises complete.")
sys.exit(0 if passed == total else 1)

"""Day 120 starter -- nine exercises, one function each.

Each function below is a working skeleton: the setup is written for you, and
exactly one line is left for you to write, marked with the sentinel name
`_FILL_THIS_IN`. Replace that name with real code. Leaving it as-is raises a
clear NameError when the function runs, which `check_progress.py` catches
and reports -- it does not crash the whole script.

Read `../examples/` for the fully worked reference AFTER you have tried each
one yourself; that is where every one of these ideas is explained in
comments. Run your progress with:

    python3 check_progress.py

from inside this `starter/` directory (or `python3 starter/check_progress.py`
from the lab root).
"""

import numpy as np
import pandas as pd


def ex01_build_three_ways():
    """Build a Series from a dict {"a": 10, "b": 20, "c": 30}, and read off
    its index as a plain Python list. Return (series, index_list)."""
    s = pd.Series({"a": 10, "b": 20, "c": 30})
    index_list = _FILL_THIS_IN  # convert s.index to a plain list
    return s, index_list


def ex02_alignment():
    """Add x (index a, b, c) to y (index b, c, d) and report which labels
    come back as NaN. Return the set of NaN labels."""
    x = pd.Series([1, 2, 3], index=["a", "b", "c"])
    y = pd.Series([10, 20, 30], index=["b", "c", "d"])
    z = x + y
    nan_labels = _FILL_THIS_IN  # set of index labels where z is NaN -- use z.isna()
    return nan_labels


def ex03_dtype_promotion():
    """Reindex a clean int64 Series onto a 4th label that was never there,
    and report the resulting dtype as a string."""
    ids = pd.Series([1001, 1002, 1003], dtype="int64")
    reindexed = ids.reindex([0, 1, 2, 3])
    dtype_name = _FILL_THIS_IN  # str(reindexed.dtype)
    return dtype_name


def ex04_copy_on_write():
    """Show that chained assignment does nothing, and .loc does. Return the
    'b' column's values (before, after_chained, after_loc) as three lists."""
    df = pd.DataFrame({"a": [1, 2, 3], "b": [10, 20, 30]})
    before = df["b"].tolist()
    df[df["a"] > 1]["b"] = 0  # chained assignment -- do not "fix" this line
    after_chained = df["b"].tolist()
    _FILL_THIS_IN  # write the ONE .loc statement that actually changes df["b"] where a > 1, to 0
    after_loc = df["b"].tolist()
    return before, after_chained, after_loc


def ex05_loc_vs_iloc():
    """On a frame indexed a..e, return (len(.loc['b':'d']), len(.iloc[1:3]))."""
    df = pd.DataFrame({"val": [10, 20, 30, 40, 50]}, index=["a", "b", "c", "d", "e"])
    by_label = df.loc["b":"d"]
    by_position = _FILL_THIS_IN  # the .iloc slice that STOPS BEFORE position 3
    return len(by_label), len(by_position)


def ex06_nan_semantics():
    """Return whether float('nan') == float('nan') (should be False)."""
    result = _FILL_THIS_IN  # the actual comparison, not a hard-coded boolean
    return result


def ex07_vectorized_vs_apply():
    """Compute price * 1.08 vectorised on a Series -- no .apply, no lambda."""
    prices = pd.Series([100.0, 200.0, 300.0])
    result = _FILL_THIS_IN  # one vectorised expression, no .apply
    return result.tolist()


def ex08_string_dtype():
    """Return the dtype of pd.Series(['a', 'b']) as a string."""
    s = pd.Series(["a", "b"])
    dtype_name = _FILL_THIS_IN  # str(s.dtype)
    return dtype_name


def ex09_describe():
    """Return the count, mean, min and max of [2, 4, 4, 4, 5, 5, 7, 9] using
    .describe(), as a 4-tuple of floats."""
    values = [2, 4, 4, 4, 5, 5, 7, 9]
    desc = pd.Series(values).describe()
    result = _FILL_THIS_IN  # (desc["count"], desc["mean"], desc["min"], desc["max"])
    return result

#!/usr/bin/env bash
# Tests for the Day 120 lab. Run from the lab directory:
#   bash tests/run_tests.sh
#
# The harness proves the lesson's claims by running code and reading real
# values, never by reading source:
#
#   * a Series built from a dict takes the dict's keys as its index, and a
#     DataFrame built from a bare NumPy array has NO labels until you supply
#     them yourself;
#   * adding two Series with partially overlapping indexes aligns on LABEL,
#     not position -- labels a and d, present on only one side, become NaN --
#     and .to_numpy() addition gives a different, purely positional answer;
#   * an int64 column reindexed onto a label that was never there is
#     silently promoted to float64, losing exact precision past 2**53, while
#     the nullable Int64 dtype stays exact;
#   * chained assignment (`df[mask]['col'] = value`) leaves the ORIGINAL
#     frame completely unchanged -- pandas 3.0.5 warns about it with a
#     ChainedAssignmentError, but the statement still "succeeds" -- and the
#     single-.loc form is the one that actually writes;
#   * .loc['b':'d'] and .iloc[1:4] return the same three rows, but .iloc[1:3]
#     -- the "matching" number -- silently drops one, because .loc's stop is
#     inclusive of a label and .iloc's stop is exclusive of a position;
#   * float('nan') != float('nan'), and comparing a Series against np.nan
#     with == finds nothing, ever -- .isna() is the only reliable test;
#   * vectorised arithmetic beats .apply(lambda ...) by at least 20x on
#     200,000 rows, measured as a ratio and a shape, never a millisecond
#     figure;
#   * pd.Series(['a', 'b']).dtype is 'str' on pandas 3.0.5, not the 'object'
#     every pre-3.0 tutorial describes;
#   * .describe() on a known eight-value column matches hand computation
#     exactly for count, mean, min and max, and matches Day 116's
#     Bessel-corrected sample standard deviation to the ninth decimal;
#   * nothing is left behind on disk.
#
# Everything after the one-time install runs offline. Nothing binds a port,
# nothing writes outside the lab, nothing needs a key. Deterministic,
# non-interactive, exits 0 only if every check passes.
set -u

export PYTHONDONTWRITEBYTECODE=1

lab_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"

# Bytecode left by an EARLIER command is not this run's litter. The README
# documents `python3 starter/check_progress.py`, and running it writes .pyc
# files that would then fail the cleanliness check at the end of this
# script -- failing the reader for following the instructions. Clearing them
# here makes that final check measure what it claims to: what THIS run left
# behind. `.venv` is untouched, because the packages' own bytecode is
# theirs, not ours.
find "${lab_dir}" -name '.venv' -prune -o -type d -name '__pycache__' -exec rm -rf {} + 2>/dev/null || true
find "${lab_dir}" -name '.venv' -prune -o -type d -name '.pytest_cache' -exec rm -rf {} + 2>/dev/null || true

failures=0
checks=0

check() {
  local label="$1" ok="$2"
  checks=$((checks + 1))
  if [ "${ok}" = "yes" ]; then
    echo "  ok: ${label}"
  else
    echo "  FAIL: ${label}"
    failures=$((failures + 1))
  fi
}

check_eq() {
  # check_eq <label> <expected> <actual>
  if [ "$2" = "$3" ]; then
    check "$1" "yes"
  else
    check "$1 (expected [$2], got [$3])" "no"
  fi
}

# Resolve python: an explicit override, then this lab's .venv, then PATH.
# Fails loudly with instructions rather than silently skipping checks.
resolve_tool() {
  local tool="$1" override="$2"
  if [ -n "${override}" ] && [ -x "${override}" ]; then echo "${override}"; return 0; fi
  if [ -x "${lab_dir}/.venv/bin/${tool}" ]; then echo "${lab_dir}/.venv/bin/${tool}"; return 0; fi
  if command -v "${tool}" >/dev/null 2>&1; then command -v "${tool}"; return 0; fi
  return 1
}

python_bin="$(resolve_tool python3 "${PYTHON:-}")" || {
  echo "FAIL: python3 not found." >&2
  echo "  Install the lab's dependencies with:" >&2
  echo "    python3 -m venv .venv" >&2
  echo "    .venv/bin/pip install -r requirements/requirements.txt" >&2
  echo "  Or point this suite at an existing python3:" >&2
  echo "    PYTHON=/path/to/python3 bash tests/run_tests.sh" >&2
  exit 1
}

if ! "${python_bin}" -c "import pandas, pyarrow" >/dev/null 2>&1; then
  echo "FAIL: pandas/pyarrow are not importable from ${python_bin}." >&2
  echo "  Install the lab's dependencies with:" >&2
  echo "    python3 -m venv .venv" >&2
  echo "    .venv/bin/pip install -r requirements/requirements.txt" >&2
  exit 1
fi

echo "Day 120 — Frames You Can Trust"
echo

# --------------------------------------------------------------------------
echo "1. The tools and the versions this lab was written against"
# --------------------------------------------------------------------------

versions="$("${python_bin}" - <<'PY'
import platform
import sys
from importlib.metadata import version

print(f"python   {platform.python_version()}")
for name in ("pandas", "pyarrow", "numpy"):
    print(f"{name:<8} {version(name)}")
print(f"platform {platform.platform()}")
print(f"exe      {sys.executable.rsplit('/', 3)[-1]}")
PY
)"
echo "${versions}" | sed 's/^/  /'

pinned_pandas="$(grep -E '^pandas==' "${lab_dir}/requirements/requirements.txt" | cut -d= -f3)"
installed_pandas="$("${python_bin}" -c "from importlib.metadata import version; print(version('pandas'))")"
check_eq "installed pandas matches requirements.txt" "${pinned_pandas}" "${installed_pandas}"

pandas_major="$("${python_bin}" -c "import pandas; print(pandas.__version__.split('.')[0])")"
check_eq "pandas is version 3 or later (this lab's captured output is 3.0.5-specific)" "3" "${pandas_major}"

# --------------------------------------------------------------------------
echo
echo "2. Every reference script runs and every assertion inside it holds"
# --------------------------------------------------------------------------

for script in 01_three_ways_to_build 02_alignment 03_dtype_promotion \
              04_copy_on_write 05_loc_vs_iloc 06_nan_semantics \
              07_vectorized_vs_apply 08_string_dtype 09_describe_known_column; do
  out="$(cd "${lab_dir}/examples" && "${python_bin}" "${script}.py" 2>&1)"
  status=$?
  if [ "${status}" -ne 0 ]; then
    check "${script}.py exits 0" "no"
    echo "${out}" | tail -8 | sed 's/^/      /'
  else
    check "${script}.py exits 0" "yes"
  fi
  case "${out}" in
    *"${script}.py: every assertion held."*)
      check "${script}.py reports every assertion held" "yes" ;;
    *) check "${script}.py reports every assertion held" "no" ;;
  esac
done

# --------------------------------------------------------------------------
echo
echo "3. The starter checker: honest progress, both directions"
# --------------------------------------------------------------------------

starter_out="$(cd "${lab_dir}" && "${python_bin}" starter/check_progress.py 2>&1)"
starter_status=$?
echo "${starter_out}" | tail -3 | sed 's/^/  /'
case "${starter_out}" in
  *"0 of 9 exercises complete."*)
    check "an untouched starter checkout reports 0 of 9 complete" "yes" ;;
  *)
    check "an untouched starter checkout reports 0 of 9 complete" "no" ;;
esac
if [ "${starter_status}" -ne 0 ]; then
  check "the starter checker exits non-zero when incomplete" "yes"
else
  check "the starter checker exits non-zero when incomplete" "no"
fi

# Prove the checker can also report success: solve every exercise in a
# scratch copy, confirm 9 of 9 and exit 0, then discard the copy. The real
# starter/exercises.py on disk is never modified by this.
solved_dir="$(mktemp -d)"
trap 'rm -rf "${solved_dir}"' EXIT
cp "${lab_dir}/starter/exercises.py" "${solved_dir}/exercises.py"
cp "${lab_dir}/starter/check_progress.py" "${solved_dir}/check_progress.py"
"${python_bin}" - "${solved_dir}/exercises.py" <<'PY'
import sys
path = sys.argv[1]
s = open(path).read()
replacements = [
    ('index_list = _FILL_THIS_IN  # convert s.index to a plain list', 'index_list = list(s.index)'),
    ('nan_labels = _FILL_THIS_IN  # set of index labels where z is NaN -- use z.isna()', 'nan_labels = set(z.index[z.isna()])'),
    ('dtype_name = _FILL_THIS_IN  # str(reindexed.dtype)', 'dtype_name = str(reindexed.dtype)'),
    ('_FILL_THIS_IN  # write the ONE .loc statement that actually changes df["b"] where a > 1, to 0', 'df.loc[df["a"] > 1, "b"] = 0'),
    ('by_position = _FILL_THIS_IN  # the .iloc slice that STOPS BEFORE position 3', 'by_position = df.iloc[1:3]'),
    ('result = _FILL_THIS_IN  # the actual comparison, not a hard-coded boolean', 'result = float("nan") == float("nan")'),
    ('result = _FILL_THIS_IN  # one vectorised expression, no .apply', 'result = prices * 1.08'),
    ('dtype_name = _FILL_THIS_IN  # str(s.dtype)', 'dtype_name = str(s.dtype)'),
    ('result = _FILL_THIS_IN  # (desc["count"], desc["mean"], desc["min"], desc["max"])', 'result = (desc["count"], desc["mean"], desc["min"], desc["max"])'),
]
for old, new in replacements:
    if old not in s:
        raise SystemExit(f"pattern not found, starter/exercises.py has drifted: {old!r}")
    s = s.replace(old, new)
open(path, "w").write(s)
PY
solved_out="$(cd "${solved_dir}" && "${python_bin}" check_progress.py 2>&1)"
solved_status=$?
echo "${solved_out}" | tail -3 | sed 's/^/  /'
case "${solved_out}" in
  *"9 of 9 exercises complete."*)
    check "a fully solved copy reports 9 of 9 complete" "yes" ;;
  *)
    check "a fully solved copy reports 9 of 9 complete" "no" ;;
esac
check_eq "the checker exits 0 once every exercise is correct" "0" "${solved_status}"
rm -rf "${solved_dir}"
trap - EXIT

# --------------------------------------------------------------------------
echo
echo "4. The lesson's sharpest claims, checked one value at a time"
# --------------------------------------------------------------------------

facts="$(cd "${lab_dir}/examples" && "${python_bin}" - <<'PY'
import warnings

import numpy as np
import pandas as pd

# Copy-on-Write: chained assignment does nothing; .loc does.
df = pd.DataFrame({"a": [1, 2, 3], "b": [10, 20, 30]})
before = df["b"].tolist()
with warnings.catch_warnings(record=True) as caught:
    warnings.simplefilter("always")
    df[df["a"] > 1]["b"] = 0
    warned = any(w.category.__name__ == "ChainedAssignmentError" for w in caught)
after_chained = df["b"].tolist()
df.loc[df["a"] > 1, "b"] = 0
after_loc = df["b"].tolist()
print("cow_chained_unchanged", after_chained == before)
print("cow_warned", warned)
print("cow_loc_changed", after_loc)

# String dtype default.
print("string_dtype", str(pd.Series(["a", "b"]).dtype))

# Alignment.
x = pd.Series([1, 2, 3], index=["a", "b", "c"])
y = pd.Series([10, 20, 30], index=["b", "c", "d"])
z = x + y
print("alignment_nan_labels", "|".join(sorted(z.index[z.isna()])))
print("alignment_b", z["b"])
print("alignment_c", z["c"])

# loc vs iloc, the corrected framing.
df5 = pd.DataFrame({"val": [10, 20, 30, 40, 50]}, index=["a", "b", "c", "d", "e"])
print("loc_iloc4_equal", df5.loc["b":"d"].equals(df5.iloc[1:4]))
print("iloc3_shorter", len(df5.iloc[1:3]) == len(df5.loc["b":"d"]) - 1)

# Int64 stays exact past 2**53; plain int64 promoted through reindex does not.
big_id = 2**53 + 1
lost = int(pd.Series([big_id], dtype="int64").reindex([0, 1]).iloc[0])
kept = int(pd.Series([big_id], dtype="Int64").reindex([0, 1]).iloc[0])
print("big_id_lost_precision", lost != big_id)
print("big_id_kept_precision", kept == big_id)
PY
)"
echo "${facts}" | sed 's/^/  /'

get_fact() { printf '%s\n' "${facts}" | grep "^$1 " | cut -d' ' -f2-; }

check_eq "chained assignment leaves df['b'] byte-for-byte unchanged" "True" "$(get_fact cow_chained_unchanged)"
check_eq "pandas 3.0.5 raises a ChainedAssignmentError warning on that statement" "True" "$(get_fact cow_warned)"
check_eq "the .loc form changes b to [10, 0, 0]" "[10, 0, 0]" "$(get_fact cow_loc_changed)"
check_eq "pd.Series(['a','b']).dtype is the pandas-3.0 default str, not object" "str" "$(get_fact string_dtype)"
check_eq "exactly labels a and d go NaN under alignment" "a|d" "$(get_fact alignment_nan_labels)"
check_eq "label b aligns to 2 + 10 = 12.0" "12.0" "$(get_fact alignment_b)"
check_eq "label c aligns to 3 + 20 = 23.0" "23.0" "$(get_fact alignment_c)"
check_eq "loc['b':'d'] equals iloc[1:4] (different stop values, same rows)" "True" "$(get_fact loc_iloc4_equal)"
check_eq "iloc[1:3] is exactly one row shorter than loc['b':'d']" "True" "$(get_fact iloc3_shorter)"
check_eq "plain int64 loses exact precision past 2**53 once promoted" "True" "$(get_fact big_id_lost_precision)"
check_eq "nullable Int64 keeps exact precision past 2**53" "True" "$(get_fact big_id_kept_precision)"

# --------------------------------------------------------------------------
echo
echo "5. Prove the harness can fail, then restore it"
# --------------------------------------------------------------------------

broken_script="${lab_dir}/examples/08_string_dtype.py"
cp "${broken_script}" "${broken_script}.bak"
sed -i.tmp 's/str(s.dtype) == "str"/str(s.dtype) == "object"/' "${broken_script}"
rm -f "${broken_script}.tmp"
broken_out="$(cd "${lab_dir}/examples" && "${python_bin}" 08_string_dtype.py 2>&1)"
broken_status=$?
mv "${broken_script}.bak" "${broken_script}"
if [ "${broken_status}" -ne 0 ]; then
  check "a deliberately wrong assertion makes 08_string_dtype.py exit non-zero" "yes"
else
  check "a deliberately wrong assertion makes 08_string_dtype.py exit non-zero" "no"
fi
case "${broken_out}" in
  *FAIL:*) check "the broken run reports a FAIL line" "yes" ;;
  *) check "the broken run reports a FAIL line" "no" ;;
esac
restored_out="$(cd "${lab_dir}/examples" && "${python_bin}" 08_string_dtype.py 2>&1)"
restored_status=$?
check_eq "the script is restored and exits 0 again" "0" "${restored_status}"
case "${restored_out}" in
  *"08_string_dtype.py: every assertion held."*)
    check "the restored script reports every assertion held" "yes" ;;
  *) check "the restored script reports every assertion held" "no" ;;
esac

# --------------------------------------------------------------------------
echo
echo "6. Nothing left behind, and no network dependency baked into the lab"
# --------------------------------------------------------------------------

if grep -rInE 'https?://' "${lab_dir}/examples" "${lab_dir}/starter" >/dev/null 2>&1; then
  check "no URL appears in examples/ or starter/" "no"
else
  check "no URL appears in examples/ or starter/" "yes"
fi

find "${lab_dir}" -name '.venv' -prune -o -type d -name '__pycache__' -exec rm -rf {} + 2>/dev/null || true
find "${lab_dir}" -name '.venv' -prune -o -type d -name '.pytest_cache' -exec rm -rf {} + 2>/dev/null || true

stray="$(find "${lab_dir}" -name '.venv' -prune -o -type d \( -name '__pycache__' -o -name '.pytest_cache' \) -print 2>/dev/null)"
if [ -z "${stray}" ]; then
  check "no __pycache__ or .pytest_cache directories were left behind" "yes"
else
  check "no __pycache__ or .pytest_cache directories were left behind" "no"
  echo "${stray}" | sed 's/^/      /'
fi

echo
echo "${checks} checks, ${failures} failure(s)."
if [ "${failures}" -ne 0 ]; then
  exit 1
fi
exit 0