Hands-on lab — Day 65: CSV and JSON in the Real World

{
  "input_csv": "data/messy_orders.csv",
  "encoding": "utf-8-sig",
  "delimiter": ",",
  "columns": ["order_id", "customer", "items", "notes", "total"],
  "defaults": {
    "notes": "",
    "total": "0.00"
  },
  "numeric_columns": ["total"],
  "output_dir": "out",
  "output_json": "orders.json",
  "output_jsonl": "orders.jsonl",
  "output_csv": "orders-clean.csv"
}

order_id,customer,items,notes,total
1001,"Ada Lovelace","widget, large",Priority shipping,49.50
1002,"Grace Hopper","cable, 2 m","Leave at door.
Ring the bell twice.",12.00
1003,"Alan ""Turing"" Jr.",monitor,Fragile,240.00
1004,Katherine Johnson,notebook,,3.25
1005,Barbara Liskov,pen

#!/usr/bin/env python3
"""csv_field_parser.py — a CSV parser built from scratch (REFERENCE).

This is the whole of RFC 4180 quoting in about fifty lines: a state
machine that walks the text one character at a time and decides, for each
character, whether it is data, a field boundary, a record boundary, or a
quote that changes the rules.

The four states
---------------
  field-start      between fields; the next character decides everything
  in-field         inside an UNQUOTED field; a comma or newline ends it
  in-quoted        inside a QUOTED field; commas and newlines are DATA
  quote-in-quoted  saw a quote while quoted; the next character says whether
                   that quote was an escaped quote ("") or the closing quote

Run this file directly to prove the parser agrees with the standard
library on the lab's deliberately messy file:

    python3 examples/csv_field_parser.py
"""
import csv
import io
from pathlib import Path

LAB_DIR = Path(__file__).resolve().parent.parent


def parse_csv(text, delimiter=",", quotechar='"'):
    """Parse CSV text into a list of rows, each row a list of field strings.

    Implements RFC 4180 quoting: a field may be wrapped in quotes, in which
    case it may contain the delimiter, newlines, and doubled quotes ("")
    that stand for one literal quote. Carriage returns outside a quoted
    field are ignored so that CRLF and LF files parse the same way.
    """
    rows = []
    row = []
    field = []
    state = "field-start"

    for char in text:
        if state == "field-start":
            if char == quotechar:
                state = "in-quoted"
            elif char == delimiter:
                row.append("")
            elif char == "\n":
                row.append("")
                rows.append(row)
                row = []
            elif char != "\r":
                field.append(char)
                state = "in-field"

        elif state == "in-field":
            if char == delimiter:
                row.append("".join(field))
                field = []
                state = "field-start"
            elif char == "\n":
                row.append("".join(field))
                field = []
                rows.append(row)
                row = []
                state = "field-start"
            elif char != "\r":
                field.append(char)

        elif state == "in-quoted":
            if char == quotechar:
                state = "quote-in-quoted"
            else:
                field.append(char)  # commas and newlines are plain data here

        elif state == "quote-in-quoted":
            if char == quotechar:
                field.append(quotechar)  # "" means one literal quote
                state = "in-quoted"
            elif char == delimiter:
                row.append("".join(field))
                field = []
                state = "field-start"
            elif char == "\n":
                row.append("".join(field))
                field = []
                rows.append(row)
                row = []
                state = "field-start"
            elif char != "\r":
                field.append(char)
                state = "in-field"

    if field or row or state != "field-start":
        row.append("".join(field))
        rows.append(row)
    return rows


def parse_with_stdlib(text, delimiter=","):
    """Parse the same text with the standard library, for comparison."""
    return [list(row) for row in csv.reader(io.StringIO(text, newline=""), delimiter=delimiter)]


def compare(text, delimiter=","):
    """Return (mine, theirs, agree) for one blob of CSV text."""
    mine = parse_csv(text, delimiter=delimiter)
    theirs = parse_with_stdlib(text, delimiter=delimiter)
    return mine, theirs, mine == theirs


def main():
    text = (LAB_DIR / "data" / "messy_orders.csv").read_text(encoding="utf-8-sig")
    mine, theirs, agree = compare(text)
    print(f"rows parsed by the from-scratch parser: {len(mine)}")
    print(f"rows parsed by the csv module:          {len(theirs)}")
    for index, (a, b) in enumerate(zip(mine, theirs)):
        mark = "same" if a == b else "DIFFERENT"
        print(f"  row {index}: {mark}  {a}")
    print(f"agree on every row: {agree}")
    return 0 if agree else 1


if __name__ == "__main__":
    raise SystemExit(main())

#!/usr/bin/env python3
"""wrangle.py — the finished data-wrangling pipeline (REFERENCE).

Reads a deliberately messy CSV (byte-order mark, embedded commas, a quoted
newline, doubled quotes, an empty field, a ragged row), proves that naive
splitting on commas mangles it, parses it properly with csv.DictReader,
cleans it against a JSON config, writes it back out as JSON, JSON Lines,
and a well-formed CSV, and then reloads each of those to prove the round
trip is lossless.

    python3 examples/wrangle.py

Everything it writes lands in out/ next to this lab; nothing else on your
machine is touched.
"""
import csv
import json
from pathlib import Path

LAB_DIR = Path(__file__).resolve().parent.parent


# --- step 1: the naive parser, shown failing -------------------------------

def naive_split_rows(text, delimiter=","):
    """Split each line on the delimiter — the bug factory, kept for contrast."""
    return [line.split(delimiter) for line in text.splitlines()]


def naive_report(text, expected_columns):
    """Return a list of (line_number, field_count) for lines naive splitting breaks."""
    broken = []
    for number, row in enumerate(naive_split_rows(text), start=1):
        if len(row) != expected_columns:
            broken.append((number, len(row)))
    return broken


# --- step 2: parse properly ------------------------------------------------

def read_rows(csv_path, encoding, delimiter):
    """Read the CSV into a list of dicts using csv.DictReader.

    encoding='utf-8-sig' strips a leading byte-order mark if one is there.
    newline='' is required: the csv module does its own newline handling,
    and without it a quoted field containing a newline can be split in two.
    """
    with open(csv_path, "r", encoding=encoding, newline="") as handle:
        reader = csv.DictReader(handle, delimiter=delimiter)
        return [dict(row) for row in reader]


# --- step 3: clean ---------------------------------------------------------

def clean_row(row, config):
    """Return one tidy record: no None values, stripped text, numbers as numbers."""
    cleaned = {}
    for column in config["columns"]:
        value = row.get(column)
        if value is None:
            value = config["defaults"].get(column, "")
        cleaned[column] = value.strip() if isinstance(value, str) else value
    for column in config["numeric_columns"]:
        cleaned[column] = float(cleaned[column])
    return cleaned


def clean_rows(rows, config):
    """Clean every row."""
    return [clean_row(row, config) for row in rows]


# --- step 4: write and read back -------------------------------------------

def write_json(records, path):
    """Write one JSON array. indent=2 for humans; ensure_ascii=False keeps text readable."""
    path.write_text(
        json.dumps(records, indent=2, ensure_ascii=False, sort_keys=False) + "\n",
        encoding="utf-8",
    )


def read_json(path):
    """Read the JSON array back."""
    return json.loads(path.read_text(encoding="utf-8"))


def write_jsonl(records, path):
    """Write JSON Lines: one compact JSON object per line, no outer array."""
    with open(path, "w", encoding="utf-8") as handle:
        for record in records:
            handle.write(json.dumps(record, ensure_ascii=False) + "\n")


def read_jsonl(path):
    """Read JSON Lines back, one object per non-empty line."""
    records = []
    with open(path, "r", encoding="utf-8") as handle:
        for line in handle:
            line = line.strip()
            if line:
                records.append(json.loads(line))
    return records


def write_csv(records, path, columns, delimiter):
    """Write a well-formed CSV with csv.DictWriter — it quotes whatever needs it."""
    with open(path, "w", encoding="utf-8", newline="") as handle:
        writer = csv.DictWriter(handle, fieldnames=columns, delimiter=delimiter)
        writer.writeheader()
        writer.writerows(records)


def read_csv_typed(path, columns, delimiter, numeric_columns):
    """Read a clean CSV back, restoring the numeric columns (CSV has no types)."""
    with open(path, "r", encoding="utf-8", newline="") as handle:
        rows = [dict(row) for row in csv.DictReader(handle, delimiter=delimiter)]
    for row in rows:
        for column in numeric_columns:
            row[column] = float(row[column])
    return rows


# --- the pipeline ----------------------------------------------------------

def main():
    config = json.loads((LAB_DIR / "data" / "config.json").read_text(encoding="utf-8"))
    csv_path = LAB_DIR / config["input_csv"]
    raw_text = csv_path.read_text(encoding=config["encoding"])
    delimiter = config["delimiter"]
    columns = config["columns"]

    print("=== 1. naive line.split(',') ===")
    broken = naive_report(raw_text, len(columns))
    print(f"lines whose field count is not {len(columns)}: {len(broken)}")
    for number, count in broken:
        print(f"  line {number}: {count} fields")
    naive = naive_split_rows(raw_text)
    print(f"naive row for order 1001: {naive[1]}")
    print(f"naive row for order 1003: {naive[4]}")
    print("(order 1003 has 5 fields, so the count check passes — and the data is still wrong)")

    print()
    print("=== 2. csv.DictReader ===")
    rows = read_rows(csv_path, config["encoding"], delimiter)
    print(f"records parsed: {len(rows)}")
    print(f"first key is 'order_id' (byte-order mark stripped): {list(rows[0])[0] == 'order_id'}")
    print(f"order 1002 notes: {rows[1]['notes']!r}")
    print(f"order 1003 customer: {rows[2]['customer']!r}")
    print(f"order 1005 (ragged) total: {rows[4]['total']!r}")

    print()
    print("=== 3. clean ===")
    records = clean_rows(rows, config)
    for record in records:
        print(f"  {record['order_id']} {record['customer']:<18} {record['total']:>7.2f}")

    print()
    print("=== 4. write and round-trip ===")
    out_dir = LAB_DIR / config["output_dir"]
    out_dir.mkdir(exist_ok=True)
    json_path = out_dir / config["output_json"]
    jsonl_path = out_dir / config["output_jsonl"]
    clean_csv_path = out_dir / config["output_csv"]

    write_json(records, json_path)
    write_jsonl(records, jsonl_path)
    write_csv(records, clean_csv_path, columns, delimiter)

    from_json = read_json(json_path)
    from_jsonl = read_jsonl(jsonl_path)
    from_csv = read_csv_typed(clean_csv_path, columns, delimiter, config["numeric_columns"])

    assert from_json == records, "JSON round trip lost or changed data"
    assert from_jsonl == records, "JSON Lines round trip lost or changed data"
    assert from_csv == records, "CSV round trip lost or changed data"

    print(f"wrote {json_path.name}  ({json_path.stat().st_size} bytes)")
    print(f"wrote {jsonl_path.name} ({jsonl_path.stat().st_size} bytes, {len(records)} lines)")
    print(f"wrote {clean_csv_path.name} ({clean_csv_path.stat().st_size} bytes)")
    print("round trip lossless: JSON yes, JSONL yes, CSV yes")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())

lesson_id: D065
day: 65
kind: coding
languages: [python, bash]
setup_commands:
  - cd labs/sections/programming-with-python/day-065-csv-and-json-in-the-real
  - python3 --version
run_commands:
  - head -c 3 data/messy_orders.csv | xxd
  - python3 examples/wrangle.py
  - python3 examples/csv_field_parser.py
  - cat out/orders.jsonl
  - python3 starter/wrangle.py
  - python3 starter/csv_field_parser.py
test_commands:
  - bash tests/run_tests.sh
cleanup_commands:
  - rm -rf out
  - 'git checkout -- starter/  # optional: reset your work'
requires_network: false
requires_api_key: false
estimated_minutes: 30
last_executed: '2026-07-19'
executed_on: 'macOS 26.5.1 (Apple Silicon), Python 3.14.0, bash 3.2 — bash tests/run_tests.sh -> 26 checks, 0 failure(s), exit 0 (34 checks, 0 failure(s) with the starter exercises completed)'

# Dependencies — Day 065 lab

**Python 3 only. No third-party packages, no network, no API key.**

- `python3` (3.8 or newer; tested on 3.14.0). You set this up on Day 43.
- `bash` for the test runner (preinstalled on macOS and Linux).
- Standard library only: `csv`, `json`, and `pathlib` in the lab files,
  plus `io` in the from-scratch parser so it can hand a string to
  `csv.reader`. There is deliberately no `requirements.txt`.

That last point is the lesson in miniature. CSV and JSON are so central to
data work that Python ships a reader, a writer, an encoder, and a decoder
for them in the box. You will meet `pandas` later, and it is excellent —
but reaching for a 30 MB dependency to read a five-row file is a habit
worth resisting, and you cannot debug what `pandas.read_csv` did to your
quoting unless you understand what the `csv` module does first.

Check your Python is present and new enough:

```bash
python3 --version
```

If that prints `Python 3.8` or higher, you are ready.

Windows users: run the commands inside WSL, or use `python` in place of
`python3` if that is how Python is exposed on your system. One genuine
difference matters on Windows and is covered in `troubleshooting.md`: if
you forget `newline=''` when opening a file for the `csv` module, Windows
translates the `\n` the writer emits into `\r\n`, and you get a blank line
between every record. The lab files all pass `newline=''`, so they behave
identically everywhere.

#!/usr/bin/env python3
"""csv_field_parser.py — YOUR working file: a CSV parser from scratch.

You have used the csv module. Now build the part of it that matters, so
that quoting stops being magic. All of RFC 4180's quoting rules fit in one
state machine that walks the text one character at a time.

The four states
---------------
  field-start      between fields; the next character decides everything
  in-field         inside an UNQUOTED field; a comma or newline ends it
  in-quoted        inside a QUOTED field; commas and newlines are DATA
  quote-in-quoted  saw a quote while quoted; the next character says whether
                   that quote was an escaped quote ("") or the closing quote

Finish Exercise 5, then run this file. It parses the lab's messy CSV twice
— once with your parser, once with the standard library — and prints
whether they agree on every row.

    python3 starter/csv_field_parser.py
"""
import csv
import io
from pathlib import Path

LAB_DIR = Path(__file__).resolve().parent.parent


def parse_csv(text, delimiter=",", quotechar='"'):
    """Parse CSV text into a list of rows, each row a list of field strings."""
    rows = []      # finished rows
    row = []       # fields of the row being built
    field = []     # characters of the field being built
    state = "field-start"

    for char in text:
        # Exercise 5: THE QUOTING STATE MACHINE.
        #
        # Handle each state in turn. The rules, in full:
        #
        # state == "field-start"
        #   char is the quote character -> state becomes "in-quoted"
        #   char is the delimiter       -> the field was empty: row.append("")
        #   char is "\n"                -> empty last field, end the row:
        #                                  row.append(""), rows.append(row), row = []
        #   char is "\r"                -> ignore it (so CRLF and LF agree)
        #   anything else               -> field.append(char); state = "in-field"
        #
        # state == "in-field"
        #   char is the delimiter -> finish the field:
        #        row.append("".join(field)); field = []; state = "field-start"
        #   char is "\n"          -> finish the field AND the row, then
        #        rows.append(row); row = []; state = "field-start"
        #   char is "\r"          -> ignore it
        #   anything else         -> field.append(char)
        #
        # state == "in-quoted"      <-- the state that makes CSV work
        #   char is the quote character -> state = "quote-in-quoted"
        #   anything else               -> field.append(char)
        #        Yes, ANYTHING: commas and newlines are ordinary data in here.
        #        That single line is why hand-splitting on commas is a bug.
        #
        # state == "quote-in-quoted"
        #   char is the quote character -> "" means one literal quote:
        #        field.append(quotechar); state = "in-quoted"
        #   char is the delimiter -> the quoted field ended; finish the field
        #        and go back to "field-start"
        #   char is "\n"          -> the quoted field and the row both ended
        #   char is "\r"          -> ignore it
        #   anything else         -> stray text after a closing quote; the
        #        forgiving choice is field.append(char); state = "in-field"
        raise NotImplementedError("Exercise 5: implement the state machine")

    # Flush the final field/row when the text does not end with a newline.
    if field or row or state != "field-start":
        row.append("".join(field))
        rows.append(row)
    return rows


def parse_with_stdlib(text, delimiter=","):
    """Parse the same text with the standard library, for comparison (provided)."""
    return [list(row) for row in csv.reader(io.StringIO(text, newline=""), delimiter=delimiter)]


def compare(text, delimiter=","):
    """Return (mine, theirs, agree) for one blob of CSV text (provided)."""
    mine = parse_csv(text, delimiter=delimiter)
    theirs = parse_with_stdlib(text, delimiter=delimiter)
    return mine, theirs, mine == theirs


def main():
    text = (LAB_DIR / "data" / "messy_orders.csv").read_text(encoding="utf-8-sig")
    mine, theirs, agree = compare(text)
    print(f"rows parsed by the from-scratch parser: {len(mine)}")
    print(f"rows parsed by the csv module:          {len(theirs)}")
    for index, (a, b) in enumerate(zip(mine, theirs)):
        mark = "same" if a == b else "DIFFERENT"
        print(f"  row {index}: {mark}  {a}")
    print(f"agree on every row: {agree}")
    return 0 if agree else 1


if __name__ == "__main__":
    raise SystemExit(main())

#!/usr/bin/env python3
"""wrangle.py — YOUR working file: the data-wrangling pipeline.

The messy file waits for you at data/messy_orders.csv. It contains, on
purpose, every violation you will meet in the wild:

  * a byte-order mark (three invisible bytes) before the header
  * a comma inside a quoted field          ("widget, large")
  * a newline inside a quoted field        (order 1002's notes)
  * doubled quotes standing for one quote  (Alan ""Turing"" Jr.)
  * an empty field                         (order 1004 has no notes)
  * a ragged row with only three fields    (order 1005)

Your job is to load it correctly, clean it, and round-trip it to JSON and
JSON Lines without losing a byte. The pipeline in main() is written for
you; the four numbered exercises below are the pieces it calls.

Everything you write lands in out/ next to this lab. Nothing else on your
machine is touched.

Finish the exercises, then run:
    python3 starter/wrangle.py
    bash tests/run_tests.sh
"""
import csv
import json
from pathlib import Path

LAB_DIR = Path(__file__).resolve().parent.parent


# --- step 1: the naive parser, shown failing -------------------------------

def naive_split_rows(text, delimiter=","):
    """Split each line on the delimiter — the bug factory, provided complete."""
    return [line.split(delimiter) for line in text.splitlines()]


def naive_report(text, expected_columns):
    """Return a list of (line_number, field_count) for lines naive splitting breaks.

    Line numbers start at 1, counting physical lines of the file.
    """
    # Exercise 1: PROVE THE BUG.
    # 1. Start with an empty list called broken.
    # 2. Loop over naive_split_rows(text) with enumerate(..., start=1) so you
    #    get (number, row) pairs.
    # 3. If len(row) != expected_columns, append the tuple (number, len(row)).
    # 4. Return broken.
    # When you run it you will find three bad lines — and, more unsettling,
    # that one badly parsed line has the RIGHT field count and the WRONG data.
    raise NotImplementedError("Exercise 1: implement naive_report")


# --- step 2: parse properly ------------------------------------------------

def read_rows(csv_path, encoding, delimiter):
    """Read the CSV into a list of plain dicts using csv.DictReader."""
    # Exercise 2: PARSE PROPERLY.
    # 1. open(csv_path, "r", encoding=encoding, newline="") — both keyword
    #    arguments matter. encoding is "utf-8-sig" here, which strips the
    #    byte-order mark; newline="" hands newline handling to the csv module
    #    so a quoted newline does not split a row in two.
    # 2. Build reader = csv.DictReader(handle, delimiter=delimiter).
    # 3. Return [dict(row) for row in reader]  (dict() gives you a plain dict).
    # The ragged row will come back with None for its missing values — that is
    # DictReader telling you the truth, and Exercise 3 decides what to do.
    raise NotImplementedError("Exercise 2: implement read_rows")


# --- step 3: clean ---------------------------------------------------------

def clean_row(row, config):
    """Return one tidy record: no None values, stripped text, numbers as numbers."""
    # Exercise 3: CLEAN.
    # 1. Make an empty dict called cleaned.
    # 2. For each column in config["columns"]:
    #      value = row.get(column)
    #      if value is None: use config["defaults"].get(column, "") instead
    #      store value.strip() when it is a string, otherwise value as-is
    #        (hint: isinstance(value, str))
    # 3. For each column in config["numeric_columns"], replace the stored
    #    string with float(...) of it. CSV has no types; you add them here.
    # 4. Return cleaned.
    raise NotImplementedError("Exercise 3: implement clean_row")


def clean_rows(rows, config):
    """Clean every row (provided complete)."""
    return [clean_row(row, config) for row in rows]


# --- step 4: write and read back -------------------------------------------

def write_json(records, path):
    """Write one JSON array (provided complete)."""
    path.write_text(
        json.dumps(records, indent=2, ensure_ascii=False, sort_keys=False) + "\n",
        encoding="utf-8",
    )


def read_json(path):
    """Read the JSON array back (provided complete)."""
    return json.loads(path.read_text(encoding="utf-8"))


def write_jsonl(records, path):
    """Write JSON Lines: one compact JSON object per line, no outer array."""
    # Exercise 4a: WRITE JSON LINES.
    # 1. open(path, "w", encoding="utf-8") in a with-block.
    # 2. For each record, write json.dumps(record, ensure_ascii=False) then "\n".
    #    Do NOT pass indent — a JSON Lines record must be exactly one line.
    raise NotImplementedError("Exercise 4a: implement write_jsonl")


def read_jsonl(path):
    """Read JSON Lines back, one object per non-empty line."""
    # Exercise 4b: READ JSON LINES.
    # 1. Start with an empty list called records.
    # 2. open(path, "r", encoding="utf-8") and loop over the file line by line.
    # 3. Strip each line; skip it if it is empty; otherwise append
    #    json.loads(line) to records.
    # 4. Return records.
    raise NotImplementedError("Exercise 4b: implement read_jsonl")


def write_csv(records, path, columns, delimiter):
    """Write a well-formed CSV with csv.DictWriter (provided complete)."""
    with open(path, "w", encoding="utf-8", newline="") as handle:
        writer = csv.DictWriter(handle, fieldnames=columns, delimiter=delimiter)
        writer.writeheader()
        writer.writerows(records)


def read_csv_typed(path, columns, delimiter, numeric_columns):
    """Read a clean CSV back, restoring the numeric columns (provided complete)."""
    with open(path, "r", encoding="utf-8", newline="") as handle:
        rows = [dict(row) for row in csv.DictReader(handle, delimiter=delimiter)]
    for row in rows:
        for column in numeric_columns:
            row[column] = float(row[column])
    return rows


# --- the pipeline (provided complete) --------------------------------------

def main():
    config = json.loads((LAB_DIR / "data" / "config.json").read_text(encoding="utf-8"))
    csv_path = LAB_DIR / config["input_csv"]
    raw_text = csv_path.read_text(encoding=config["encoding"])
    delimiter = config["delimiter"]
    columns = config["columns"]

    print("=== 1. naive line.split(',') ===")
    broken = naive_report(raw_text, len(columns))
    print(f"lines whose field count is not {len(columns)}: {len(broken)}")
    for number, count in broken:
        print(f"  line {number}: {count} fields")
    naive = naive_split_rows(raw_text)
    print(f"naive row for order 1001: {naive[1]}")
    print(f"naive row for order 1003: {naive[4]}")
    print("(order 1003 has 5 fields, so the count check passes — and the data is still wrong)")

    print()
    print("=== 2. csv.DictReader ===")
    rows = read_rows(csv_path, config["encoding"], delimiter)
    print(f"records parsed: {len(rows)}")
    print(f"first key is 'order_id' (byte-order mark stripped): {list(rows[0])[0] == 'order_id'}")
    print(f"order 1002 notes: {rows[1]['notes']!r}")
    print(f"order 1003 customer: {rows[2]['customer']!r}")
    print(f"order 1005 (ragged) total: {rows[4]['total']!r}")

    print()
    print("=== 3. clean ===")
    records = clean_rows(rows, config)
    for record in records:
        print(f"  {record['order_id']} {record['customer']:<18} {record['total']:>7.2f}")

    print()
    print("=== 4. write and round-trip ===")
    out_dir = LAB_DIR / config["output_dir"]
    out_dir.mkdir(exist_ok=True)
    json_path = out_dir / config["output_json"]
    jsonl_path = out_dir / config["output_jsonl"]
    clean_csv_path = out_dir / config["output_csv"]

    write_json(records, json_path)
    write_jsonl(records, jsonl_path)
    write_csv(records, clean_csv_path, columns, delimiter)

    from_json = read_json(json_path)
    from_jsonl = read_jsonl(jsonl_path)
    from_csv = read_csv_typed(clean_csv_path, columns, delimiter, config["numeric_columns"])

    assert from_json == records, "JSON round trip lost or changed data"
    assert from_jsonl == records, "JSON Lines round trip lost or changed data"
    assert from_csv == records, "CSV round trip lost or changed data"

    print(f"wrote {json_path.name}  ({json_path.stat().st_size} bytes)")
    print(f"wrote {jsonl_path.name} ({jsonl_path.stat().st_size} bytes, {len(records)} lines)")
    print(f"wrote {clean_csv_path.name} ({clean_csv_path.stat().st_size} bytes)")
    print("round trip lossless: JSON yes, JSONL yes, CSV yes")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())

#!/usr/bin/env bash
# Tests for the Day 065 lab. Run from the lab directory:
#   bash tests/run_tests.sh
#
# These checks exercise real behaviour, not file existence: the messy CSV
# really does carry a byte-order mark; naive comma-splitting really does
# mangle it; csv.DictReader really does recover the quoted newline, the
# doubled quotes, and the ragged row; the JSON / JSON Lines / CSV round
# trips really are lossless; and the from-scratch quoting state machine
# really does agree with the standard library on every row.
#
# The reference in examples/ is always tested strictly. The learner's
# starter/ is tested structurally while exercises are unfinished, and to
# the same strict standard once they are complete.
# No network, non-interactive. Exits 0 only if every check passes.
set -u

export PYTHONDONTWRITEBYTECODE=1

lab_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
tmp_dir="$(mktemp -d -t day065-tests.XXXXXX)"
trap 'rm -rf "${tmp_dir}"' EXIT

failures=0
checks=0

check() {
  local label="$1" ok="$2"
  checks=$((checks + 1))
  if [ "${ok}" = "yes" ]; then
    echo "  ok: ${label}"
  else
    echo "  FAIL: ${label}"
    failures=$((failures + 1))
  fi
}

# check_py <label> <module_dir> <python-body>
# Runs an assertion body with module_dir on the import path. A clean exit
# (every assert holds) is a pass.
check_py() {
  local label="$1" module_dir="$2" body="$3"
  if PYTHONPATH="${module_dir}" TMP_DIR="${tmp_dir}" LAB="${lab_dir}" python3 -c "${body}" 2>/dev/null; then
    check "${label}" "yes"
  else
    check "${label}" "no"
  fi
}

# --- the input data itself -------------------------------------------------

echo "Testing the messy input file ..."
check_py "messy_orders.csv really starts with a byte-order mark" "${lab_dir}" \
"import os
raw = open(os.path.join(os.environ['LAB'], 'data', 'messy_orders.csv'), 'rb').read()
assert raw[:3] == b'\xef\xbb\xbf', raw[:3]"

check_py "messy_orders.csv really contains a newline inside a quoted field" "${lab_dir}" \
"import os
text = open(os.path.join(os.environ['LAB'], 'data', 'messy_orders.csv'), encoding='utf-8-sig').read()
assert 'Leave at door.\nRing the bell twice.' in text
assert len(text.splitlines()) == 7  # 6 records spread over 7 physical lines"

check_py "config.json is valid JSON with the five expected columns" "${lab_dir}" \
"import json, os
cfg = json.load(open(os.path.join(os.environ['LAB'], 'data', 'config.json'), encoding='utf-8'))
assert cfg['columns'] == ['order_id', 'customer', 'items', 'notes', 'total']
assert cfg['encoding'] == 'utf-8-sig'"

# --- the pipeline ----------------------------------------------------------

run_wrangle_checks() {
  local module_dir="$1"
  echo "Testing the wrangling pipeline in ${module_dir} ..."

  check_py "naive splitting breaks three physical lines" "${module_dir}" \
"import os, wrangle
text = open(os.path.join(os.environ['LAB'], 'data', 'messy_orders.csv'), encoding='utf-8-sig').read()
broken = wrangle.naive_report(text, 5)
assert [n for n, _ in broken] == [2, 4, 7], broken
assert dict(broken)[2] == 6 and dict(broken)[4] == 2 and dict(broken)[7] == 3"

  check_py "naive splitting also corrupts a line whose field count looks right" "${module_dir}" \
"import os, wrangle
text = open(os.path.join(os.environ['LAB'], 'data', 'messy_orders.csv'), encoding='utf-8-sig').read()
row = wrangle.naive_split_rows(text)[4]
assert len(row) == 5
assert row[1] == '\"Alan \"\"Turing\"\" Jr.\"'  # quotes never removed, escape never resolved"

  check_py "DictReader returns five records and strips the byte-order mark" "${module_dir}" \
"import os, wrangle
rows = wrangle.read_rows(os.path.join(os.environ['LAB'], 'data', 'messy_orders.csv'), 'utf-8-sig', ',')
assert len(rows) == 5, len(rows)
assert list(rows[0])[0] == 'order_id', list(rows[0])[0]"

  check_py "the quoted newline, the embedded comma and the doubled quotes survive" "${module_dir}" \
"import os, wrangle
rows = wrangle.read_rows(os.path.join(os.environ['LAB'], 'data', 'messy_orders.csv'), 'utf-8-sig', ',')
assert rows[0]['items'] == 'widget, large'
assert rows[1]['notes'] == 'Leave at door.\nRing the bell twice.'
assert rows[2]['customer'] == 'Alan \"Turing\" Jr.'"

  check_py "the ragged row comes back with None for its missing fields" "${module_dir}" \
"import os, wrangle
rows = wrangle.read_rows(os.path.join(os.environ['LAB'], 'data', 'messy_orders.csv'), 'utf-8-sig', ',')
assert rows[4]['order_id'] == '1005'
assert rows[4]['notes'] is None and rows[4]['total'] is None"

  check_py "cleaning fills defaults and turns total into a real float" "${module_dir}" \
"import json, os, wrangle
cfg = json.load(open(os.path.join(os.environ['LAB'], 'data', 'config.json'), encoding='utf-8'))
rows = wrangle.read_rows(os.path.join(os.environ['LAB'], 'data', 'messy_orders.csv'), 'utf-8-sig', ',')
recs = wrangle.clean_rows(rows, cfg)
assert all(v is not None for r in recs for v in r.values())
assert recs[4]['notes'] == '' and recs[4]['total'] == 0.0
assert isinstance(recs[0]['total'], float) and recs[0]['total'] == 49.5"

  check_py "the JSON round trip is lossless" "${module_dir}" \
"import json, os, wrangle
from pathlib import Path
cfg = json.load(open(os.path.join(os.environ['LAB'], 'data', 'config.json'), encoding='utf-8'))
rows = wrangle.read_rows(os.path.join(os.environ['LAB'], 'data', 'messy_orders.csv'), 'utf-8-sig', ',')
recs = wrangle.clean_rows(rows, cfg)
p = Path(os.environ['TMP_DIR']) / 'rt.json'
wrangle.write_json(recs, p)
assert wrangle.read_json(p) == recs"

  check_py "the JSON Lines round trip is lossless and is one line per record" "${module_dir}" \
"import json, os, wrangle
from pathlib import Path
cfg = json.load(open(os.path.join(os.environ['LAB'], 'data', 'config.json'), encoding='utf-8'))
rows = wrangle.read_rows(os.path.join(os.environ['LAB'], 'data', 'messy_orders.csv'), 'utf-8-sig', ',')
recs = wrangle.clean_rows(rows, cfg)
p = Path(os.environ['TMP_DIR']) / 'rt.jsonl'
wrangle.write_jsonl(recs, p)
assert wrangle.read_jsonl(p) == recs
lines = [ln for ln in p.read_text(encoding='utf-8').splitlines() if ln.strip()]
assert len(lines) == len(recs) == 5, len(lines)
assert all(json.loads(ln) for ln in lines)"

  check_py "the CSV round trip is lossless once types are restored" "${module_dir}" \
"import json, os, wrangle
from pathlib import Path
cfg = json.load(open(os.path.join(os.environ['LAB'], 'data', 'config.json'), encoding='utf-8'))
rows = wrangle.read_rows(os.path.join(os.environ['LAB'], 'data', 'messy_orders.csv'), 'utf-8-sig', ',')
recs = wrangle.clean_rows(rows, cfg)
p = Path(os.environ['TMP_DIR']) / 'rt.csv'
wrangle.write_csv(recs, p, cfg['columns'], ',')
assert wrangle.read_csv_typed(p, cfg['columns'], ',', cfg['numeric_columns']) == recs
assert '\"widget, large\"' in p.read_text(encoding='utf-8')  # DictWriter re-quotes for us"
}

run_parser_checks() {
  local module_dir="$1"
  echo "Testing the from-scratch quoting state machine in ${module_dir} ..."

  check_py "the from-scratch parser agrees with the csv module on the messy file" "${module_dir}" \
"import os, csv_field_parser as p
text = open(os.path.join(os.environ['LAB'], 'data', 'messy_orders.csv'), encoding='utf-8-sig').read()
mine, theirs, agree = p.compare(text)
assert agree, list(zip(mine, theirs))
assert len(mine) == 6"

  check_py "the from-scratch parser handles embedded commas, newlines and doubled quotes" "${module_dir}" \
"import csv_field_parser as p
assert p.parse_csv('a,\"b,c\",d\n') == [['a', 'b,c', 'd']]
assert p.parse_csv('a,\"line1\nline2\",d\n') == [['a', 'line1\nline2', 'd']]
assert p.parse_csv('a,\"say \"\"hi\"\"\",d\n') == [['a', 'say \"hi\"', 'd']]"

  check_py "the from-scratch parser handles empty fields, no trailing newline and CRLF" "${module_dir}" \
"import csv_field_parser as p
assert p.parse_csv('a,,c\n') == [['a', '', 'c']]
assert p.parse_csv('a,b,c') == [['a', 'b', 'c']]
assert p.parse_csv('a,b\r\nc,d\r\n') == [['a', 'b'], ['c', 'd']]
assert p.parse_csv('a,b,') == [['a', 'b', '']]"

  check_py "the from-scratch parser agrees with the csv module on a semicolon dialect" "${module_dir}" \
"import csv_field_parser as p
text = 'name;amount\nWien;1.234,50\n\"Berlin; Mitte\";2.000,00\n'
mine, theirs, agree = p.compare(text, delimiter=';')
assert agree, list(zip(mine, theirs))
assert mine[2][0] == 'Berlin; Mitte'"
}

run_end_to_end() {
  local script="$1"
  echo "Running ${script} end to end ..."
  local out code
  out="$(cd "${lab_dir}" && python3 "${script}" 2>&1)"
  code=$?
  if [ "${code}" -eq 0 ] && printf '%s' "${out}" | grep -qF "round trip lossless: JSON yes, JSONL yes, CSV yes"; then
    check "${script} runs end to end and reports a lossless round trip" "yes"
  else
    check "${script} runs end to end and reports a lossless round trip" "no"
    echo "    (exit ${code}; output: ${out})"
  fi
}

# --- reference: always strict ---
run_wrangle_checks "${lab_dir}/examples"
run_parser_checks "${lab_dir}/examples"
run_end_to_end "examples/wrangle.py"

check_py "the written out/orders.jsonl parses line by line as JSON" "${lab_dir}/examples" \
"import json, os
from pathlib import Path
p = Path(os.environ['LAB']) / 'out' / 'orders.jsonl'
recs = [json.loads(ln) for ln in p.read_text(encoding='utf-8').splitlines() if ln.strip()]
assert len(recs) == 5
assert recs[2]['customer'] == 'Alan \"Turing\" Jr.'"

# --- learner starter ---
echo "Testing starter/ ..."
starter_wrangle="${lab_dir}/starter/wrangle.py"
starter_parser="${lab_dir}/starter/csv_field_parser.py"

for f in "${starter_wrangle}" "${starter_parser}"; do
  if python3 -c "compile(open('${f}').read(), '${f}', 'exec')" 2>/dev/null; then
    check "$(basename "${f}") is valid Python" "yes"
  else
    check "$(basename "${f}") is valid Python" "no"
  fi
done

if grep -q 'NotImplementedError' "${starter_wrangle}" || grep -q 'NotImplementedError' "${starter_parser}"; then
  echo "Note: starter/ still has unfinished exercises — testing structure only."
  for name in naive_report read_rows clean_row write_jsonl read_jsonl; do
    if grep -q "def ${name}" "${starter_wrangle}"; then
      check "starter defines ${name}" "yes"
    else
      check "starter defines ${name}" "no"
    fi
  done
  if grep -q 'def parse_csv' "${starter_parser}"; then
    check "starter defines parse_csv" "yes"
  else
    check "starter defines parse_csv" "no"
  fi
else
  run_wrangle_checks "${lab_dir}/starter"
  run_parser_checks "${lab_dir}/starter"
  run_end_to_end "starter/wrangle.py"
fi

echo
echo "${checks} checks, ${failures} failure(s)."
[ "${failures}" -eq 0 ]