Hands-on lab — Day 55: Comprehensions and Iterator Thinking

#!/usr/bin/env python3
"""Comprehensions and a lazy generator pipeline.

A complete, small, real program that demonstrates both halves of Day 55:

  1. Comprehensions (list, dict, set) — the everyday transforms that map and
     filter a collection in one readable line.
  2. A lazy generator pipeline — reading, filtering, and aggregating a stream
     of records one at a time, holding only a single record in memory, and
     proving it returns the same answer as a plain eager loop.

Run it:
    python3 examples/pipeline.py

It reads no files and takes no arguments: the "stream" is a small in-memory
list of rows standing in for a data source far too large to hold at once.
"""
import itertools
import sys

# A tiny in-memory "stream". In real life these rows would arrive from a huge
# file or a network socket, one at a time, far too large to hold in memory.
RECORDS = [
    "alice,34,engineering,88",
    "bob,29,design,72",
    "carol,41,engineering,95",
    "dave,38,design,60",
    "erin,25,engineering,79",
    "frank,52,marketing,84",
]


def parse(row):
    """Turn a raw 'name,age,team,score' row into a typed dict."""
    name, age, team, score = row.split(",")
    return {"name": name, "age": int(age), "team": team, "score": int(score)}


# --- Comprehensions: the everyday transforms -------------------------------

def high_scorer_names(records):
    """List comprehension: names of records scoring >= 80, upper-cased."""
    return [r["name"].upper() for r in records if r["score"] >= 80]


def name_to_score(records):
    """Dict comprehension: map each name to its score."""
    return {r["name"]: r["score"] for r in records}


def distinct_teams(records):
    """Set comprehension: the distinct teams present (order not kept)."""
    return {r["team"] for r in records}


# --- Lazy generator pipeline -----------------------------------------------

def read_records(rows):
    """Generator: yield one parsed record at a time (O(1) memory)."""
    for row in rows:
        yield parse(row)


def only_team(records, team):
    """Generator: keep only records for one team, lazily."""
    for r in records:
        if r["team"] == team:
            yield r


def scores(records):
    """Generator: yield just the score of each record."""
    for r in records:
        yield r["score"]


def average_score_lazy(rows, team):
    """Average score for a team via a lazy generator pipeline.

    Each record is read, filtered, and consumed one at a time — the program
    never holds more than a single record, however long `rows` is.
    """
    pipeline = scores(only_team(read_records(rows), team))
    total = 0
    count = 0
    for score in pipeline:
        total += score
        count += 1
    return total / count if count else 0.0


def average_score_loop(rows, team):
    """The same answer computed eagerly with a plain loop (the baseline)."""
    matching = []
    for row in rows:
        record = parse(row)
        if record["team"] == team:
            matching.append(record["score"])
    return sum(matching) / len(matching) if matching else 0.0


def first_ids(n):
    """itertools.count + islice: the first n ids from an endless counter."""
    return list(itertools.islice(itertools.count(1000), n))


def main(argv):
    """Print every result and assert the lazy pipeline matches the loop."""
    team = "engineering"

    print("high scorers (list):  ", high_scorer_names([parse(r) for r in RECORDS]))
    print("name -> score (dict): ", name_to_score([parse(r) for r in RECORDS]))
    print("distinct teams (set): ", sorted(distinct_teams([parse(r) for r in RECORDS])))
    print("first 3 ids (itertools):", first_ids(3))

    lazy = average_score_lazy(RECORDS, team)
    loop = average_score_loop(RECORDS, team)
    print(f"{team} average (lazy pipeline): {lazy:.1f}")
    print(f"{team} average (loop baseline): {loop:.1f}")

    assert lazy == loop, "lazy pipeline must match the loop baseline"
    print("match: lazy pipeline == loop baseline")
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv))

lesson_id: D055
day: 55
kind: data-processing
languages: [python]
setup_commands:
  - cd labs/sections/programming-with-python/day-055-comprehensions-and-iterator-thinking
  - python3 --version
run_commands:
  - python3 examples/pipeline.py
  - python3 starter/pipeline.py
test_commands:
  - bash tests/run_tests.sh
cleanup_commands:
  - 'git checkout -- starter/pipeline.py  # optional: reset your work'
requires_network: false
requires_api_key: false
estimated_minutes: 30
last_executed: '2026-07-13'
executed_on: 'macOS (Apple Silicon), Python 3.14.0, bash tests/run_tests.sh → 12 checks, 0 failure(s), exit 0'

# Dependencies — Day 055 lab

**Python 3 only. No third-party packages.**

- `python3` (3.8 or newer; tested on 3.14). You set this up on Day 43.
- `bash` for the test runner (preinstalled on macOS and Linux).
- Only the Python standard library is used — the `sys` and `itertools`
  modules, both of which ship with Python. There is deliberately no
  `requirements.txt`: comprehensions and generators are core language
  features that run on a plain Python install with nothing to install first.

Check your Python is present and new enough:

```bash
python3 --version
```

If that prints `Python 3.8` or higher, you are ready. Windows users: run the
commands inside WSL, or use `python` in place of `python3` if that is how
Python is exposed on your system. The program is pure standard-library
Python and behaves identically on every platform.

# Pipeline design worksheet — Day 055

Design your second pipeline *before* you code it (practice assignment). Pick
a different set of records — books, transactions, songs — and plan the same
five pieces you built for the sample data. Keep this file; the Week 8 project
(Terminal Task Manager) reuses these transform-and-filter habits.

## 1. Your records

Write 5-8 sample rows and the fields each one has.

| Field | Type | Example |
| ----- | ---- | ------- |
|       |      |         |
|       |      |         |
|       |      |         |
|       |      |         |

## 2. List comprehension (map + filter)

- What do you keep (the `if`)? ____________________
- What do you build for each kept item (the output expression)? ____________________
- The one line:

```python

```

## 3. Dict comprehension (a lookup)

- Key: ____________________  Value: ____________________
- The one line:

```python

```

## 4. Set comprehension (distinct values)

- Which field's distinct values? ____________________
- The one line:

```python

```

## 5. Lazy generator pipeline

- Stage 1 — reader (`yield` one parsed record): ____________________
- Stage 2 — filter (`yield` only records that qualify): ____________________
- Aggregate computed at the end (sum or average of what?): ____________________
- The `assert` that proves the lazy result equals the plain-loop result:

```python
assert lazy_result == loop_result
```

## 6. itertools.islice

- Paste the line that prints just the first two items of your reader, and
  what it printed:

```text

```

## 7. What the program printed (fill in after building)

```text

```

#!/usr/bin/env python3
"""Comprehensions and a lazy generator pipeline — YOUR working file.

Complete the five numbered exercises below. Each names exactly what to write.
The finished reference is in examples/pipeline.py — try each exercise
yourself before peeking.

When all five are done, this file prints the same output as the reference:

    python3 starter/pipeline.py

Then run:  bash tests/run_tests.sh
"""
import itertools
import sys

RECORDS = [
    "alice,34,engineering,88",
    "bob,29,design,72",
    "carol,41,engineering,95",
    "dave,38,design,60",
    "erin,25,engineering,79",
    "frank,52,marketing,84",
]


def parse(row):
    """Turn a raw 'name,age,team,score' row into a typed dict. (Provided.)"""
    name, age, team, score = row.split(",")
    return {"name": name, "age": int(age), "team": team, "score": int(score)}


# --- Comprehensions ---------------------------------------------------------

def high_scorer_names(records):
    """List comprehension: names of records scoring >= 80, upper-cased."""
    # Exercise 1: LIST COMPREHENSION.
    # Return a list of r["name"].upper() for each record r whose
    # r["score"] is >= 80. One line: [ ... for r in records if ... ].
    # Expected on the sample data: ['ALICE', 'CAROL', 'FRANK'].
    raise NotImplementedError("Exercise 1: implement high_scorer_names")


def name_to_score(records):
    """Dict comprehension: map each name to its score."""
    # Exercise 2: DICT COMPREHENSION.
    # Return {r["name"]: r["score"] for r in records}. The key is the name,
    # the value is the score.
    raise NotImplementedError("Exercise 2: implement name_to_score")


def distinct_teams(records):
    """Set comprehension: the distinct teams present."""
    # Exercise 3: SET COMPREHENSION.
    # Return a set of each r["team"]. Duplicates collapse automatically.
    raise NotImplementedError("Exercise 3: implement distinct_teams")


# --- Lazy generator pipeline ------------------------------------------------

def read_records(rows):
    """Generator: yield one parsed record at a time (O(1) memory). (Provided.)"""
    for row in rows:
        yield parse(row)


def only_team(records, team):
    """Generator: keep only records for one team, lazily."""
    # Exercise 4: GENERATOR FUNCTION.
    # Loop over records; for each r whose r["team"] == team, `yield r`.
    # Use yield (not return, not append) so this stays lazy.
    raise NotImplementedError("Exercise 4: implement only_team")


def scores(records):
    """Generator: yield just the score of each record. (Provided.)"""
    for r in records:
        yield r["score"]


def average_score_lazy(rows, team):
    """Average score for a team via a lazy generator pipeline."""
    # Exercise 5: ASSEMBLE THE PIPELINE.
    # Build:  pipeline = scores(only_team(read_records(rows), team))
    # Then loop over the pipeline, summing the scores and counting them,
    # and return total / count (or 0.0 if count is 0). Each record is
    # touched exactly once, one at a time.
    raise NotImplementedError("Exercise 5: implement average_score_lazy")


def average_score_loop(rows, team):
    """The same answer computed eagerly with a plain loop. (Provided baseline.)"""
    matching = []
    for row in rows:
        record = parse(row)
        if record["team"] == team:
            matching.append(record["score"])
    return sum(matching) / len(matching) if matching else 0.0


def first_ids(n):
    """itertools.count + islice: the first n ids from an endless counter. (Provided.)"""
    return list(itertools.islice(itertools.count(1000), n))


def main(argv):
    """Print every result and assert the lazy pipeline matches the loop."""
    team = "engineering"

    print("high scorers (list):  ", high_scorer_names([parse(r) for r in RECORDS]))
    print("name -> score (dict): ", name_to_score([parse(r) for r in RECORDS]))
    print("distinct teams (set): ", sorted(distinct_teams([parse(r) for r in RECORDS])))
    print("first 3 ids (itertools):", first_ids(3))

    lazy = average_score_lazy(RECORDS, team)
    loop = average_score_loop(RECORDS, team)
    print(f"{team} average (lazy pipeline): {lazy:.1f}")
    print(f"{team} average (loop baseline): {loop:.1f}")

    assert lazy == loop, "lazy pipeline must match the loop baseline"
    print("match: lazy pipeline == loop baseline")
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv))

#!/usr/bin/env bash
# Tests for the Day 055 lab. Run from the lab directory:
#   bash tests/run_tests.sh
#
# The checks are assert-based: they import the reference module and assert the
# return values of the comprehensions and the generator pipeline, then run the
# whole program and confirm the lazy pipeline matches the loop baseline.
# Finally they check the learner's starter: structurally while exercises are
# unfinished, and to the same strict standard once they are complete.
# No network, non-interactive. Exits 0 only if every check passes.
set -u

# Keep the working tree clean: do not let imported modules write __pycache__.
export PYTHONDONTWRITEBYTECODE=1

lab_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
ref="${lab_dir}/examples/pipeline.py"
starter="${lab_dir}/starter/pipeline.py"
examples_dir="${lab_dir}/examples"
failures=0
checks=0

check() {
  local label="$1" ok="$2"
  checks=$((checks + 1))
  if [ "${ok}" = "yes" ]; then
    echo "  ok: ${label}"
  else
    echo "  FAIL: ${label}"
    failures=$((failures + 1))
  fi
}

# assert_py <label> <python-code>
# Runs an assert-based snippet against the examples module; passes only if
# python exits 0 (every assert held).
assert_py() {
  local label="$1" code="$2"
  if python3 -c "import sys; sys.path.insert(0, '${examples_dir}'); ${code}" 2>/dev/null; then
    check "${label}" "yes"
  else
    check "${label}" "no"
  fi
}

echo "Asserting comprehension results (examples/pipeline.py) ..."
assert_py "list comp: high scorers upper-cased" \
  "from pipeline import high_scorer_names, parse, RECORDS; recs=[parse(r) for r in RECORDS]; assert high_scorer_names(recs) == ['ALICE','CAROL','FRANK']"
assert_py "dict comp: name -> score" \
  "from pipeline import name_to_score, parse, RECORDS; recs=[parse(r) for r in RECORDS]; assert name_to_score(recs)['carol'] == 95 and len(name_to_score(recs)) == 6"
assert_py "set comp: distinct teams" \
  "from pipeline import distinct_teams, parse, RECORDS; recs=[parse(r) for r in RECORDS]; assert distinct_teams(recs) == {'engineering','design','marketing'}"

echo "Asserting generator behaviour ..."
assert_py "read_records yields one record at a time" \
  "from pipeline import read_records, RECORDS; g=read_records(RECORDS); first=next(g); assert first['name']=='alice' and isinstance(first['score'], int)"
assert_py "only_team filters lazily and is a generator" \
  "import types; from pipeline import only_team, read_records, RECORDS; g=only_team(read_records(RECORDS),'design'); assert isinstance(g, types.GeneratorType); assert [r['name'] for r in g] == ['bob','dave']"
assert_py "lazy pipeline == loop baseline (engineering avg)" \
  "from pipeline import average_score_lazy, average_score_loop, RECORDS; assert average_score_lazy(RECORDS,'engineering') == average_score_loop(RECORDS,'engineering')"
assert_py "engineering average is 262/3" \
  "from pipeline import average_score_lazy, RECORDS; assert abs(average_score_lazy(RECORDS,'engineering') - 262/3) < 1e-9"
assert_py "itertools first_ids uses count+islice" \
  "from pipeline import first_ids; assert first_ids(3) == [1000,1001,1002]"

echo "Running the whole reference program ..."
out="$(python3 "${ref}" 2>&1)"; code=$?
if [ "${code}" -eq 0 ] && printf '%s' "${out}" | grep -qF "match: lazy pipeline == loop baseline"; then
  check "reference program runs and self-checks (exit 0)" "yes"
else
  check "reference program runs and self-checks (exit 0)" "no"
  echo "    (exit ${code}; output: ${out})"
fi

# --- Learner starter ---
echo "Testing starter/pipeline.py ..."
if python3 -c "compile(open('${starter}').read(), '${starter}', 'exec')" 2>/dev/null; then
  check "starter is valid Python" "yes"
else
  check "starter is valid Python" "no"
fi

if grep -q 'NotImplementedError' "${starter}"; then
  echo "Note: starter/pipeline.py still has unfinished exercises — testing structure only."
  grep -q 'def high_scorer_names' "${starter}" && check "starter defines high_scorer_names" "yes" || check "starter defines high_scorer_names" "no"
  grep -q 'yield' "${starter}" && check "starter uses yield somewhere" "yes" || check "starter uses yield somewhere" "no"
else
  # Learner finished: hold the starter to the same strict standard.
  s_out="$(python3 "${starter}" 2>&1)"; s_code=$?
  if [ "${s_code}" -eq 0 ] && printf '%s' "${s_out}" | grep -qF "match: lazy pipeline == loop baseline"; then
    check "completed starter runs and self-checks (exit 0)" "yes"
  else
    check "completed starter runs and self-checks (exit 0)" "no"
    echo "    (exit ${s_code}; output: ${s_out})"
  fi
  if python3 -c "import sys; sys.path.insert(0, '${lab_dir}/starter'); from pipeline import high_scorer_names, parse, RECORDS; recs=[parse(r) for r in RECORDS]; assert high_scorer_names(recs) == ['ALICE','CAROL','FRANK']" 2>/dev/null; then
    check "completed starter list comp is correct" "yes"
  else
    check "completed starter list comp is correct" "no"
  fi
fi

echo
echo "${checks} checks, ${failures} failure(s)."
[ "${failures}" -eq 0 ]