Hands-on lab — Day 59: Modules, Imports, and Project Layout

The cat sat on the mat.
The dog sat on the log.
The cat and the dog are friends.

"""wordstats — a tiny, multi-module package that counts word frequencies.

This file, ``__init__.py``, is what turns the ``wordstats/`` directory into a
*package*: an importable name that groups several modules. It also defines the
package's public API by re-exporting the useful names from the submodules, so
a caller can write::

    from wordstats import tokenize, top_n

without needing to know that ``tokenize`` lives in ``tokens.py`` and ``top_n``
lives in ``stats.py``. The two imports below are *relative* imports (the
leading dot means "from this same package"), which is the normal way one part
of a package refers to another.
"""
from .tokens import tokenize
from .stats import count_words, top_n

__version__ = "1.0.0"
__all__ = ["tokenize", "count_words", "top_n", "__version__"]

"""__main__.py — the package's entry point: ``python3 -m wordstats FILE``.

Because this file is named ``__main__.py``, running ``python3 -m wordstats``
executes it. It reads a text file (or standard input when no file is given),
asks ``tokens`` and ``stats`` to do their jobs, and prints a small report.

The ``if __name__ == "__main__":`` guard at the very bottom is what lets this
same file be *imported* (to reuse ``report`` in a test) as well as *run* (as a
program): the guarded code runs only when the file is executed directly, not
when another module imports it.
"""
import sys

from .tokens import tokenize
from .stats import top_n


def report(text, n=5):
    """Return the printable top-n report for a block of text (no input/output).

    Keeping this pure — text in, string out, nothing read or printed — is what
    makes it importable and testable without a file or a terminal.
    """
    words = tokenize(text)
    lines = [f"{len(words)} words, {len(set(words))} unique"]
    for rank, (word, count) in enumerate(top_n(words, n), start=1):
        lines.append(f"{rank}. {word}: {count}")
    return "\n".join(lines)


def main(argv):
    """Read the file named on the command line (or stdin), print the report."""
    if len(argv) > 1:
        with open(argv[1], "r", encoding="utf-8") as handle:
            text = handle.read()
    else:
        text = sys.stdin.read()
    print(report(text))
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv))

"""stats.py — one responsibility: turn a list of words into frequency numbers.

Like ``tokens``, this module depends only on the standard library and does not
import its siblings. It takes plain Python lists in and returns plain Python
values out, so it can be imported and tested on its own.
"""
from collections import Counter


def count_words(words):
    """Return a plain dict mapping each word to how many times it appears."""
    return dict(Counter(words))


def top_n(words, n=5):
    """Return the n most common (word, count) pairs, most frequent first.

    Ties are broken alphabetically so the result is deterministic on every
    machine. ``top_n(["a", "b", "a"], 1)`` returns ``[('a', 2)]``.
    """
    counts = count_words(words)
    ordered = sorted(counts.items(), key=lambda pair: (-pair[1], pair[0]))
    return ordered[:n]

"""tokens.py — one responsibility: turn raw text into a clean list of words.

This module knows nothing about counting, reporting, or the command line. It
imports only the standard library and never imports its sibling ``stats`` — a
module that does one job and does not depend on the rest of the package is the
easiest to test and the least likely to cause a circular import.
"""
import re

# Matches runs of letters, digits, and apostrophes, so "don't" stays one word.
_WORD_RE = re.compile(r"[a-z0-9']+")


def normalize(text):
    """Return text lowercased — a tiny, separately testable step."""
    return text.lower()


def tokenize(text):
    """Split text into a list of lowercase word tokens.

    Punctuation and whitespace act as separators; internal apostrophes are
    kept. ``tokenize("Hello, HELLO world!")`` returns
    ``['hello', 'hello', 'world']``.
    """
    return _WORD_RE.findall(normalize(text))

lesson_id: D059
day: 59
kind: python-program
languages: [python]
setup_commands:
  - cd labs/sections/programming-with-python/day-059-modules-imports-and-project-layout
  - python3 --version
run_commands:
  - cd examples && python3 -m wordstats sample.txt
  - printf 'red red blue green red blue\n' | (cd examples && python3 -m wordstats)
  - python3 -c "import sys; sys.path.insert(0, 'examples'); from wordstats import tokenize, top_n; print(top_n(tokenize('a a b'), 2))"
test_commands:
  - bash tests/run_tests.sh
cleanup_commands:
  - find . -name __pycache__ -exec rm -rf {} +
  - 'git checkout -- starter/  # optional: reset your work'
requires_network: false
requires_api_key: false
estimated_minutes: 30
last_executed: '2026-07-13'
executed_on: 'macOS (Apple Silicon), Python 3.14.0, bash tests/run_tests.sh -> 18 checks, 0 failure(s), exit 0'

# Dependencies — Day 059 lab

**Python 3 only. No third-party packages.**

- `python3` (3.8 or newer; tested on 3.14.0). Preinstalled on most Linux
  distributions and installable on macOS; you set this up on Day 43.
- `bash` for the test runner (preinstalled on macOS and Linux).
- Only the Python standard library is used — specifically `re`, `collections`
  (for `Counter`), `sys`, and `importlib` (used only inside the tests). All of
  these ship with Python. There is deliberately no `requirements.txt`: a
  package this size should run on a plain Python install with nothing to
  download first.

Check your Python is present and new enough:

```bash
python3 --version
```

If that prints `Python 3.8` or higher, you are ready. Windows users: run the
commands inside WSL, or use `python` in place of `python3` if that is how
Python is exposed on your system. The package itself is pure standard-library
Python and behaves identically everywhere.

# Module-layout worksheet

Fill this in *before* you refactor `single_file.py`. Deciding the split on
paper first — which responsibility goes in which module, and which module
imports which — is exactly the discipline the lesson teaches. A layout you can
draw is a layout you can test.

## The program

- **What the whole program does (one sentence):**
- **Where the input comes from (file argument / standard input):**
- **What it prints:**

## Responsibilities → modules

List each distinct responsibility in the single-file program and the module it
will live in. One responsibility per module; a module should do one job.

| Responsibility (what it does) | Module (`.py` file) | Public functions |
| ----------------------------- | ------------------- | ---------------- |
| Text → list of words          | `tokens.py`         | `tokenize`       |
| Words → frequency numbers      | `stats.py`          | `count_words`, `top_n` |
| Read input, print the report   | `__main__.py`       | `report`, `main` |
| Define the package's public API | `__init__.py`      | (re-exports)     |

## Import direction (avoid circular imports)

Draw the arrows: which module imports which? They must never form a loop.

- `__main__.py` imports from: __________ and __________
- `__init__.py` imports from: __________ and __________
- `tokens.py` imports from: __________ (should be standard library only)
- `stats.py` imports from: __________ (should be standard library only)
- Is there any cycle (A imports B and B imports A)?  yes / no  → if yes, fix it.

## Import styles used

Name where in the package each style appears:

- `import name` (plain): __________
- `from module import name` (from-import): __________
- `import module as alias` (import-as): __________
- Relative import (leading dot, e.g. `from .tokens import tokenize`): __________
- Absolute import (full path, e.g. `from wordstats.tokens import tokenize`): __________

## The entry point

- Which file is run by `python3 -m wordstats`?  __________
- What line makes it run only when executed, not when imported?  __________
- Which function stays pure (no printing, no file reading) so it can be tested?
  __________

## Recorded behaviour (fill in after you build it)

- One good run (command and its output):
- One import (a `python3 -c "from wordstats import ..."` line and its output):

The cat sat on the mat.
The dog sat on the log.
The cat and the dog are friends.

#!/usr/bin/env python3
"""single_file.py — the "before": one script that does everything at once.

This is a working word-frequency tool with tokenizing, counting, the report,
and the command-line handling all tangled together in one block. It runs, but
nothing in it can be imported or tested on its own, because the real work only
happens inside the ``if __name__ == "__main__":`` block. Your task in the lab
is to split this into the ``wordstats/`` package, one responsibility per
module. Run it first to see what it does:

    python3 single_file.py sample.txt
"""
import re
import sys
from collections import Counter

if __name__ == "__main__":
    if len(sys.argv) > 1:
        with open(sys.argv[1], "r", encoding="utf-8") as handle:
            text = handle.read()
    else:
        text = sys.stdin.read()
    words = re.findall(r"[a-z0-9']+", text.lower())
    counts = Counter(words)
    ordered = sorted(counts.items(), key=lambda pair: (-pair[1], pair[0]))
    print(f"{len(words)} words, {len(set(words))} unique")
    for rank, (word, count) in enumerate(ordered[:5], start=1):
        print(f"{rank}. {word}: {count}")

"""wordstats — the package you are building.

The mere presence of this ``__init__.py`` is what makes ``wordstats/`` an
importable package. It should also define the package's public API by
re-exporting the useful names from the submodules, so callers can write
``from wordstats import tokenize, top_n`` without knowing which file each lives
in. The reference is in ``examples/wordstats/__init__.py``.
"""
# Exercise 5: DEFINE THE PUBLIC API.
# Uncomment the two RELATIVE imports below (the leading dot means "this same
# package") so that `from wordstats import tokenize, count_words, top_n` works:
#
# from .tokens import tokenize
# from .stats import count_words, top_n

__version__ = "1.0.0"
__all__ = ["tokenize", "count_words", "top_n", "__version__"]

"""__main__.py — the package's entry point: ``python3 -m wordstats FILE``.

Runs when you execute ``python3 -m wordstats``. It reads a file (or standard
input), asks the sibling modules to do their jobs, and prints a report. The
imports below are RELATIVE (the leading dot means "this same package").
"""
import sys

from .tokens import tokenize
from .stats import top_n


def report(text, n=5):
    """Return the printable top-n report for a block of text (no input/output)."""
    # Exercise 4: THE REPORT (pure: text in, string out — no printing here).
    # 1. words = tokenize(text)
    # 2. Start a list of lines with:
    #        f"{len(words)} words, {len(set(words))} unique"
    # 3. For each (word, count) in top_n(words, n), numbered from 1, append:
    #        f"{rank}. {word}: {count}"
    #    Hint: enumerate(top_n(words, n), start=1) gives (rank, (word, count)).
    # 4. Return "\n".join(lines).
    raise NotImplementedError("Exercise 4: implement report")


def main(argv):
    """Read the file named on the command line (or stdin), print the report."""
    if len(argv) > 1:
        with open(argv[1], "r", encoding="utf-8") as handle:
            text = handle.read()
    else:
        text = sys.stdin.read()
    print(report(text))
    return 0


# Exercise 6: ADD THE MAIN GUARD.
# Below this comment, add the guard so that running the package executes main
# but importing this module (for example to test report) does not:
#
#     if __name__ == "__main__":
#         sys.exit(main(sys.argv))

"""stats.py — one responsibility: turn a list of words into frequency numbers.

Depends only on the standard library; does not import its siblings. Plain
lists in, plain values out. The reference is in ``examples/wordstats/stats.py``.
"""
from collections import Counter


def count_words(words):
    """Return a plain dict mapping each word to how many times it appears."""
    # Exercise 2: COUNT WORDS.
    # Return dict(Counter(words)) — a normal dict of word -> count.
    raise NotImplementedError("Exercise 2: implement count_words")


def top_n(words, n=5):
    """Return the n most common (word, count) pairs, most frequent first."""
    # Exercise 3: TOP N.
    # 1. counts = count_words(words)
    # 2. Sort counts.items() by count DESCENDING, then word ASCENDING, so ties
    #    break alphabetically and the result is deterministic:
    #        sorted(counts.items(), key=lambda pair: (-pair[1], pair[0]))
    # 3. Return the first n of that sorted list.
    raise NotImplementedError("Exercise 3: implement top_n")

"""tokens.py — one responsibility: turn raw text into a clean list of words.

This module should import only the standard library and must NOT import its
sibling ``stats`` — keeping the dependency one-directional is how you avoid
circular imports. The finished reference is in
``examples/wordstats/tokens.py`` — try each exercise before peeking.
"""
import re

# Matches runs of letters, digits, and apostrophes, so "don't" stays one word.
_WORD_RE = re.compile(r"[a-z0-9']+")


def normalize(text):
    """Return text lowercased — a tiny, separately testable step. (Provided.)"""
    return text.lower()


def tokenize(text):
    """Split text into a list of lowercase word tokens."""
    # Exercise 1: TOKENIZE.
    # Return a list of lowercase word tokens found in text. Lowercase first
    # with normalize(text), then use _WORD_RE.findall(...) on the result.
    # tokenize("Hello, HELLO world!") must return ['hello', 'hello', 'world'].
    raise NotImplementedError("Exercise 1: implement tokenize")

#!/usr/bin/env bash
# Tests for the Day 059 lab. Run from the lab directory:
#   bash tests/run_tests.sh
#
# Verifies the reference package (examples/wordstats) by IMPORTING its modules
# and asserting behaviour, by running it as `python3 -m wordstats`, by proving
# each responsibility module loads on its own (no circular imports), and by
# checking the module cache imports a module only once. It then checks the
# learner's starter package — structurally while the exercises are unfinished,
# and to the same strict standard once they are complete. No network, no
# prompts. Exits 0 only if every check passes.
set -u

# Keep the working tree clean: no __pycache__ directories from importing.
export PYTHONDONTWRITEBYTECODE=1

lab_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
examples="${lab_dir}/examples"
starter="${lab_dir}/starter"
failures=0
checks=0

check() {
  local label="$1" ok="$2"
  checks=$((checks + 1))
  if [ "${ok}" = "yes" ]; then
    echo "  ok: ${label}"
  else
    echo "  FAIL: ${label}"
    failures=$((failures + 1))
  fi
}

# assert_py <label> <dir-with-package> <python code>
# Runs the code with <dir> prepended to sys.path; passes when it exits 0.
assert_py() {
  local label="$1" dir="$2" code="$3"
  if python3 -c "import sys; sys.path.insert(0, '${dir}')
${code}" >/dev/null 2>&1; then
    check "${label}" "yes"
  else
    check "${label}" "no"
  fi
}

# assert_run <label> <dir-with-package> <expect_exit> <needle> [stdin]
# Runs `python3 -m wordstats` from <dir> (so the package is found on the path);
# if a 5th argument is given it is piped in as stdin, otherwise sample.txt is
# passed as a file argument. Checks the exit code and that output contains the
# needle.
assert_run() {
  local label="$1" dir="$2" expect_exit="$3" needle="$4" stdin="${5:-}"
  local out code
  if [ -n "${stdin}" ]; then
    out="$(printf '%s\n' "${stdin}" | (cd "${dir}" && python3 -m wordstats) 2>&1)"
  else
    out="$( (cd "${dir}" && python3 -m wordstats sample.txt) 2>&1)"
  fi
  code=$?
  if [ "${code}" -eq "${expect_exit}" ] && printf '%s' "${out}" | grep -qF "${needle}"; then
    check "${label}" "yes"
  else
    check "${label}" "no"
    echo "    (exit ${code}, expected ${expect_exit}; output: ${out})"
  fi
}

# All the behaviour a finished package must have, run against a package dir.
run_pkg_checks() {
  local dir="$1" label="$2"
  echo "Testing package in ${label} ..."
  # The public API is importable from the package root (via __init__.py).
  assert_py "import wordstats, __version__ == 1.0.0" "${dir}" \
    "import wordstats; assert wordstats.__version__ == '1.0.0'"
  # from-import of a name re-exported by __init__.py.
  assert_py "from wordstats import tokenize -> ['hello','hello','world']" "${dir}" \
    "from wordstats import tokenize; assert tokenize('Hello, HELLO world!') == ['hello', 'hello', 'world'], tokenize('Hello, HELLO world!')"
  # Submodule function, imported directly.
  assert_py "wordstats.stats.count_words counts correctly" "${dir}" \
    "from wordstats.stats import count_words; assert count_words(['x','y','x']) == {'x': 2, 'y': 1}"
  # Ties break alphabetically -> deterministic.
  assert_py "top_n orders by count then alphabetically" "${dir}" \
    "from wordstats.stats import top_n; assert top_n(['b','a','b','a','c'], 2) == [('a', 2), ('b', 2)], top_n(['b','a','b','a','c'], 2)"
  # import ... as ... binds the module under a new name.
  assert_py "import wordstats.stats as st (import-as)" "${dir}" \
    "import wordstats.stats as st; assert st.count_words(['q']) == {'q': 1}"
  # The module cache: importing twice yields the very same module object.
  assert_py "module imported once (sys.modules cache)" "${dir}" \
    "import wordstats.tokens as a; import wordstats.tokens as b; import sys; assert a is b and 'wordstats.tokens' in sys.modules"
  # tokens.py stands alone: loaded outside the package it still works, proving
  # it does not import its siblings (no circular import).
  assert_py "tokens.py is self-contained (no sibling import)" "${dir}" \
    "import importlib.util as u; s=u.spec_from_file_location('t', '${dir}/wordstats/tokens.py'); m=u.module_from_spec(s); s.loader.exec_module(m); assert m.tokenize('A a') == ['a', 'a']"
  assert_py "stats.py is self-contained (no sibling import)" "${dir}" \
    "import importlib.util as u; s=u.spec_from_file_location('s', '${dir}/wordstats/stats.py'); m=u.module_from_spec(s); s.loader.exec_module(m); assert m.count_words(['a','a']) == {'a': 2}"
  # report() is importable and pure — the payoff of the __main__ guard.
  assert_py "report() importable and pure (guard holds main back)" "${dir}" \
    "from wordstats.__main__ import report; r = report('a a b'); assert r.startswith('3 words, 2 unique'), r"
  # Runnable as a program: file argument and standard input.
  assert_run "python3 -m wordstats sample.txt -> counts" "${dir}" 0 "19 words, 10 unique"
  assert_run "python3 -m wordstats sample.txt -> top word" "${dir}" 0 "1. the: 6"
  assert_run "python3 -m wordstats (stdin)" "${dir}" 0 "1. red: 3" "red red blue green red blue"
}

# --- Reference package: always tested strictly ---
run_pkg_checks "${examples}" "examples/"

# --- Learner starter ---
echo "Testing starter/ ..."
starter_valid="yes"
for f in "${starter}/single_file.py" "${starter}/wordstats/__init__.py" \
         "${starter}/wordstats/tokens.py" "${starter}/wordstats/stats.py" \
         "${starter}/wordstats/__main__.py"; do
  python3 -c "compile(open('${f}').read(), '${f}', 'exec')" 2>/dev/null || starter_valid="no"
done
check "starter files are valid Python" "${starter_valid}"

if grep -rq 'NotImplementedError' "${starter}/wordstats"; then
  echo "Note: starter/wordstats still has unfinished exercises — testing structure only."
  grep -q 'def tokenize' "${starter}/wordstats/tokens.py" && check "starter defines tokenize" "yes" || check "starter defines tokenize" "no"
  grep -q 'def count_words' "${starter}/wordstats/stats.py" && check "starter defines count_words" "yes" || check "starter defines count_words" "no"
  grep -q 'def top_n' "${starter}/wordstats/stats.py" && check "starter defines top_n" "yes" || check "starter defines top_n" "no"
  grep -q 'def report' "${starter}/wordstats/__main__.py" && check "starter defines report" "yes" || check "starter defines report" "no"
  test -f "${starter}/wordstats/__init__.py" && check "starter has wordstats/__init__.py" "yes" || check "starter has wordstats/__init__.py" "no"
else
  run_pkg_checks "${starter}" "starter/"
  grep -q '__name__ == "__main__"' "${starter}/wordstats/__main__.py" && check "starter has the main guard" "yes" || check "starter has the main guard" "no"
fi

echo
echo "${checks} checks, ${failures} failure(s)."
[ "${failures}" -eq 0 ]