Hands-on lab — Day 21: Inspecting Traffic with curl and Developer Tools

#!/usr/bin/env bash
# Day 021 lab — Debug a Request with curl (completed reference implementation).
#
# Demonstrates the core traffic-inspection skills against two public test
# servers: https://httpbin.org (echoes requests, returns any status you ask
# for) and https://example.com. Every section maps to a stage of the
# failed-request decision tree from the lesson.
#
# Run from the lab directory:  bash examples/debug_request.sh
#
# Requires network access. If the network (or the test server) is
# unreachable, each section degrades gracefully with a clear message and the
# script still exits 0, so it is safe to run offline.
set -u

HTTPBIN="https://httpbin.org"
EXAMPLE="https://example.com"

rule() { printf '\n=== %s ===\n' "$1"; }

# Return 0 if we appear to have network access to the test host, else 1.
have_network() {
  curl -s -o /dev/null --max-time 12 "${EXAMPLE}" 2>/dev/null
}

# ---------------------------------------------------------------------------
# 1. curl -v : see the whole conversation (DNS, TCP, TLS, request, response).
# ---------------------------------------------------------------------------
show_verbose() {
  rule "1. curl -v — the full request/response conversation"
  echo "\$ curl -v ${EXAMPLE}   (connection notes and headers shown)"
  # -v prints to stderr; keep the * (connection), > (request) and < (response
  # header) lines and drop the noisy download-progress meter and body.
  curl -v -o /dev/null --max-time 15 "${EXAMPLE}" 2>&1 \
    | grep -E '^[*<>]' \
    | grep -vE 'Trying|Establish|schannel|CApath|CAfile|TLSv.*handshake' \
    || echo "(could not reach ${EXAMPLE})"
}

# ---------------------------------------------------------------------------
# 2. curl -IL : follow a redirect to its final destination, headers only.
# ---------------------------------------------------------------------------
show_redirect() {
  rule "2. curl -IL — follow a redirect (httpbin/redirect/1 -> /get)"
  echo "\$ curl -IL ${HTTPBIN}/redirect/1"
  # Shown as two explicit steps so the output is always clean regardless of
  # httpbin's transient gateway hiccups. -w writes its line ONCE after the
  # final transfer (not per retry), and -o /dev/null discards the header dump,
  # so there are never duplicate lines. A real 302 is not a transient error,
  # so --retry returns it as-is; a transient 5xx is retried away.
  local hop1 code1 loc final
  hop1="$(curl -sI -o /dev/null --max-time 25 --retry 5 --retry-delay 1 \
    -w '%{http_code} %{redirect_url}' "${HTTPBIN}/redirect/1" 2>/dev/null)" || hop1=""
  if [ -z "${hop1}" ]; then echo "(could not reach ${HTTPBIN})"; return; fi
  read -r code1 loc <<< "${hop1}"
  echo "first request:  HTTP ${code1}  ->  location: ${loc:-<none>}"
  final="$(curl -o /dev/null -sIL --max-time 25 --retry 5 --retry-delay 1 \
    -w '%{http_code}' "${HTTPBIN}/redirect/1" 2>/dev/null)" || final="000"
  echo "after -L follows the location header, final status: HTTP ${final}"
}

# Describe a status code by its class — always truthful about what came back.
classify_status() {
  case "$1" in
    2*) echo "success" ;;
    3*) echo "a redirect — follow it with -L" ;;
    4*) echo "a CLIENT error — fix your request (URL, auth, or body)" ;;
    5*) echo "a SERVER error — the server failed a valid request" ;;
    *)  echo "no valid response (timeout or connection failure)" ;;
  esac
}

# ---------------------------------------------------------------------------
# 3. curl -w : timing breakdown — which stage was slow?
# ---------------------------------------------------------------------------
show_timing() {
  rule "3. curl -w — timing breakdown (seconds)"
  echo "\$ curl -o /dev/null -s -w '...' ${EXAMPLE}"
  curl -o /dev/null -s --max-time 15 \
    -w 'dns:%{time_namelookup}  connect:%{time_connect}  tls:%{time_appconnect}  ttfb:%{time_starttransfer}  total:%{time_total}\n' \
    "${EXAMPLE}" \
    || echo "(could not reach ${EXAMPLE})"
}

# ---------------------------------------------------------------------------
# 4. curl -H : prove which headers you actually sent (echoed back).
# ---------------------------------------------------------------------------
show_headers_echo() {
  rule "4. curl -H — the headers endpoint echoes what you sent"
  echo "\$ curl -H 'Accept: application/json' ${HTTPBIN}/headers"
  # -f (--fail) makes curl print nothing and return non-zero on an HTTP error,
  # so a transient 503 page is never dumped; --retry rides through the hiccup.
  curl -sf --max-time 25 --retry 4 --retry-delay 2 -H "Accept: application/json" "${HTTPBIN}/headers" \
    || echo "(could not reach ${HTTPBIN} — transient server error; try again in a moment)"
  echo
}

# ---------------------------------------------------------------------------
# 5. Read status codes deliberately: a 4xx (client) and a 5xx (server).
# ---------------------------------------------------------------------------
show_status_codes() {
  rule "5. Reading status codes — a deliberate 404 and 500"
  echo "\$ curl -o /dev/null -s -w '%{http_code}' ${HTTPBIN}/status/404"
  local code404 code500
  # --retry 5 rides through httpbin's transient gateway 5xx (502/503/504) to
  # get the deterministic 404; a real 404 is NOT a transient error, so curl
  # returns it without retrying.
  code404="$(curl -o /dev/null -s --max-time 25 --retry 5 --retry-delay 1 -w '%{http_code}' "${HTTPBIN}/status/404")" || code404="000"
  printf '404 endpoint returned: HTTP %s  — %s\n' "${code404}" "$(classify_status "${code404}")"
  # No --retry here: curl treats 5xx as transient and would retry it away;
  # we WANT to see the server error, whatever 5xx httpbin returns this run.
  echo "\$ curl -o /dev/null -s -w '%{http_code}' ${HTTPBIN}/status/500"
  code500="$(curl -o /dev/null -s --max-time 25 -w '%{http_code}' "${HTTPBIN}/status/500")" || code500="000"
  printf '500 endpoint returned: HTTP %s  — %s\n' "${code500}" "$(classify_status "${code500}")"
  echo "Note: httpbin's error endpoints can themselves return 502/503/504 under load — still a 5xx, same lesson."
}

# ---------------------------------------------------------------------------
# 6. A systematic diagnose function: walk the decision tree for any URL.
# ---------------------------------------------------------------------------
diagnose() {
  local url="$1"
  rule "6. diagnose() — walk the decision tree for ${url}"
  # Ask curl for the status code and each timing gate in one shot.
  local out code namelookup connect appconnect
  out="$(curl -o /dev/null -s --max-time 25 \
    -w '%{http_code} %{time_namelookup} %{time_connect} %{time_appconnect}' \
    "${url}" 2>/dev/null)" || out=""

  if [ -z "${out}" ]; then
    echo "Gate 1 (DNS/connect): FAILED — host did not resolve or connect. Check the name and your network."
    return
  fi
  read -r code namelookup connect appconnect <<< "${out}"

  # Gate 1: DNS resolved if namelookup time is > 0.
  if awk "BEGIN{exit !(${namelookup} > 0)}"; then
    echo "Gate 1 (DNS):     ok  — resolved in ${namelookup}s"
  else
    echo "Gate 1 (DNS):     FAILED — hostname did not resolve"; return
  fi
  # Gate 2: TCP connected.
  if awk "BEGIN{exit !(${connect} > 0)}"; then
    echo "Gate 2 (connect): ok  — TCP connected in ${connect}s"
  else
    echo "Gate 2 (connect): FAILED — could not open a connection"; return
  fi
  # Gate 3: TLS handshake (only for https URLs).
  case "${url}" in
    https://*)
      if awk "BEGIN{exit !(${appconnect} > 0)}"; then
        echo "Gate 3 (TLS):     ok  — handshake done in ${appconnect}s"
      else
        echo "Gate 3 (TLS):     FAILED — certificate or handshake problem"; return
      fi ;;
    *) echo "Gate 3 (TLS):     n/a — not an https URL" ;;
  esac
  # Gate 4: status class.
  case "${code}" in
    2*) echo "Gate 4 (status):  ok  — HTTP ${code} (success)" ;;
    3*) echo "Gate 4 (status):  HTTP ${code} — a redirect; re-run with -L to follow it" ;;
    4*) echo "Gate 4 (status):  HTTP ${code} — CLIENT error: fix your request (URL, auth, or body)" ;;
    5*) echo "Gate 4 (status):  HTTP ${code} — SERVER error: the server failed a valid request" ;;
    *)  echo "Gate 4 (status):  HTTP ${code} — no valid response" ;;
  esac
}

main() {
  echo "Day 021 — Debug a Request with curl"
  echo "Test servers: ${EXAMPLE} and ${HTTPBIN}"
  if ! have_network; then
    echo
    echo "OFFLINE: could not reach ${EXAMPLE}. The commands below need network"
    echo "access. Connect to the internet and re-run: bash examples/debug_request.sh"
    exit 0
  fi
  show_verbose
  show_redirect
  show_timing
  show_headers_echo
  show_status_codes
  diagnose "${EXAMPLE}"
  diagnose "${HTTPBIN}/status/404"
  echo
  echo "Done. Every section above maps to a stage of the failed-request decision tree."
}

main "$@"

lesson_id: D021
day: 21
kind: api-example
languages: [bash]
setup_commands:
  - cd labs/sections/computing-foundations/day-021-inspecting-traffic-with-curl-and-developer
run_commands:
  - bash examples/debug_request.sh
  - bash starter/debug_request.sh
test_commands:
  - bash tests/run_tests.sh
cleanup_commands:
  - 'git checkout -- starter/debug_request.sh  # optional: reset your work'
requires_network: true
requires_api_key: false
estimated_minutes: 30
last_executed: '2026-07-12'
executed_on: 'macOS (curl 8.7.1), online, bash tests/run_tests.sh → 15 checks, 0 failure(s), 0 skip(s)'

# Dependencies — Day 021 lab

**`curl` only** — preinstalled on macOS and mainstream Linux (and available
on Windows 10+ / WSL). No package installs, no accounts, no API keys.

The lab reaches two free public test services — `https://example.com` and
`https://httpbin.org` — so it needs network access to show live output. With
no network it degrades gracefully: the scripts print a clear message and the
tests skip (never fail) the live checks. No sudo required.

#!/usr/bin/env bash
# Day 021 starter — Debug a Request with curl.
# Complete the five exercises below. Each names the exact curl command to use.
# Run from the lab directory:  bash starter/debug_request.sh
# The completed reference is in ../examples/debug_request.sh — try it yourself first.
set -u

EXAMPLE="https://example.com"
HTTPBIN="https://httpbin.org"
rule() { printf '\n=== %s ===\n' "$1"; }

echo "Day 021 starter — Debug a Request with curl"

# Exercise 1: show the full conversation.
# Replace the echo with: curl -v -o /dev/null --max-time 15 "$EXAMPLE" 2>&1 | grep -E '^[*<>]'
rule "1. Verbose conversation"
echo "EXERCISE — your turn: run curl -v against $EXAMPLE and show the * < > lines"

# Exercise 2: follow a redirect.
# Replace with: curl -o /dev/null -sIL --max-time 25 -w '%{http_code}\n' "$HTTPBIN/redirect/1"
rule "2. Follow a redirect"
echo "EXERCISE — your turn: follow $HTTPBIN/redirect/1 with -L and print the final status"

# Exercise 3: timing breakdown.
# Replace with: curl -o /dev/null -s --max-time 15 -w 'dns:%{time_namelookup} connect:%{time_connect} tls:%{time_appconnect} total:%{time_total}\n' "$EXAMPLE"
rule "3. Timing breakdown"
echo "EXERCISE — your turn: print the DNS/connect/TLS/total timings for $EXAMPLE"

# Exercise 4: echo the headers you sent.
# Replace with: curl -sf --max-time 25 --retry 4 -H "Accept: application/json" "$HTTPBIN/headers"
rule "4. Headers echo"
echo "EXERCISE — your turn: send an Accept header to $HTTPBIN/headers and show the echo"

# Exercise 5: read a deliberate 404 status code.
# Replace with: curl -o /dev/null -s --max-time 25 --retry 5 -w '%{http_code}\n' "$HTTPBIN/status/404"
rule "5. Read a status code"
echo "EXERCISE — your turn: print the status code returned by $HTTPBIN/status/404"

echo
echo "When all five are done, compare your output with ../examples/debug_request.sh"

# Debug worksheet — Day 021

Run the commands (from the examples or your completed starter) and record what
YOUR machine and network returned. Values vary — that is expected.

## 1. A redirect you followed
- Command: `curl -IL https://httpbin.org/redirect/1`
- First response status: `____`  (should be a 3xx)
- Final status after `-L` follows it: `____`  (should be 200)

## 2. A timing breakdown
- Command: `curl -o /dev/null -s -w 'dns:%{time_namelookup} connect:%{time_connect} tls:%{time_appconnect} total:%{time_total}\n' https://example.com`
- DNS: `____`  connect: `____`  TLS: `____`  total: `____`
- Which stage took the longest? `____`

## 3. A status code you read deliberately
- Command: `curl -o /dev/null -s -w '%{http_code}\n' https://httpbin.org/status/503`
- Status returned: `____`  — which class is it, and what does it mean? `____`

## 4. One sentence
Using the decision tree (DNS → connect → TLS → status), describe how you would
diagnose a request that "just fails."

> _your answer here_

#!/usr/bin/env bash
# Tests for the Day 021 lab. Run from the lab directory:
#   bash tests/run_tests.sh
#
# Two kinds of checks:
#   * Structure checks always run and must pass (files present, the example
#     script defines the expected sections, the starter names its exercises,
#     both scripts parse).
#   * Network checks run only when the internet is reachable. Offline, they
#     are SKIPPED with a message. When online but the public test server
#     (httpbin.org) is transiently unavailable, the affected check is also
#     SKIPPED rather than failed — an external outage is not the learner's bug.
#
# The script exits 0 when no structure check failed, whether online or offline.
set -u

lab_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
HTTPBIN="https://httpbin.org"
EXAMPLE="https://example.com"
failures=0
checks=0
skips=0

pass() { checks=$((checks + 1)); echo "  ok:   $1"; }
fail() { checks=$((checks + 1)); failures=$((failures + 1)); echo "  FAIL: $1"; }
skip() { skips=$((skips + 1)); echo "  skip: $1"; }

echo "== Structure checks =="
example="${lab_dir}/examples/debug_request.sh"
starter="${lab_dir}/starter/debug_request.sh"

[ -f "${example}" ] && pass "example script exists" || fail "example script missing"
[ -f "${starter}" ] && pass "starter script exists" || fail "starter script missing"
[ -f "${lab_dir}/starter/debug-worksheet.md" ] && pass "worksheet exists" || fail "worksheet missing"

# The example must exercise each core flag/section.
for needle in "curl -v" "curl -IL" "%{http_code}" "time_namelookup" "diagnose"; do
  if grep -q -- "${needle}" "${example}"; then pass "example uses '${needle}'"; else fail "example missing '${needle}'"; fi
done

# The starter must name its five numbered exercises.
ex_count="$(grep -cE 'Exercise [1-5]' "${starter}" 2>/dev/null || true)"
if [ "${ex_count:-0}" -ge 5 ]; then pass "starter has 5 numbered exercises"; else fail "starter has 5 numbered exercises (found ${ex_count:-0})"; fi

# Both scripts must be valid bash.
if bash -n "${example}" 2>/dev/null; then pass "example has valid bash syntax"; else fail "example has a syntax error"; fi
if bash -n "${starter}" 2>/dev/null; then pass "starter has valid bash syntax"; else fail "starter has a syntax error"; fi

echo
echo "== Network checks =="
if ! curl -s -o /dev/null --max-time 12 "${EXAMPLE}" 2>/dev/null; then
  skip "no network access — skipping all live-request checks (expected offline)"
else
  pass "network reachable (${EXAMPLE} responded)"

  # Check 1: curl follows a redirect through to a final 200 (redirect/1 -> /get).
  final="$(curl -o /dev/null -sIL --max-time 25 --retry 5 --retry-delay 1 \
    -w '%{http_code}' "${HTTPBIN}/redirect/1" 2>/dev/null)" || final="000"
  case "${final}" in
    200) pass "curl -L followed the redirect to a final HTTP 200" ;;
    5* | 000) skip "redirect check — httpbin transiently unavailable (got '${final}'); retry later" ;;
    *) fail "curl -L should reach HTTP 200 (got '${final}')" ;;
  esac

  # Check 2: reads a 404 status correctly. --retry rides past transient 5xx;
  # a real 404 is not a transient error, so curl returns it without retrying.
  code404="$(curl -o /dev/null -s --max-time 25 --retry 5 --retry-delay 1 \
    -w '%{http_code}' "${HTTPBIN}/status/404" 2>/dev/null)" || code404="000"
  case "${code404}" in
    404) pass "curl read the deliberate 404 status correctly" ;;
    5* | 000) skip "404 check — httpbin transiently unavailable (got '${code404}'); retry later" ;;
    *) fail "curl should read HTTP 404 (got '${code404}')" ;;
  esac

  # Check 3: the headers endpoint echoes back a header we set with -H.
  echoed="$(curl -sf --max-time 25 --retry 5 --retry-delay 1 \
    -H "X-Day-21-Test: inspecting" "${HTTPBIN}/headers" 2>/dev/null)" || echoed=""
  if [ -z "${echoed}" ]; then
    skip "headers-echo check — httpbin transiently unavailable; retry later"
  elif printf '%s' "${echoed}" | grep -q "X-Day-21-Test"; then
    pass "the headers endpoint echoed back the header sent with -H"
  else
    fail "the headers endpoint should echo the -H header we sent"
  fi
fi

echo
echo "${checks} checks, ${failures} failure(s), ${skips} skip(s)."
[ "${failures}" -eq 0 ]