Hands-on lab — Day 182: Writing a Model Report

import json

REQUIRED_SECTIONS = [
    "model_details",
    "intended_use",
    "factors",
    "metrics",
    "evaluation_data",
    "training_data",
    "quantitative_analyses",
    "ethical_considerations",
    "caveats_and_recommendations"
]

def validate_model_card_schema(card_data):
    """
    Validate whether a model card dictionary conforms to the Mitchell et al. 2019 standard.
    """
    missing = [sec for sec in REQUIRED_SECTIONS if sec not in card_data or not card_data[sec]]
    if missing:
        return {
            "is_valid": False,
            "missing_sections": missing,
            "error": f"Model Card is missing mandatory sections: {missing}"
        }
        
    # Verify Intended Use has Out-of-Scope declarations
    intended = card_data.get("intended_use", {})
    if "out_of_scope_uses" not in intended or not intended["out_of_scope_uses"]:
        return {
            "is_valid": False,
            "missing_sections": ["intended_use.out_of_scope_uses"],
            "error": "Model Card MUST explicitly specify out-of-scope / prohibited use cases."
        }
        
    return {"is_valid": True, "missing_sections": []}

def generate_model_card_markdown(card_data):
    """
    Render a production Model Card in GitHub-flavored Markdown.
    """
    val = validate_model_card_schema(card_data)
    if not val["is_valid"]:
        raise ValueError(val["error"])
        
    md = f"""# Model Card: {card_data['model_details']['name']} (v{card_data['model_details']['version']})

## 1. Model Details
- **Organization / Developer:** {card_data['model_details']['developer']}
- **Release Date:** {card_data['model_details']['date']}
- **Model Architecture:** {card_data['model_details']['architecture']}
- **License:** {card_data['model_details']['license']}
- **Contact:** {card_data['model_details']['contact']}

## 2. Intended Use
### Primary Intended Uses
{chr(10).join(f"- {u}" for u in card_data['intended_use']['primary_uses'])}

### Out-of-Scope & Prohibited Uses
{chr(10).join(f"- ⚠️ {u}" for u in card_data['intended_use']['out_of_scope_uses'])}

## 3. Factors & Demographic Subgroups
- **Demographic Slices:** {', '.join(card_data['factors']['demographics'])}
- **Operational Environments:** {', '.join(card_data['factors']['environments'])}

## 4. Metrics & Evaluation Setup
- **Primary Optimization Metric:** {card_data['metrics']['primary_metric']}
- **Decision Threshold:** {card_data['metrics']['decision_threshold']}
- **Baseline Comparison:** {card_data['metrics']['baseline_summary']}

## 5. Evaluation Data
- **Dataset:** {card_data['evaluation_data']['name']} ({card_data['evaluation_data']['sample_count']} records)
- **Validation Split Strategy:** {card_data['evaluation_data']['split_strategy']}

## 6. Training Data
- **Dataset:** {card_data['training_data']['name']} ({card_data['training_data']['sample_count']} records)
- **Ingestion Filters:** {card_data['training_data']['filters']}

## 7. Quantitative Analyses & Slices
| Subgroup Slice | Sample Count | Primary Metric Value | Error Rate |
| --- | --- | --- | --- |
"""
    for row in card_data['quantitative_analyses']['slices']:
        md += f"| {row['slice']} | {row['count']} | {row['metric']} | {row['error_rate']} |\n"
        
    md += f"""
### Fairness Audit Summary
- **Demographic Parity Ratio (Disparate Impact):** {card_data['quantitative_analyses']['fairness']['disparate_impact_ratio']}
- **Equal Opportunity Difference (TPR Delta):** {card_data['quantitative_analyses']['fairness']['equal_opportunity_difference']}

## 8. Ethical Considerations & Risk Mitigations
{chr(10).join(f"- {e}" for e in card_data['ethical_considerations'])}

## 9. Caveats and Recommendations
{chr(10).join(f"- {c}" for c in card_data['caveats_and_recommendations'])}
"""
    return md

import pytest
from model_card_lib import validate_model_card_schema, generate_model_card_markdown

def get_sample_valid_card():
    return {
        "model_details": {
            "name": "ClinicalSepsisClassifier",
            "version": "1.2.0",
            "developer": "Clinical ML Team",
            "date": "2026-08-29",
            "architecture": "LightGBM Classifier",
            "license": "Apache 2.0",
            "contact": "ml-safety@hospital.org"
        },
        "intended_use": {
            "primary_uses": ["ICU inpatient early sepsis deterioration warning"],
            "out_of_scope_uses": ["Pediatric patients (< 18 yrs)", "Autonomous medication dispensing"]
        },
        "factors": {
            "demographics": ["Age", "Biological Sex"],
            "environments": ["Adult ICU Wards"]
        },
        "metrics": {
            "primary_metric": "PR-AUC (0.845)",
            "decision_threshold": "0.35 (Calibrated for 90% Recall)",
            "baseline_summary": "Beats SOFA clinical score heuristic by +0.14 PR-AUC"
        },
        "evaluation_data": {
            "name": "ICU Holdout 2025-2026",
            "sample_count": "5,000",
            "split_strategy": "Purged TimeSeriesSplit with Patient Grouping"
        },
        "training_data": {
            "name": "ICU Ingestion 2020-2024",
            "sample_count": "45,000",
            "filters": "Adult patients with >= 4 vital sign recordings"
        },
        "quantitative_analyses": {
            "slices": [
                {"slice": "Age < 50", "count": "1500", "metric": "0.860", "error_rate": "5.2%"},
                {"slice": "Age >= 50", "count": "3500", "metric": "0.838", "error_rate": "6.8%"}
            ],
            "fairness": {
                "disparate_impact_ratio": "0.94",
                "equal_opportunity_difference": "0.02"
            }
        },
        "ethical_considerations": [
            "Model is an advisory warning; human clinician must confirm all interventions.",
            "Protected demographic features are excluded from feature store."
        ],
        "caveats_and_recommendations": [
            "Model requires vital signs recorded within last 2 hours.",
            "Retrain annually or if ICU admission protocol changes."
        ]
    }

def test_validate_model_card_schema_success():
    card = get_sample_valid_card()
    res = validate_model_card_schema(card)
    assert res["is_valid"] is True
    assert len(res["missing_sections"]) == 0

def test_validate_model_card_schema_missing_out_of_scope():
    card = get_sample_valid_card()
    card["intended_use"]["out_of_scope_uses"] = []
    res = validate_model_card_schema(card)
    assert res["is_valid"] is False
    assert "intended_use.out_of_scope_uses" in res["missing_sections"]

def test_generate_model_card_markdown():
    card = get_sample_valid_card()
    md = generate_model_card_markdown(card)
    assert "# Model Card: ClinicalSepsisClassifier" in md
    assert "## 1. Model Details" in md
    assert "## 7. Quantitative Analyses & Slices" in md
    assert "⚠️ Pediatric patients (< 18 yrs)" in md

lesson_id: D182
day: 182
kind: applied-ml-model-cards-governance
languages:
  - python
setup_commands:
  - python3 -m venv .venv
  - .venv/bin/pip install -r requirements/requirements.txt
run_commands:
  - .venv/bin/python examples/writing_a_model_report_lib.py
test_commands:
  - ./tests/run_tests.sh
cleanup_commands:
  - rm -rf .venv __pycache__ .pytest_cache
requires_network: false
requires_api_key: false
estimated_minutes: 45
last_executed: '2026-08-29'
executed_on: >-
  macOS (Apple Silicon, arm64, CPU only), Python 3.14.0, scikit-learn 1.9.0, pytest 9.1.1 -- bash tests/run_tests.sh -> 4 checks, 0 failure(s), exit 0. Verified Day 182 implementation.

numpy>=1.24.0
scipy>=1.10.0
pytest>=7.4.0

def validate_model_card_schema(card_data):
    # TODO: Implement Mitchell et al. 2019 Model Card schema validator
    pass

def generate_model_card_markdown(card_data):
    # TODO: Render enterprise Markdown Model Card
    pass

import pytest
from model_card_lib import validate_model_card_schema, generate_model_card_markdown

def get_sample_valid_card():
    return {
        "model_details": {
            "name": "ClinicalSepsisClassifier",
            "version": "1.2.0",
            "developer": "Clinical ML Team",
            "date": "2026-08-29",
            "architecture": "LightGBM Classifier",
            "license": "Apache 2.0",
            "contact": "ml-safety@hospital.org"
        },
        "intended_use": {
            "primary_uses": ["ICU inpatient early sepsis deterioration warning"],
            "out_of_scope_uses": ["Pediatric patients (< 18 yrs)", "Autonomous medication dispensing"]
        },
        "factors": {
            "demographics": ["Age", "Biological Sex"],
            "environments": ["Adult ICU Wards"]
        },
        "metrics": {
            "primary_metric": "PR-AUC (0.845)",
            "decision_threshold": "0.35 (Calibrated for 90% Recall)",
            "baseline_summary": "Beats SOFA clinical score heuristic by +0.14 PR-AUC"
        },
        "evaluation_data": {
            "name": "ICU Holdout 2025-2026",
            "sample_count": "5,000",
            "split_strategy": "Purged TimeSeriesSplit with Patient Grouping"
        },
        "training_data": {
            "name": "ICU Ingestion 2020-2024",
            "sample_count": "45,000",
            "filters": "Adult patients with >= 4 vital sign recordings"
        },
        "quantitative_analyses": {
            "slices": [
                {"slice": "Age < 50", "count": "1500", "metric": "0.860", "error_rate": "5.2%"},
                {"slice": "Age >= 50", "count": "3500", "metric": "0.838", "error_rate": "6.8%"}
            ],
            "fairness": {
                "disparate_impact_ratio": "0.94",
                "equal_opportunity_difference": "0.02"
            }
        },
        "ethical_considerations": [
            "Model is an advisory warning; human clinician must confirm all interventions.",
            "Protected demographic features are excluded from feature store."
        ],
        "caveats_and_recommendations": [
            "Model requires vital signs recorded within last 2 hours.",
            "Retrain annually or if ICU admission protocol changes."
        ]
    }

def test_validate_model_card_schema_success():
    card = get_sample_valid_card()
    res = validate_model_card_schema(card)
    assert res["is_valid"] is True
    assert len(res["missing_sections"]) == 0

def test_validate_model_card_schema_missing_out_of_scope():
    card = get_sample_valid_card()
    card["intended_use"]["out_of_scope_uses"] = []
    res = validate_model_card_schema(card)
    assert res["is_valid"] is False
    assert "intended_use.out_of_scope_uses" in res["missing_sections"]

def test_generate_model_card_markdown():
    card = get_sample_valid_card()
    md = generate_model_card_markdown(card)
    assert "# Model Card: ClinicalSepsisClassifier" in md
    assert "## 1. Model Details" in md
    assert "## 7. Quantitative Analyses & Slices" in md
    assert "⚠️ Pediatric patients (< 18 yrs)" in md

#!/bin/bash
set -e
pytest tests/ -v