diff --git a/scripts/gen_ac_traceability.py b/scripts/gen_ac_traceability.py new file mode 100644 index 0000000..1f3f19c --- /dev/null +++ b/scripts/gen_ac_traceability.py @@ -0,0 +1,164 @@ +"""Generate .planning/AC-TRACEABILITY.md from pytest collection. + +Bidirectional AC<->test traceability per Phase 2 / AC-06. + +Forward orphan : declared AC with no test -> visible + (with --check) exit 1 +Backward orphan : test references unknown AC ID -> visible + (with --check) exit 1 +Deferred-hw AC : `validation_method: deferred-hardware` in the AC entry + -> excluded from orphan check, rendered as "DEFERRED (hardware)" + +Usage: + python scripts/gen_ac_traceability.py # write matrix, exit 0 + python scripts/gen_ac_traceability.py --check # CI: exit 1 on drift +""" +from __future__ import annotations + +import argparse +import json +import logging +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +logging.basicConfig(level=logging.INFO, format="%(message)s") +log = logging.getLogger("gen_ac_traceability") + +ROOT = Path(__file__).resolve().parent.parent +AC_DOC = ROOT / "_docs" / "00_problem" / "acceptance_criteria.md" +OUT_MD = ROOT / ".planning" / "AC-TRACEABILITY.md" + +_AC_HEADER_RE = re.compile(r"^\s*-\s*\*\*(AC-(?:\d+\.\d+[a-z]?|NEW-\d+))\*\*") +_DEFERRED_RE = re.compile(r"deferred-hardware", re.IGNORECASE) + + +def collect_acs_from_doc() -> dict[str, dict]: + """Parse _docs/00_problem/acceptance_criteria.md. + + Returns: {ac_id: {"deferred": bool}}. + An AC is `deferred` if the literal token `deferred-hardware` (case-insensitive) + appears anywhere in the block between this AC's header and the next AC's header + (or EOF). + """ + if not AC_DOC.exists(): + log.error("AC doc not found: %s", AC_DOC) + sys.exit(2) + + acs: dict[str, dict] = {} + current_id: str | None = None + for line in AC_DOC.read_text(encoding="utf-8").splitlines(): + m = _AC_HEADER_RE.match(line) + if m: + current_id = m.group(1) + acs[current_id] = {"deferred": False} + continue + if current_id is not None and _DEFERRED_RE.search(line): + acs[current_id]["deferred"] = True + return acs + + +def collect_acs_from_tests() -> dict[str, list[str]]: + """Run `pytest --collect-only --ac-dump=` and parse the JSON. + + Returns: {ac_id: [test_nodeid, ...]}. + """ + with tempfile.NamedTemporaryFile(suffix=".json", delete=False) as tmp: + tmp_path = Path(tmp.name) + try: + result = subprocess.run( + ["pytest", "--collect-only", "-q", f"--ac-dump={tmp_path}", "tests/"], + cwd=ROOT, + check=False, # collection may report 0 errors but exit non-zero on warnings -- handle below + capture_output=True, + text=True, + ) + # Collection errors are a fatal traceability problem; surface stderr. + if result.returncode != 0 and not tmp_path.exists(): + log.error("pytest --collect-only failed:\n%s", result.stderr or result.stdout) + sys.exit(2) + if not tmp_path.exists() or tmp_path.stat().st_size == 0: + return {} + return json.loads(tmp_path.read_text(encoding="utf-8")) + finally: + tmp_path.unlink(missing_ok=True) + + +def render_md(doc_acs: dict[str, dict], test_map: dict[str, list[str]]) -> str: + """Render the AC-TRACEABILITY.md content.""" + declared = sorted(doc_acs) + tested = set(test_map) + declared_set = set(declared) + + lines: list[str] = [ + "# AC Traceability Matrix", + "", + "> Auto-generated by `scripts/gen_ac_traceability.py`. Do not edit by hand.", + "> Run `python scripts/gen_ac_traceability.py` to regenerate after AC doc or test edits.", + "", + f"**ACs declared in acceptance_criteria.md:** {len(declared)}", + f"**ACs covered by at least one test:** {len(declared_set & tested)}", + f"**ACs deferred to hardware:** {sum(1 for a in declared if doc_acs[a]['deferred'])}", + "", + "## AC -> Test mapping", + "", + "| AC ID | Test count | Tests | Status |", + "|-------|-----------|-------|--------|", + ] + for ac_id in declared: + tests = test_map.get(ac_id, []) + if doc_acs[ac_id]["deferred"]: + status = "DEFERRED (hardware)" + elif tests: + status = "OK" + else: + status = "**ORPHAN -- no test**" + tests_str = "
".join(f"`{t}`" for t in tests) if tests else "_none_" + lines.append(f"| {ac_id} | {len(tests)} | {tests_str} | {status} |") + + unknown = sorted(tested - declared_set) + if unknown: + lines += ["", "## Tests reference unknown AC IDs (backward orphans)", ""] + for ac_id in unknown: + for nid in test_map[ac_id]: + lines.append(f"- `{ac_id}` <- `{nid}`") + + return "\n".join(lines) + "\n" + + +def main() -> int: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument( + "--check", "--strict", action="store_true", + help="Exit 1 if any non-deferred AC has 0 tests, or any test references an AC ID not in the doc.", + ) + args = p.parse_args() + + doc_acs = collect_acs_from_doc() + test_map = collect_acs_from_tests() + out = render_md(doc_acs, test_map) + OUT_MD.parent.mkdir(parents=True, exist_ok=True) + OUT_MD.write_text(out, encoding="utf-8") + log.info("wrote %s (%d AC, %d tagged, %d deferred)", + OUT_MD.relative_to(ROOT), + len(doc_acs), + sum(1 for a in doc_acs if test_map.get(a)), + sum(1 for a in doc_acs if doc_acs[a]["deferred"])) + + if args.check: + bad_orphans = sorted( + a for a, meta in doc_acs.items() if not meta["deferred"] and not test_map.get(a) + ) + unknown = sorted(set(test_map) - set(doc_acs)) + if bad_orphans or unknown: + log.error("AC traceability drift detected:") + for a in bad_orphans: + log.error(" ORPHAN AC (no test): %s", a) + for a in unknown: + log.error(" UNKNOWN AC ID in test: %s", a) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main())