diff --git a/_docs/02_tasks/done/AZ-597_scenario_stub_cleanup.md b/_docs/02_tasks/done/AZ-597_scenario_stub_cleanup.md new file mode 100644 index 0000000..2497d8d --- /dev/null +++ b/_docs/02_tasks/done/AZ-597_scenario_stub_cleanup.md @@ -0,0 +1,102 @@ +# Scenario stub cleanup (replay_mode helpers + 13 rewires) + +**Task**: AZ-597_scenario_stub_cleanup +**Name**: Add `runner/helpers/replay_mode.py` + rewire 13 scenarios off local `_resolve_*` / `_drive_*` / `_push_*` stubs +**Description**: After AZ-594/595/596 landed the three core harness helpers, sitl_observer, and the fc_proxy_runtime driver, the last unimplemented layer is a grab-bag of local per-scenario stubs that all share the same FDR-replay no-op pattern. Bundle them into one shared `runner/helpers/replay_mode.py` module so the offline FDR-replay path is end-to-end executable once the SITL replay fixture builder lands. +**Complexity**: 3 points +**Dependencies**: AZ-594, AZ-595, AZ-596 +**Component**: Blackbox Tests / Test Infrastructure (epic AZ-262) +**Tracker**: AZ-597 +**Epic**: AZ-262 (E-BBT) + +## Problem + +Despite the AZ-594/595/596 arc, 13 scenarios still carry local +`_resolve_*` / `_drive_*` / `_push_*` `NotImplementedError` stubs: + +| Stub | Scenarios | +|------|-----------| +| `_resolve_frame_sink()` | 13 (FT-P-01/02/04/05/07/08/09-AP/09-iNav/10/11, FT-N-01/02/03/04) | +| `_resolve_fc_inbound_emitter(fc_adapter[, host])` | 3 (FT-P-02/04/10) | +| `_drive_imu_replay(csv_path)` | 2 (FT-P-07, FT-N-02) | +| `_resolve_frame_period_ms()` | 2 (FT-N-03/04) | +| `_resolve_outage_injection_frames()` | 1 (FT-N-03) | +| `_resolve_gt_per_frame(report)` | 1 (FT-N-01) | +| `_push_single_image_and_observe(...)` | 1 (FT-P-03/14) | + +These are unreachable today (the b75 `sitl_replay_ready` gate skips +before they're called) so this cleanup can land safely under the +unit-test regression gate. The value: once the SITL replay fixture +builder ships, scenarios become runnable with no further per-scenario +edits. + +## Surfaces (`runner/helpers/replay_mode.py`) + +* `NullFrameSink` — implements `FrameSink` protocol. `write_frame` + is a counter; exposes `frames_written: int`. +* `NullFcInboundEmitter` — implements `FcInboundEmitter` protocol. + `emit` is a counter; exposes `samples_emitted: int`. +* `DEFAULT_FRAME_PERIOD_MS = 33` + `default_frame_period_ms() -> int`. +* `load_replay_json(filename: str) -> dict | list` — reads + `${E2E_SITL_REPLAY_DIR}/`. Raises `FileNotFoundError` + when env var unset OR file missing; `ValueError` with file pointer + on malformed JSON. +* `resolve_replay_subdir(name: str) -> Path` — returns + `${E2E_SITL_REPLAY_DIR}//`. Raises `FileNotFoundError` when + env var unset OR directory missing. +* `imu_replay_noop(csv_path: Path) -> None` — no-op stand-in for the + per-scenario `_drive_imu_replay` (IMU is pre-baked into the FDR + archive in replay mode; the CSV path is preserved as a parameter + for diagnostic logging only). + +## Per-scenario rewire pattern + +```python +# Before: +def _resolve_frame_sink(): + raise NotImplementedError(...) + +# After: +from runner.helpers.replay_mode import NullFrameSink +def _resolve_frame_sink(): + return NullFrameSink() +``` + +Same shape for the other six helpers. For the two scenarios that need +scenario-specific JSON (`_resolve_gt_per_frame`, +`_push_single_image_and_observe`), they call +`load_replay_json("gt_per_frame.json")` / +`load_replay_json("single_image_observation.json")` and project the +result into their scenario-local dataclass. + +## Acceptance Criteria + +**AC-1**: `NullFrameSink.write_frame` and `NullFcInboundEmitter.emit` +are pure counters. + +**AC-2**: `load_replay_json` raises `FileNotFoundError` (env unset or +file missing) and `ValueError` (malformed JSON with file pointer). + +**AC-3**: `resolve_replay_subdir` raises `FileNotFoundError` (env +unset or subdir missing). + +**AC-4**: `default_frame_period_ms()` returns 33. + +**AC-5**: All 13 scenarios have local `_resolve_*` / `_drive_*` / +`_push_*` stubs deleted and import from `runner.helpers.replay_mode`. + +**AC-6**: ≥6 unit tests on `replay_mode.py`. + +**AC-7**: Full e2e unit-test suite passes (regression gate). + +## Out of Scope + +* The actual SITL replay fixture builder. +* Live MAVLink router / pymavlink plumbing. + +## Files Touched + +* `e2e/runner/helpers/replay_mode.py` (new) +* `e2e/_unit_tests/helpers/test_replay_mode.py` (new) +* `e2e/_unit_tests/test_directory_layout.py` (register new module) +* 13 scenario files under `e2e/tests/{positive,negative}/` diff --git a/_docs/03_implementation/batch_77_report.md b/_docs/03_implementation/batch_77_report.md new file mode 100644 index 0000000..d752939 --- /dev/null +++ b/_docs/03_implementation/batch_77_report.md @@ -0,0 +1,96 @@ +# Batch 77 Report — replay_mode helpers + 13 scenario stub rewires (cycle 1, batch 11 of test phase) + +**Batch**: 77 +**Date**: 2026-05-17 +**Context**: Test implementation (greenfield Step 10 — Implement Tests) +**Tasks**: AZ-597 (3 cp) — 1 task (scenario stub cleanup bundle) +**Cycle**: 1 +**Verdict**: COMPLETE — PASS (self-reviewed; see `reviews/batch_77_review.md`) + +## Summary + +Closes the offline FDR-replay path that AZ-594 (b74), AZ-595 (b75), +and AZ-596 (b76) opened. After those three batches, the only remaining +`NotImplementedError` stubs in the scenario suite were a grab-bag of +local `_resolve_*` / `_drive_*` / `_push_*` helpers duplicated across +13 scenario files. They all reduced to the same FDR-replay pattern — +either a no-op counter (frame sink, FC inbound emitter, IMU replay +driver) or a JSON read from `${E2E_SITL_REPLAY_DIR}/` (per-frame GT, +single-image observation, outage frames subdir). + +This batch bundles those into one shared `runner/helpers/replay_mode.py` +module + rewires the 13 scenarios off their local stubs. After the +batch: + +* `grep raise NotImplementedError` under `e2e/tests/` returns **zero** + matches. +* Once the SITL replay fixture builder lands (separate ticket), every + scenario becomes runnable end-to-end with no further per-scenario + edits. +* Unit-test mode is unchanged — the b75 `sitl_replay_ready` skip + gate keeps the loaders unreached when `E2E_SITL_REPLAY_DIR` is unset. + +### AZ-597 — replay_mode helpers + 13 scenario rewires (3 cp) + +* **`runner/helpers/replay_mode.py`** (new): + * `NullFrameSink` — counter-only `FrameSink` (`frames_written: int`). + * `NullFcInboundEmitter` — counter-only `FcInboundEmitter` + (`samples_emitted: int`). + * `default_frame_period_ms() -> int` + `DEFAULT_FRAME_PERIOD_MS = 33` + (30 fps). + * `load_replay_json(filename)` — generic JSON loader. Raises + `FileNotFoundError` (env-unset / file-missing) or `ValueError` + (malformed, file pointer included). + * `resolve_replay_subdir(name)` — directory loader. Raises + `FileNotFoundError` (env-unset / subdir-missing). + * `imu_replay_noop(csv_path)` — explicit no-op; signature mirrors + `imu_replay.ImuReplayer.replay` for future live-mode parity. + * Single shared `_resolve_replay_root_or_raise(reason)` enforces + the `E2E_SITL_REPLAY_DIR` semantics exactly once. +* **13 scenarios rewired** (all `_resolve_*` / `_drive_*` / `_push_*` + stubs deleted): + * `_resolve_frame_sink` → `NullFrameSink()` in: FT-P-01, FT-P-02, + FT-P-04, FT-P-05, FT-P-07, FT-P-08, FT-P-09-AP, FT-P-09-iNav, + FT-P-10, FT-P-11, FT-N-01, FT-N-02, FT-N-03, FT-N-04. + * `_resolve_fc_inbound_emitter` → `NullFcInboundEmitter()` in: + FT-P-02, FT-P-04, FT-P-10. + * `_drive_imu_replay` → `imu_replay_noop(...)` in: FT-P-07, FT-N-02. + * `_resolve_frame_period_ms` → `default_frame_period_ms()` in: + FT-N-03, FT-N-04. + * `_resolve_outage_injection_frames` → `resolve_replay_subdir("outage_frames")` + in: FT-N-03. + * `_resolve_gt_per_frame` → `load_replay_json("gt_per_frame.json")` + + dataclass projection in: FT-N-01. + * `_push_single_image_and_observe` → `load_replay_json("single_image_observation.json")` + + tuple projection in: FT-P-03/14. +* **`e2e/_unit_tests/test_directory_layout.py`** — registers the new + `runner/helpers/replay_mode.py` path. + +## Out of scope (deferred) + +* The actual SITL replay fixture builder (separate ticket — will + populate `${E2E_SITL_REPLAY_DIR}/` with `gps_state.json`, + `gt_per_frame.json`, `single_image_observation.json`, + `outage_frames/`, `ekf_divergence_events.json`, etc.). +* Live MAVLink router / pymavlink plumbing (separate live-mode + infrastructure ticket). + +## Test Results + +* New unit tests: **17** (2 null-sink, 2 null-emitter, 1 + frame-period, 2 imu-replay-noop, 6 load_replay_json, 4 + resolve_replay_subdir). +* Full `e2e/_unit_tests` suite: **626 passed in 127 s** (previous + cumulative: 608 → +18 net = +17 new replay_mode tests + 1 new + directory-layout parametrize entry). +* No new linter errors. +* `grep raise NotImplementedError` under `e2e/tests/` returns + **zero** matches. + +## State + +* Spec moved: `_docs/02_tasks/todo/AZ-597_scenario_stub_cleanup.md` + → `_docs/02_tasks/done/`. +* `_docs/_autodev_state.md` advanced to `last_completed_batch: 77`. +* `last_cumulative_review` remains `batches_73-75`; next K=3 + cumulative review fires at the end of batch 78. diff --git a/_docs/03_implementation/reviews/batch_77_review.md b/_docs/03_implementation/reviews/batch_77_review.md new file mode 100644 index 0000000..bf17fcb --- /dev/null +++ b/_docs/03_implementation/reviews/batch_77_review.md @@ -0,0 +1,152 @@ +# Code Review Report + +**Batch**: 77 — AZ-597 (replay_mode helpers + 13 scenario stub rewires) +**Date**: 2026-05-17 +**Verdict**: PASS + +## Findings + +(none) + +## Findings Sweep + +### Phase 1 — Context Loading + +Read the AZ-597 task spec, the `FrameSink` / `FcInboundEmitter` +Protocol definitions in `frame_source_replay.py` and `imu_replay.py` +(b74) to verify the new `Null*` implementations match the exact +method signatures, the `outlier_tolerance_evaluator.GtPose` dataclass +shape consumed by FT-N-01, and the surface used by FT-P-03/14's +`_push_single_image_and_observe` return tuple. Re-read the b75 +`sitl_observer.replay_dir` env-var resolution pattern for symmetry +(`E2E_SITL_REPLAY_DIR`, empty-string-as-None semantics). + +### Phase 2 — Spec Compliance + +| AC | Coverage | Status | +|----|----------|--------| +| AC-1 (`NullFrameSink.write_frame` / `NullFcInboundEmitter.emit` are pure counters) | `test_null_frame_sink_counts_writes`, `test_null_frame_sink_starts_at_zero`, `test_null_emitter_counts_emits`, `test_null_emitter_starts_at_zero` | Covered | +| AC-2 (`load_replay_json` raises `FileNotFoundError` env-unset + file-missing; `ValueError` malformed; round-trips otherwise) | `test_load_replay_json_raises_when_env_unset`, `test_load_replay_json_raises_when_env_empty`, `test_load_replay_json_raises_when_file_missing`, `test_load_replay_json_raises_on_malformed_json`, `test_load_replay_json_round_trips_dict`, `test_load_replay_json_round_trips_list` | Covered | +| AC-3 (`resolve_replay_subdir` raises `FileNotFoundError` env-unset + subdir-missing; returns Path otherwise) | `test_resolve_replay_subdir_raises_when_env_unset`, `test_resolve_replay_subdir_raises_when_subdir_missing`, `test_resolve_replay_subdir_returns_path_when_exists`, `test_resolve_replay_subdir_rejects_file_at_path` | Covered | +| AC-4 (`default_frame_period_ms()` returns 33; documented) | `test_default_frame_period_ms_is_30_fps` (asserts both function + constant); module docstring documents 30 fps default | Covered | +| AC-5 (13 scenarios have local `_resolve_*` / `_drive_*` / `_push_*` stubs deleted; import from `runner.helpers.replay_mode`) | Verified by `Grep raise NotImplementedError under e2e/tests` returning **no matches**. The 13 scenarios touch: FT-P-01/02/04/05/07/08/09-AP/09-iNav/10/11, FT-N-01/02/03/04, and FT-P-03/14. | Covered | +| AC-6 (≥6 unit tests for `replay_mode.py`) | 17 tests total (2 null-sink, 2 null-emitter, 1 frame-period, 2 imu-replay-noop, 6 load_replay_json, 4 resolve_replay_subdir) | Covered (exceeds floor) | +| AC-7 (full suite passes) | 626 passed (+18 from 608; +17 new replay_mode tests + 1 new directory-layout parametrize entry) | Covered | + +### Phase 3 — Code Quality + +* **Single responsibility**: each surface in `replay_mode.py` owns + exactly one concern. + * `NullFrameSink` / `NullFcInboundEmitter` — Protocol-compatible + counter sinks. No I/O, no JSON, no env-var reads. Pure data. + * `default_frame_period_ms` — constant lookup. Trivial; lives in + the same module as the constant it wraps so callers see the + rationale next to the value. + * `imu_replay_noop` — explicit no-op with a comment explaining + why the IMU CSV is ignored in replay mode. Signature mirrors + `imu_replay.ImuReplayer.replay` so a future live-mode driver + can be slotted in. + * `load_replay_json` / `resolve_replay_subdir` — two file-system + surfaces, distinct contracts ("file must exist + parse" vs + "directory must exist"). Both go through one shared + `_resolve_replay_root_or_raise` so env-var semantics are + enforced exactly once. +* **No suppressed errors**: + * `load_replay_json` converts `json.JSONDecodeError` → `ValueError` + with the offending file path AND `raise … from exc` preserves + the original. + * `_resolve_replay_root_or_raise` includes the calling surface in + the error message (`"load_replay_json('foo.json'): ${ENV} not set"`) + so a test author seeing the failure knows exactly which scenario + fired which loader. + * No bare `except`, no `2>/dev/null`, no empty `pass`. +* **AAA comment discipline**: all 17 new tests use + `# Arrange / # Act / # Assert`; sections omitted when not needed. +* **Code comments**: only the module docstring narrates "why + replay-mode no-ops are correct". Per-function docstrings document + contracts and the live-mode follow-up. No line-narration. +* **Public boundary**: `replay_mode.py` imports stdlib only + (`json`, `os`, `pathlib`). Zero `from gps_denied_onboard ...` imports. + +### Phase 4 — Security + +* **No new credentials, secrets, or network surfaces**. All work is + in-process counter state + file I/O over a controlled env-var-rooted + path. +* **`E2E_SITL_REPLAY_DIR`** read consistently with b75/b76 (set → + use; unset / empty / whitespace → treated as absent). No + shell-injection surface — the path is fed straight into `Path` + arithmetic. +* **No `eval`, `exec`, `pickle`, `subprocess`, or + `yaml.load(unsafe=True)`** in the new module. +* **JSON parse is pure stdlib** with explicit error wrapping. No + schema validation — that's the caller's job (the scenarios that + consume the parsed payload validate field types at the call site). + +### Phase 5 — Performance + +* All surfaces are O(1) or O(N) where N is the input JSON size + (single `json.loads` call). No file I/O at module-import time. +* `NullFrameSink` / `NullFcInboundEmitter` are constant-time per call + with single integer increment + no allocations. + +### Phase 6 — Cross-Task Consistency + +* **Env-var pattern matches b75 (`sitl_observer.replay_dir`) and b76 + (`fc_proxy_runtime._resolve_replay_dir`)**: same env var, same + "empty string → None" semantics, same lazy resolution per call. + The three modules deliberately do not share a helper — each owns + its own resolution so the import graph stays flat + (`replay_mode` ↛ `sitl_observer`, `replay_mode` ↛ `fc_proxy_runtime`). + The cost is ~12 lines of duplicate env-var code across three + modules; the benefit is no cross-dependency surface. +* **Skip-gate interaction**: the b75 `sitl_replay_ready` fixture + still skips before any of these loaders fire in unit-test mode. + When the SITL replay fixture builder lands and the env var is set, + scenarios will reach the loaders — at which point the explicit + `FileNotFoundError` messages ("replay fixture 'gt_per_frame.json' + not found at …") provide a precise pointer to which fixture file + is missing. +* **`FileNotFoundError` / `ValueError` discipline matches the rest + of `e2e/runner/helpers/`** (b73-b76 cumulative): missing inputs → + `FileNotFoundError`, malformed inputs → `ValueError` with a file + pointer. +* **Scenario-side import convention**: every rewired stub imports + inside the function body, not at module top-level. This matches + the existing scenario convention (`from runner.helpers import …` + is deferred so that `pytest --collect-only` doesn't pay the import + cost). 13 scenarios, one pattern. +* **`_push_single_image_and_observe` and `_resolve_gt_per_frame` + field-name discipline**: both load JSON and project into a + scenario-local dataclass / tuple. The JSON keys (`frame_idx`, + `lat_deg`, `lon_deg`, `record`, `source_label`) match exactly what + the evaluators / consumers downstream already expect — no schema + translation layer required. + +### Phase 7 — Architecture Compliance + +* **Module placement**: `e2e/runner/helpers/replay_mode.py` (new) + + `e2e/_unit_tests/helpers/test_replay_mode.py` (new). Both + registered in `e2e/_unit_tests/test_directory_layout.py`; the + layout invariant test still passes. +* **No `src/gps_denied_onboard` imports** anywhere. Confirmed. +* **No new top-level dependencies** — stdlib only. `requirements.txt` + untouched. +* **Backwards-compatible scenario contract**: every `_resolve_*` / + `_drive_*` / `_push_*` keeps its original name + signature + return + type. The 13 rewires are body-only changes — no call site changes + in the scenario test functions themselves. + +## Test Results + +* New unit tests: **17** (2 null-sink + 2 null-emitter + 1 + frame-period + 2 imu-replay-noop + 6 load_replay_json + 4 + resolve_replay_subdir). +* Full `e2e/_unit_tests` suite: **626 passed in 127 s** (previous + cumulative: 608 → +18 net = +17 new replay_mode tests + 1 new + directory-layout parametrize entry). +* No new linter errors (`ReadLints` clean on `replay_mode.py`, + `test_replay_mode.py`, `test_directory_layout.py`, and all 13 + rewired scenario files). +* `Grep raise NotImplementedError` under `e2e/tests/` returns **no + matches** — confirming AC-5 (every scenario stub deleted). diff --git a/_docs/_autodev_state.md b/_docs/_autodev_state.md index 77015b9..ad7e49f 100644 --- a/_docs/_autodev_state.md +++ b/_docs/_autodev_state.md @@ -12,9 +12,9 @@ sub_step: retry_count: 0 cycle: 1 tracker: jira -last_completed_batch: 76 +last_completed_batch: 77 last_cumulative_review: batches_73-75 -current_batch: 77 +current_batch: 78 current_batch_tasks: "" last_step_outcomes: step_8: "Code is testable — no changes needed (testability_assessment.md committed; no list-of-changes, no source edits)" diff --git a/e2e/_unit_tests/helpers/test_replay_mode.py b/e2e/_unit_tests/helpers/test_replay_mode.py new file mode 100644 index 0000000..a603a7f --- /dev/null +++ b/e2e/_unit_tests/helpers/test_replay_mode.py @@ -0,0 +1,184 @@ +"""Unit tests for `e2e/runner/helpers/replay_mode.py` (AZ-597).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from e2e.runner.helpers import replay_mode as rm + + +@pytest.fixture +def replay_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + monkeypatch.setenv("E2E_SITL_REPLAY_DIR", str(tmp_path)) + return tmp_path + + +@pytest.fixture +def unset_replay_dir(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("E2E_SITL_REPLAY_DIR", raising=False) + + +# NullFrameSink + + +def test_null_frame_sink_counts_writes(): + # Arrange + sink = rm.NullFrameSink() + + # Act + sink.write_frame(b"jpeg-bytes-1", timestamp_ms=100) + sink.write_frame(b"jpeg-bytes-2", timestamp_ms=200) + sink.write_frame(b"jpeg-bytes-3", timestamp_ms=300) + + # Assert + assert sink.frames_written == 3 + + +def test_null_frame_sink_starts_at_zero(): + # Assert + assert rm.NullFrameSink().frames_written == 0 + + +# NullFcInboundEmitter + + +def test_null_emitter_counts_emits(): + # Arrange + emitter = rm.NullFcInboundEmitter() + + # Act + emitter.emit(object()) + emitter.emit(object()) + + # Assert + assert emitter.samples_emitted == 2 + + +def test_null_emitter_starts_at_zero(): + # Assert + assert rm.NullFcInboundEmitter().samples_emitted == 0 + + +# default_frame_period_ms + + +def test_default_frame_period_ms_is_30_fps(): + # Assert + assert rm.default_frame_period_ms() == 33 + assert rm.DEFAULT_FRAME_PERIOD_MS == 33 + + +# imu_replay_noop + + +def test_imu_replay_noop_returns_none(tmp_path: Path): + # Assert + assert rm.imu_replay_noop(tmp_path / "any.csv") is None + + +def test_imu_replay_noop_does_not_touch_disk(tmp_path: Path): + # Arrange + csv = tmp_path / "does-not-exist.csv" + + # Act + rm.imu_replay_noop(csv) + + # Assert + assert not csv.exists() + + +# load_replay_json + + +def test_load_replay_json_raises_when_env_unset(unset_replay_dir): + # Assert + with pytest.raises(FileNotFoundError, match="E2E_SITL_REPLAY_DIR.*not set"): + rm.load_replay_json("any.json") + + +def test_load_replay_json_raises_when_env_empty(monkeypatch: pytest.MonkeyPatch): + # Arrange + monkeypatch.setenv("E2E_SITL_REPLAY_DIR", " ") + + # Assert + with pytest.raises(FileNotFoundError, match="not set or empty"): + rm.load_replay_json("any.json") + + +def test_load_replay_json_raises_when_file_missing(replay_dir: Path): + # Assert + with pytest.raises(FileNotFoundError, match="replay fixture 'gone.json' not found"): + rm.load_replay_json("gone.json") + + +def test_load_replay_json_raises_on_malformed_json(replay_dir: Path): + # Arrange + (replay_dir / "bad.json").write_text("{not valid") + + # Assert + with pytest.raises(ValueError, match="malformed replay fixture JSON"): + rm.load_replay_json("bad.json") + + +def test_load_replay_json_round_trips_dict(replay_dir: Path): + # Arrange + payload = {"key": "value", "nested": {"a": 1}} + (replay_dir / "ok.json").write_text(json.dumps(payload)) + + # Act + result = rm.load_replay_json("ok.json") + + # Assert + assert result == payload + + +def test_load_replay_json_round_trips_list(replay_dir: Path): + # Arrange + payload = [{"frame_idx": i} for i in range(3)] + (replay_dir / "list.json").write_text(json.dumps(payload)) + + # Act + result = rm.load_replay_json("list.json") + + # Assert + assert result == payload + + +# resolve_replay_subdir + + +def test_resolve_replay_subdir_raises_when_env_unset(unset_replay_dir): + # Assert + with pytest.raises(FileNotFoundError, match="E2E_SITL_REPLAY_DIR.*not set"): + rm.resolve_replay_subdir("frames") + + +def test_resolve_replay_subdir_raises_when_subdir_missing(replay_dir: Path): + # Assert + with pytest.raises(FileNotFoundError, match="replay fixture subdir 'frames' not found"): + rm.resolve_replay_subdir("frames") + + +def test_resolve_replay_subdir_returns_path_when_exists(replay_dir: Path): + # Arrange + target = replay_dir / "frames" + target.mkdir() + + # Act + result = rm.resolve_replay_subdir("frames") + + # Assert + assert result == target + assert result.is_dir() + + +def test_resolve_replay_subdir_rejects_file_at_path(replay_dir: Path): + # Arrange + (replay_dir / "actually-a-file").write_text("oops") + + # Assert + with pytest.raises(FileNotFoundError, match="subdir 'actually-a-file' not found"): + rm.resolve_replay_subdir("actually-a-file") diff --git a/e2e/_unit_tests/test_directory_layout.py b/e2e/_unit_tests/test_directory_layout.py index 35eb7ae..7a2abc7 100644 --- a/e2e/_unit_tests/test_directory_layout.py +++ b/e2e/_unit_tests/test_directory_layout.py @@ -56,6 +56,7 @@ E2E_ROOT = Path(__file__).resolve().parents[1] "runner/helpers/outage_request_evaluator.py", "runner/helpers/blackout_spoof_evaluator.py", "runner/helpers/fc_proxy_runtime.py", + "runner/helpers/replay_mode.py", "fixtures/mock-suite-sat/Dockerfile", "fixtures/mock-suite-sat/app.py", "fixtures/mock-suite-sat/requirements.txt", diff --git a/e2e/runner/helpers/replay_mode.py b/e2e/runner/helpers/replay_mode.py new file mode 100644 index 0000000..e23a2fc --- /dev/null +++ b/e2e/runner/helpers/replay_mode.py @@ -0,0 +1,136 @@ +"""Shared FDR-replay helpers consumed by 13 scenario files (AZ-597). + +Closes the last gap in the offline FDR-replay path opened by AZ-595 +(`sitl_observer`) and AZ-596 (`fc_proxy_runtime`): a small grab-bag of +per-scenario stubs that all reduce to "I'm in replay mode, so the +frame / IMU / single-image push is a no-op or a JSON read". + +In replay mode, frames decoded by `FrameSourceReplayer` aren't actually +driving anything — the SUT's FDR archive already encodes what happened +when the fixture was built. The same applies to IMU samples emitted to +the FC inbound. So scenarios just need: + +* a `FrameSink` that counts but discards bytes, +* an `FcInboundEmitter` that counts but discards samples, +* a default frame-period for window-arithmetic helpers that ask for it, +* generic `${E2E_SITL_REPLAY_DIR}` JSON / sub-directory loaders for the + scenario-specific fixtures (per-frame GT, single-image observation, + outage frames directory). + +The 13 scenarios that previously carried local `_resolve_*` / +`_drive_*` / `_push_*` `NotImplementedError` stubs now import these +helpers directly. + +Public-boundary discipline: stdlib only. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +_ENV_VAR = "E2E_SITL_REPLAY_DIR" + +DEFAULT_FRAME_PERIOD_MS = 33 +"""30 fps default frame period — matches Derkachi MP4 + outlier injector cadence.""" + + +class NullFrameSink: + """`FrameSink`-compatible sink that counts but discards bytes. + + In FDR-replay mode the SUT's FDR archive already encodes the + per-frame result; the sink only needs to drain the replayer's + `write_frame` calls without storing anything. The counter is + surfaced for diagnostic asserts (e.g. "did we see 60 frames?"). + """ + + def __init__(self) -> None: + self.frames_written: int = 0 + + def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None: + self.frames_written += 1 + + +class NullFcInboundEmitter: + """`FcInboundEmitter`-compatible emitter that counts but discards samples. + + Same rationale as `NullFrameSink` — the FDR archive already + encodes the IMU-driven FC state; the emitter only needs to drain + `ImuReplayer.replay`'s `emit` calls. + """ + + def __init__(self) -> None: + self.samples_emitted: int = 0 + + def emit(self, sample: object) -> None: + self.samples_emitted += 1 + + +def default_frame_period_ms() -> int: + """Default per-frame period in ms (30 fps). + + Scenarios that need a frame-period for window arithmetic (FT-N-03, + FT-N-04) call this when the fixture builder hasn't supplied an + explicit override. The constant matches the Derkachi MP4 native + cadence and the AZ-408 outlier injector's frame stride. + """ + return DEFAULT_FRAME_PERIOD_MS + + +def imu_replay_noop(csv_path: Path) -> None: + """No-op IMU-replay driver for FDR-replay mode. + + The IMU samples are pre-baked into the FDR archive by the fixture + builder, so the runtime driver has nothing to do. `csv_path` is + accepted (and ignored) so the call-site signature matches the + live-mode `imu_replay.ImuReplayer.replay(csv_path)` for the day + a live-mode driver lands. + """ + return None + + +def load_replay_json(filename: str) -> dict | list: + """Load `${E2E_SITL_REPLAY_DIR}/` and return parsed JSON. + + Raises `FileNotFoundError` when the env var is unset OR the file + is missing; `ValueError` (with the file path) when the JSON is + malformed. + """ + root = _resolve_replay_root_or_raise(reason=f"load_replay_json({filename!r})") + path = root / filename + if not path.is_file(): + raise FileNotFoundError( + f"replay fixture {filename!r} not found at {path}" + ) + try: + return json.loads(path.read_text()) + except json.JSONDecodeError as exc: + raise ValueError( + f"malformed replay fixture JSON at {path}: {exc.msg}" + ) from exc + + +def resolve_replay_subdir(name: str) -> Path: + """Resolve `${E2E_SITL_REPLAY_DIR}//` and verify it exists. + + Raises `FileNotFoundError` when the env var is unset OR the + subdirectory is missing. + """ + root = _resolve_replay_root_or_raise(reason=f"resolve_replay_subdir({name!r})") + path = root / name + if not path.is_dir(): + raise FileNotFoundError( + f"replay fixture subdir {name!r} not found at {path}" + ) + return path + + +def _resolve_replay_root_or_raise(*, reason: str) -> Path: + raw = os.environ.get(_ENV_VAR, "").strip() + if not raw: + raise FileNotFoundError( + f"{reason}: ${_ENV_VAR} not set or empty — scenario should " + "have skipped via `sitl_replay_ready` (AZ-595) before reaching here" + ) + return Path(raw) diff --git a/e2e/tests/negative/test_ft_n_01_outlier_tolerance.py b/e2e/tests/negative/test_ft_n_01_outlier_tolerance.py index 30195c6..e69ba10 100644 --- a/e2e/tests/negative/test_ft_n_01_outlier_tolerance.py +++ b/e2e/tests/negative/test_ft_n_01_outlier_tolerance.py @@ -124,12 +124,33 @@ def test_ft_n_01_outlier_tolerance( def _resolve_frame_sink(): # type: ignore[no-untyped-def] - raise NotImplementedError( - "frame sink resolution is owned by AZ-441 / runner.helpers.frame_source_replay" - ) + """Return a replay-mode `FrameSink` (counter-only; AZ-597).""" + from runner.helpers.replay_mode import NullFrameSink + + return NullFrameSink() def _resolve_gt_per_frame(report: OutlierInjectionReport) -> list[ote.GtPose]: - raise NotImplementedError( - "Per-frame GT resolution is owned by AZ-407 / runner.helpers.tile_cache_gt" - ) + """Load per-frame GT from `${E2E_SITL_REPLAY_DIR}/gt_per_frame.json` (AZ-597). + + The fixture builder writes a list of `{frame_idx, lat_deg, lon_deg}` + records keyed off `report.out_root.name` (one file per injection + variant). In FDR-replay mode this is the GT the SUT was scored + against when the FDR archive was originally produced. + """ + from runner.helpers.replay_mode import load_replay_json + + raw = load_replay_json("gt_per_frame.json") + if not isinstance(raw, list): + raise ValueError( + "gt_per_frame.json must be a JSON list of " + "{frame_idx, lat_deg, lon_deg} records" + ) + return [ + ote.GtPose( + frame_idx=int(entry["frame_idx"]), + lat_deg=float(entry["lat_deg"]), + lon_deg=float(entry["lon_deg"]), + ) + for entry in raw + ] diff --git a/e2e/tests/negative/test_ft_n_02_sharp_turn_failure.py b/e2e/tests/negative/test_ft_n_02_sharp_turn_failure.py index f2ddcde..7f61aa7 100644 --- a/e2e/tests/negative/test_ft_n_02_sharp_turn_failure.py +++ b/e2e/tests/negative/test_ft_n_02_sharp_turn_failure.py @@ -130,12 +130,14 @@ def test_ft_n_02_sharp_turn_failure( def _resolve_frame_sink(): # type: ignore[no-untyped-def] - raise NotImplementedError( - "frame sink resolution is owned by AZ-441 / runner.helpers.frame_source_replay" - ) + """Return a replay-mode `FrameSink` (counter-only; AZ-597).""" + from runner.helpers.replay_mode import NullFrameSink + + return NullFrameSink() def _drive_imu_replay(csv_path: Path) -> None: - raise NotImplementedError( - "IMU replay driver is owned by AZ-416/AZ-417 / runner.helpers.imu_replay" - ) + """Replay-mode no-op: IMU samples pre-baked into FDR archive (AZ-597).""" + from runner.helpers.replay_mode import imu_replay_noop + + imu_replay_noop(csv_path) diff --git a/e2e/tests/negative/test_ft_n_03_outage_reloc.py b/e2e/tests/negative/test_ft_n_03_outage_reloc.py index 28ec91b..4b043ea 100644 --- a/e2e/tests/negative/test_ft_n_03_outage_reloc.py +++ b/e2e/tests/negative/test_ft_n_03_outage_reloc.py @@ -149,19 +149,21 @@ def test_ft_n_03_outage_reloc( def _resolve_outage_injection_frames() -> Path: - raise NotImplementedError( - "3-frame outage injector is owned by AZ-408 extension / " - "fixtures/injectors/outlier.py (--all-zero variant)" - ) + """Resolve `${E2E_SITL_REPLAY_DIR}/outage_frames/` (AZ-597).""" + from runner.helpers.replay_mode import resolve_replay_subdir + + return resolve_replay_subdir("outage_frames") def _resolve_frame_sink(): # type: ignore[no-untyped-def] - raise NotImplementedError( - "frame sink resolution is owned by AZ-441 / runner.helpers.frame_source_replay" - ) + """Return a replay-mode `FrameSink` (counter-only; AZ-597).""" + from runner.helpers.replay_mode import NullFrameSink + + return NullFrameSink() def _resolve_frame_period_ms() -> int: - raise NotImplementedError( - "Frame period resolution is owned by AZ-441 / runner.helpers.frame_source_replay" - ) + """Return the default 30 fps per-frame period (AZ-597).""" + from runner.helpers.replay_mode import default_frame_period_ms + + return default_frame_period_ms() diff --git a/e2e/tests/negative/test_ft_n_04_blackout_spoof.py b/e2e/tests/negative/test_ft_n_04_blackout_spoof.py index 2840866..1b6475c 100644 --- a/e2e/tests/negative/test_ft_n_04_blackout_spoof.py +++ b/e2e/tests/negative/test_ft_n_04_blackout_spoof.py @@ -216,9 +216,10 @@ def test_ft_n_04_blackout_spoof( def _resolve_frame_sink(): # type: ignore[no-untyped-def] - raise NotImplementedError( - "frame sink resolution is owned by AZ-441 / runner.helpers.frame_source_replay" - ) + """Return a replay-mode `FrameSink` (counter-only; AZ-597).""" + from runner.helpers.replay_mode import NullFrameSink + + return NullFrameSink() def _drive_fc_proxy(schedule_path: Path) -> None: @@ -228,6 +229,7 @@ def _drive_fc_proxy(schedule_path: Path) -> None: def _resolve_frame_period_ms() -> int: - raise NotImplementedError( - "Frame period resolution is owned by AZ-441 / runner.helpers.frame_source_replay" - ) + """Return the default 30 fps per-frame period (AZ-597).""" + from runner.helpers.replay_mode import default_frame_period_ms + + return default_frame_period_ms() diff --git a/e2e/tests/positive/test_ft_p_01_still_image_accuracy.py b/e2e/tests/positive/test_ft_p_01_still_image_accuracy.py index 537e59c..37ff742 100644 --- a/e2e/tests/positive/test_ft_p_01_still_image_accuracy.py +++ b/e2e/tests/positive/test_ft_p_01_still_image_accuracy.py @@ -140,7 +140,7 @@ def test_ft_p_01_still_image_accuracy( def _resolve_frame_sink(): # type: ignore[no-untyped-def] - """Stub helper resolved when the underlying replayer lands.""" - raise NotImplementedError( - "frame sink resolution is owned by AZ-441 / runner.helpers.frame_source_replay" - ) + """Return a replay-mode `FrameSink` (counter-only; AZ-597).""" + from runner.helpers.replay_mode import NullFrameSink + + return NullFrameSink() diff --git a/e2e/tests/positive/test_ft_p_02_derkachi_drift.py b/e2e/tests/positive/test_ft_p_02_derkachi_drift.py index ce84bf5..14d6cd3 100644 --- a/e2e/tests/positive/test_ft_p_02_derkachi_drift.py +++ b/e2e/tests/positive/test_ft_p_02_derkachi_drift.py @@ -146,14 +146,14 @@ def test_ft_p_02_derkachi_drift( def _resolve_frame_sink(): # type: ignore[no-untyped-def] - """Stub helper resolved when the underlying replayer lands.""" - raise NotImplementedError( - "frame sink resolution is owned by AZ-441 / runner.helpers.frame_source_replay" - ) + """Return a replay-mode `FrameSink` (counter-only; AZ-597).""" + from runner.helpers.replay_mode import NullFrameSink + + return NullFrameSink() def _resolve_fc_inbound_emitter(fc_adapter: str, host: str): # type: ignore[no-untyped-def] - """Stub helper resolved when the FC inbound emitter lands.""" - raise NotImplementedError( - "FC inbound emitter resolution is owned by AZ-416/AZ-417 / runner.helpers.imu_replay" - ) + """Return a replay-mode `FcInboundEmitter` (counter-only; AZ-597).""" + from runner.helpers.replay_mode import NullFcInboundEmitter + + return NullFcInboundEmitter() diff --git a/e2e/tests/positive/test_ft_p_03_14_schema_wgs84.py b/e2e/tests/positive/test_ft_p_03_14_schema_wgs84.py index 781d601..3cca91a 100644 --- a/e2e/tests/positive/test_ft_p_03_14_schema_wgs84.py +++ b/e2e/tests/positive/test_ft_p_03_14_schema_wgs84.py @@ -106,13 +106,19 @@ def test_wgs84_coordinate_range( def _push_single_image_and_observe(fc_adapter: str, vio_strategy: str): # type: ignore[no-untyped-def] - """Push AD000001.jpg through the SUT and return (outbound_record, source_label). + """Read the single-image observation fixture in replay mode (AZ-597). - Stub until runner.helpers.{frame_source_replay,sitl_observer,mavproxy_tlog_reader} - land; the scenario test's `sitl_replay_ready` skip gate (AZ-595) - keeps this from executing prematurely. + The fixture builder records the SUT's outbound estimate for the + single-image-push variant into + `${E2E_SITL_REPLAY_DIR}/single_image_observation.json` as + `{"record": {...estimate payload...}, "source_label": "..."}`. """ - raise NotImplementedError( - "single-image push helper is owned by AZ-407 / AZ-416 / AZ-417 " - "(runner.helpers.frame_source_replay + sitl_observer + mavproxy_tlog_reader)" - ) + from runner.helpers.replay_mode import load_replay_json + + payload = load_replay_json("single_image_observation.json") + if not isinstance(payload, dict): + raise ValueError( + "single_image_observation.json must be a JSON object with " + "'record' and 'source_label' keys" + ) + return payload["record"], payload["source_label"] diff --git a/e2e/tests/positive/test_ft_p_04_derkachi_f2f_registration.py b/e2e/tests/positive/test_ft_p_04_derkachi_f2f_registration.py index dbb1a24..195b7c6 100644 --- a/e2e/tests/positive/test_ft_p_04_derkachi_f2f_registration.py +++ b/e2e/tests/positive/test_ft_p_04_derkachi_f2f_registration.py @@ -142,14 +142,14 @@ def test_ft_p_04_derkachi_f2f_registration( def _resolve_frame_sink(): # type: ignore[no-untyped-def] - """Stub helper resolved when the underlying replayer lands.""" - raise NotImplementedError( - "frame sink resolution is owned by AZ-441 / runner.helpers.frame_source_replay" - ) + """Return a replay-mode `FrameSink` (counter-only; AZ-597).""" + from runner.helpers.replay_mode import NullFrameSink + + return NullFrameSink() def _resolve_fc_inbound_emitter(fc_adapter: str): # type: ignore[no-untyped-def] - """Stub helper resolved when the FC inbound emitter lands.""" - raise NotImplementedError( - "FC inbound emitter resolution is owned by AZ-416/AZ-417 / runner.helpers.imu_replay" - ) + """Return a replay-mode `FcInboundEmitter` (counter-only; AZ-597).""" + from runner.helpers.replay_mode import NullFcInboundEmitter + + return NullFcInboundEmitter() diff --git a/e2e/tests/positive/test_ft_p_05_sat_anchor.py b/e2e/tests/positive/test_ft_p_05_sat_anchor.py index eda6885..0ac0bf3 100644 --- a/e2e/tests/positive/test_ft_p_05_sat_anchor.py +++ b/e2e/tests/positive/test_ft_p_05_sat_anchor.py @@ -160,6 +160,7 @@ def test_ft_p_05_sat_anchor( def _resolve_frame_sink(): # type: ignore[no-untyped-def] - raise NotImplementedError( - "frame sink resolution is owned by AZ-441 / runner.helpers.frame_source_replay" - ) + """Return a replay-mode `FrameSink` (counter-only; AZ-597).""" + from runner.helpers.replay_mode import NullFrameSink + + return NullFrameSink() diff --git a/e2e/tests/positive/test_ft_p_07_sharp_turn_recovery.py b/e2e/tests/positive/test_ft_p_07_sharp_turn_recovery.py index cce1a39..1bf10f2 100644 --- a/e2e/tests/positive/test_ft_p_07_sharp_turn_recovery.py +++ b/e2e/tests/positive/test_ft_p_07_sharp_turn_recovery.py @@ -148,12 +148,14 @@ def test_ft_p_07_sharp_turn_recovery( def _resolve_frame_sink(): # type: ignore[no-untyped-def] - raise NotImplementedError( - "frame sink resolution is owned by AZ-441 / runner.helpers.frame_source_replay" - ) + """Return a replay-mode `FrameSink` (counter-only; AZ-597).""" + from runner.helpers.replay_mode import NullFrameSink + + return NullFrameSink() def _drive_imu_replay(csv_path: Path) -> None: - raise NotImplementedError( - "IMU replay driver is owned by AZ-416/AZ-417 / runner.helpers.imu_replay" - ) + """Replay-mode no-op: IMU samples pre-baked into FDR archive (AZ-597).""" + from runner.helpers.replay_mode import imu_replay_noop + + imu_replay_noop(csv_path) diff --git a/e2e/tests/positive/test_ft_p_08_multi_segment_reloc.py b/e2e/tests/positive/test_ft_p_08_multi_segment_reloc.py index 6023934..7218701 100644 --- a/e2e/tests/positive/test_ft_p_08_multi_segment_reloc.py +++ b/e2e/tests/positive/test_ft_p_08_multi_segment_reloc.py @@ -132,6 +132,7 @@ def test_ft_p_08_multi_segment_reloc( def _resolve_frame_sink(): # type: ignore[no-untyped-def] - raise NotImplementedError( - "frame sink resolution is owned by AZ-441 / runner.helpers.frame_source_replay" - ) + """Return a replay-mode `FrameSink` (counter-only; AZ-597).""" + from runner.helpers.replay_mode import NullFrameSink + + return NullFrameSink() diff --git a/e2e/tests/positive/test_ft_p_09_ap_signing.py b/e2e/tests/positive/test_ft_p_09_ap_signing.py index c94036d..139607e 100644 --- a/e2e/tests/positive/test_ft_p_09_ap_signing.py +++ b/e2e/tests/positive/test_ft_p_09_ap_signing.py @@ -148,6 +148,7 @@ def test_ft_p_09_ap_signing( def _resolve_frame_sink(): # type: ignore[no-untyped-def] - raise NotImplementedError( - "frame sink resolution is owned by AZ-441 / runner.helpers.frame_source_replay" - ) + """Return a replay-mode `FrameSink` (counter-only; AZ-597).""" + from runner.helpers.replay_mode import NullFrameSink + + return NullFrameSink() diff --git a/e2e/tests/positive/test_ft_p_09_inav.py b/e2e/tests/positive/test_ft_p_09_inav.py index bc91b42..4a16fe9 100644 --- a/e2e/tests/positive/test_ft_p_09_inav.py +++ b/e2e/tests/positive/test_ft_p_09_inav.py @@ -139,6 +139,7 @@ def test_ft_p_09_inav( def _resolve_frame_sink(): # type: ignore[no-untyped-def] - raise NotImplementedError( - "frame sink resolution is owned by AZ-441 / runner.helpers.frame_source_replay" - ) + """Return a replay-mode `FrameSink` (counter-only; AZ-597).""" + from runner.helpers.replay_mode import NullFrameSink + + return NullFrameSink() diff --git a/e2e/tests/positive/test_ft_p_10_smoothing_lookback.py b/e2e/tests/positive/test_ft_p_10_smoothing_lookback.py index f9178ec..9e75c4d 100644 --- a/e2e/tests/positive/test_ft_p_10_smoothing_lookback.py +++ b/e2e/tests/positive/test_ft_p_10_smoothing_lookback.py @@ -164,12 +164,14 @@ def test_ft_p_10_smoothing_lookback( def _resolve_frame_sink(): # type: ignore[no-untyped-def] - raise NotImplementedError( - "frame sink resolution is owned by AZ-441 / runner.helpers.frame_source_replay" - ) + """Return a replay-mode `FrameSink` (counter-only; AZ-597).""" + from runner.helpers.replay_mode import NullFrameSink + + return NullFrameSink() def _resolve_fc_inbound_emitter(fc_adapter: str): # type: ignore[no-untyped-def] - raise NotImplementedError( - "FC inbound emitter resolution is owned by AZ-416/AZ-417 / runner.helpers.imu_replay" - ) + """Return a replay-mode `FcInboundEmitter` (counter-only; AZ-597).""" + from runner.helpers.replay_mode import NullFcInboundEmitter + + return NullFcInboundEmitter() diff --git a/e2e/tests/positive/test_ft_p_11_cold_start_init.py b/e2e/tests/positive/test_ft_p_11_cold_start_init.py index 1f982fe..dd2f6de 100644 --- a/e2e/tests/positive/test_ft_p_11_cold_start_init.py +++ b/e2e/tests/positive/test_ft_p_11_cold_start_init.py @@ -262,6 +262,7 @@ def test_ft_p_11_cold_start_no_origin_aborts( def _resolve_frame_sink(): # type: ignore[no-untyped-def] - raise NotImplementedError( - "frame sink resolution is owned by AZ-441 / runner.helpers.frame_source_replay" - ) + """Return a replay-mode `FrameSink` (counter-only; AZ-597).""" + from runner.helpers.replay_mode import NullFrameSink + + return NullFrameSink()