[AZ-594] Implement core-three harness stubs (fdr_reader, frame_source_replay, imu_replay)

Replaces the NotImplementedError stubs AZ-406 reserved on three runner- side helpers; these were stranded from any tracker ticket since AZ-407/408 never came back to fill them. Concrete bodies: * fdr_reader.iter_records: JSONL parser + wire-envelope validator; recursive *.jsonl walk; projects {schema_version, ts, producer_id, kind, payload} to runner-side FdrRecord with record_type/monotonic_ms renames; yields oldest-first. * frame_source_replay.replay_video: OpenCV VideoCapture decode + JPEG re-encode; auto-detects file vs directory; injectable sleep_fn for unit-test pacing. * imu_replay.ImuReplayer.replay: csv.DictReader parse; degrees->radians attitude conversion; tolerates scientific notation; same sleep_fn injection pattern. Adds 34 unit tests (14 + 10 + 10). Full e2e unit suite: 558 passed (+31). Existing scenario _harness_helpers_implemented probes still return False because they also depend on sitl_observer / fc_proxy_runtime stubs that remain pending; scenario probe cleanup is out of AZ-594 scope. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-06-22 18:41:13 +00:00 · 2026-05-17 08:42:12 +03:00
parent 2d6d44af5d
commit 1d260f7e41
10 changed files with 1196 additions and 76 deletions
@@ -1,37 +1,214 @@
-"""Unit tests for `runner.helpers.fdr_reader.archive_size_bytes`.
-
-The full `iter_records` parser is owned by AZ-441; AZ-406 only commits to
-the directory-size helper.
-"""
+"""Unit tests for `e2e/runner/helpers/fdr_reader.py` (AZ-594 AC-1)."""

 from __future__ import annotations

+import json
 from pathlib import Path

 import pytest

-from runner.helpers.fdr_reader import archive_size_bytes
+from e2e.runner.helpers.fdr_reader import FdrRecord, archive_size_bytes, iter_records


-def test_archive_size_zero_for_missing_root(tmp_path: Path) -> None:
-    assert archive_size_bytes(tmp_path / "does-not-exist") == 0
+def _write_jsonl(path: Path, records: list[dict]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w") as fh:
+        for r in records:
+            fh.write(json.dumps(r) + "\n")


-def test_archive_size_sums_nested_files(tmp_path: Path) -> None:
-    # Arrange
-    (tmp_path / "a").mkdir()
-    (tmp_path / "a" / "b.bin").write_bytes(b"x" * 100)
-    (tmp_path / "a" / "c.bin").write_bytes(b"y" * 50)
-    (tmp_path / "top.bin").write_bytes(b"z" * 200)
-    # Act
-    size = archive_size_bytes(tmp_path)
+def _env(ts: str, *, kind: str = "vio.tick", producer_id: str = "c1.vio", payload: dict | None = None) -> dict:
+    return {
+        "schema_version": 1,
+        "ts": ts,
+        "producer_id": producer_id,
+        "kind": kind,
+        "payload": payload if payload is not None else {"frame_id": "f0"},
+    }
+
+
+def test_missing_root_raises_file_not_found(tmp_path: Path):
    # Assert
-    assert size == 350
+    with pytest.raises(FileNotFoundError, match="FDR archive root not found"):
+        list(iter_records(tmp_path / "nope"))


-def test_iter_records_raises_until_az441_lands() -> None:
-    """Until AZ-441 fills the parser in, callers must see a clear error."""
-    from runner.helpers.fdr_reader import iter_records
+def test_empty_root_yields_nothing(tmp_path: Path):
+    # Arrange
+    (tmp_path / "fdr").mkdir()

-    with pytest.raises(NotImplementedError, match="AZ-441"):
-        next(iter_records(Path("/tmp/nonexistent")))
+    # Act
+    records = list(iter_records(tmp_path / "fdr"))
+
+    # Assert
+    assert records == []
+
+
+def test_single_file_round_trip(tmp_path: Path):
+    # Arrange
+    root = tmp_path / "fdr"
+    _write_jsonl(
+        root / "segment_001.jsonl",
+        [_env("2026-05-17T08:00:00.100Z"), _env("2026-05-17T08:00:00.200Z")],
+    )
+
+    # Act
+    records = list(iter_records(root))
+
+    # Assert
+    assert len(records) == 2
+    assert all(isinstance(r, FdrRecord) for r in records)
+    assert records[0].record_type == "vio.tick"
+    assert records[0].producer_id == "c1.vio"
+    assert records[1].monotonic_ms - records[0].monotonic_ms == 100
+
+
+def test_multiple_files_are_merged_and_sorted(tmp_path: Path):
+    # Arrange — file B has older records than file A.
+    root = tmp_path / "fdr"
+    _write_jsonl(
+        root / "b_segment.jsonl",
+        [_env("2026-05-17T08:00:01.000Z")],
+    )
+    _write_jsonl(
+        root / "a_segment.jsonl",
+        [_env("2026-05-17T08:00:00.500Z")],
+    )
+
+    # Act
+    records = list(iter_records(root))
+
+    # Assert — global oldest-first regardless of filename order.
+    assert len(records) == 2
+    assert records[0].monotonic_ms < records[1].monotonic_ms
+
+
+def test_blank_lines_are_skipped(tmp_path: Path):
+    # Arrange
+    root = tmp_path / "fdr"
+    root.mkdir()
+    with (root / "segment_001.jsonl").open("w") as fh:
+        fh.write(json.dumps(_env("2026-05-17T08:00:00.100Z")) + "\n")
+        fh.write("\n")
+        fh.write("   \n")
+        fh.write(json.dumps(_env("2026-05-17T08:00:00.200Z")) + "\n")
+
+    # Act
+    records = list(iter_records(root))
+
+    # Assert
+    assert len(records) == 2
+
+
+def test_missing_envelope_key_raises(tmp_path: Path):
+    # Arrange — missing `kind`.
+    root = tmp_path / "fdr"
+    bad = {
+        "schema_version": 1,
+        "ts": "2026-05-17T08:00:00.100Z",
+        "producer_id": "c1.vio",
+        "payload": {},
+    }
+    _write_jsonl(root / "bad.jsonl", [bad])
+
+    # Act / Assert
+    with pytest.raises(ValueError, match="missing required keys \\['kind'\\]"):
+        list(iter_records(root))
+
+
+def test_non_object_line_raises(tmp_path: Path):
+    # Arrange — array at top level.
+    root = tmp_path / "fdr"
+    root.mkdir()
+    with (root / "bad.jsonl").open("w") as fh:
+        fh.write("[1, 2, 3]\n")
+
+    # Act / Assert
+    with pytest.raises(ValueError, match="not a JSON object"):
+        list(iter_records(root))
+
+
+def test_malformed_json_raises(tmp_path: Path):
+    # Arrange
+    root = tmp_path / "fdr"
+    root.mkdir()
+    with (root / "bad.jsonl").open("w") as fh:
+        fh.write("{not-json\n")
+
+    # Act / Assert
+    with pytest.raises(json.JSONDecodeError):
+        list(iter_records(root))
+
+
+def test_empty_producer_id_raises(tmp_path: Path):
+    # Arrange
+    root = tmp_path / "fdr"
+    bad = _env("2026-05-17T08:00:00.100Z", producer_id="")
+    _write_jsonl(root / "bad.jsonl", [bad])
+
+    # Act / Assert
+    with pytest.raises(ValueError, match="producer_id` must be a non-empty"):
+        list(iter_records(root))
+
+
+def test_ts_iso_without_z_parses(tmp_path: Path):
+    # Arrange — already in +00:00 form.
+    root = tmp_path / "fdr"
+    _write_jsonl(root / "a.jsonl", [_env("2026-05-17T08:00:00.250+00:00")])
+
+    # Act
+    records = list(iter_records(root))
+
+    # Assert
+    assert len(records) == 1
+
+
+def test_payload_passed_through(tmp_path: Path):
+    # Arrange
+    root = tmp_path / "fdr"
+    payload = {"frame_idx": 42, "lat_deg": 50.0, "lon_deg": 30.0, "cov_semi_major_m": 5.5}
+    _write_jsonl(
+        root / "a.jsonl",
+        [_env("2026-05-17T08:00:00.100Z", kind="outbound_estimate", payload=payload)],
+    )
+
+    # Act
+    [record] = list(iter_records(root))
+
+    # Assert
+    assert record.payload == payload
+    assert record.record_type == "outbound_estimate"
+
+
+def test_archive_size_bytes_sums_all_files(tmp_path: Path):
+    # Arrange
+    root = tmp_path / "fdr"
+    _write_jsonl(root / "a.jsonl", [_env("2026-05-17T08:00:00.100Z")])
+    _write_jsonl(root / "sub/b.jsonl", [_env("2026-05-17T08:00:00.200Z")])
+
+    # Act
+    total = archive_size_bytes(root)
+
+    # Assert
+    assert total > 0
+    a_size = (root / "a.jsonl").stat().st_size
+    b_size = (root / "sub/b.jsonl").stat().st_size
+    assert total == a_size + b_size
+
+
+def test_archive_size_bytes_missing_root_returns_zero(tmp_path: Path):
+    # Assert
+    assert archive_size_bytes(tmp_path / "nope") == 0
+
+
+def test_subdirectory_files_included(tmp_path: Path):
+    # Arrange
+    root = tmp_path / "fdr"
+    _write_jsonl(root / "seg1.jsonl", [_env("2026-05-17T08:00:00.100Z")])
+    _write_jsonl(root / "sub/seg2.jsonl", [_env("2026-05-17T08:00:00.200Z")])
+
+    # Act
+    records = list(iter_records(root))
+
+    # Assert
+    assert len(records) == 2
@@ -0,0 +1,216 @@
+"""Unit tests for `e2e/runner/helpers/frame_source_replay.py` (AZ-594 AC-2)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import cv2
+import numpy as np
+import pytest
+
+from e2e.runner.helpers.frame_source_replay import (
+    FrameSourceReplayer,
+    ReplayCadence,
+)
+
+
+@dataclass
+class _RecordingSink:
+    """In-memory FrameSink that captures every emission for assertions."""
+
+    frames: list[tuple[bytes, int]] = field(default_factory=list)
+
+    def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
+        self.frames.append((jpeg_bytes, timestamp_ms))
+
+
+@dataclass
+class _RecordingSleep:
+    """Captures the durations the replayer was asked to sleep."""
+
+    sleeps: list[float] = field(default_factory=list)
+
+    def __call__(self, duration_s: float) -> None:
+        self.sleeps.append(duration_s)
+
+
+def _write_jpg(path: Path, w: int = 64, h: int = 48, fill: int = 128) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    img = np.full((h, w, 3), fill, dtype=np.uint8)
+    cv2.imwrite(str(path), img)
+
+
+def _write_video(path: Path, n_frames: int, fps: float, w: int = 64, h: int = 48) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fourcc = cv2.VideoWriter_fourcc(*"mp4v")
+    writer = cv2.VideoWriter(str(path), fourcc, fps, (w, h))
+    if not writer.isOpened():
+        pytest.skip(f"OpenCV cannot write mp4v on this platform: {path}")
+    try:
+        for i in range(n_frames):
+            img = np.full((h, w, 3), (i * 5) % 255, dtype=np.uint8)
+            writer.write(img)
+    finally:
+        writer.release()
+
+
+# replay_image_directory
+
+
+def test_image_dir_missing_raises_file_not_found(tmp_path: Path):
+    # Arrange
+    sink = _RecordingSink()
+    replayer = FrameSourceReplayer(sink, sleep_fn=lambda _: None)
+
+    # Assert
+    with pytest.raises(FileNotFoundError, match="frame directory not found"):
+        replayer.replay_image_directory(tmp_path / "nope")
+
+
+def test_image_dir_empty_returns_zero(tmp_path: Path):
+    # Arrange
+    (tmp_path / "frames").mkdir()
+    sink = _RecordingSink()
+
+    # Act
+    emitted = FrameSourceReplayer(sink, sleep_fn=lambda _: None).replay_image_directory(
+        tmp_path / "frames"
+    )
+
+    # Assert
+    assert emitted == 0
+    assert sink.frames == []
+
+
+def test_image_dir_emits_sorted_by_name(tmp_path: Path):
+    # Arrange — files written out of order; sort must restore numeric order.
+    frames_dir = tmp_path / "frames"
+    _write_jpg(frames_dir / "AD000003.jpg", fill=30)
+    _write_jpg(frames_dir / "AD000001.jpg", fill=10)
+    _write_jpg(frames_dir / "AD000002.jpg", fill=20)
+    sink = _RecordingSink()
+
+    # Act
+    emitted = FrameSourceReplayer(
+        sink,
+        cadence=ReplayCadence(fps=10.0, realtime=False),
+        sleep_fn=lambda _: None,
+    ).replay_image_directory(frames_dir)
+
+    # Assert — three frames at 0/100/200 ms.
+    assert emitted == 3
+    assert [ts for _, ts in sink.frames] == [0, 100, 200]
+    assert all(b.startswith(b"\xff\xd8") for b, _ in sink.frames)  # JPEG SOI
+
+
+def test_image_dir_non_jpeg_reencoded(tmp_path: Path):
+    # Arrange
+    frames_dir = tmp_path / "frames"
+    png_path = frames_dir / "AD000001.png"
+    frames_dir.mkdir()
+    img = np.full((48, 64, 3), 100, dtype=np.uint8)
+    cv2.imwrite(str(png_path), img)
+    sink = _RecordingSink()
+
+    # Act
+    emitted = FrameSourceReplayer(sink, sleep_fn=lambda _: None).replay_image_directory(frames_dir)
+
+    # Assert
+    assert emitted == 1
+    assert sink.frames[0][0].startswith(b"\xff\xd8")  # JPEG, not PNG
+
+
+def test_image_dir_skips_non_image_files(tmp_path: Path):
+    # Arrange
+    frames_dir = tmp_path / "frames"
+    _write_jpg(frames_dir / "AD000001.jpg")
+    (frames_dir / "README.txt").write_text("not an image")
+    (frames_dir / "manifest.csv").write_text("col1,col2\n1,2\n")
+    sink = _RecordingSink()
+
+    # Act
+    emitted = FrameSourceReplayer(sink, sleep_fn=lambda _: None).replay_image_directory(frames_dir)
+
+    # Assert
+    assert emitted == 1
+
+
+def test_image_dir_non_realtime_does_not_sleep(tmp_path: Path):
+    # Arrange
+    frames_dir = tmp_path / "frames"
+    for i in range(3):
+        _write_jpg(frames_dir / f"AD{i:06d}.jpg")
+    sink = _RecordingSink()
+    sleep = _RecordingSleep()
+
+    # Act
+    FrameSourceReplayer(
+        sink, cadence=ReplayCadence(fps=10.0, realtime=False), sleep_fn=sleep
+    ).replay_image_directory(frames_dir)
+
+    # Assert
+    assert sleep.sleeps == []
+
+
+def test_image_dir_realtime_sleeps_per_frame(tmp_path: Path):
+    # Arrange
+    frames_dir = tmp_path / "frames"
+    for i in range(3):
+        _write_jpg(frames_dir / f"AD{i:06d}.jpg")
+    sink = _RecordingSink()
+    sleep = _RecordingSleep()
+
+    # Act
+    FrameSourceReplayer(
+        sink, cadence=ReplayCadence(fps=10.0, realtime=True), sleep_fn=sleep
+    ).replay_image_directory(frames_dir)
+
+    # Assert — sleeps once per emitted frame at 0.1 s.
+    assert sleep.sleeps == pytest.approx([0.1, 0.1, 0.1])
+
+
+# replay_video
+
+
+def test_video_missing_path_raises_file_not_found(tmp_path: Path):
+    # Arrange
+    sink = _RecordingSink()
+    replayer = FrameSourceReplayer(sink, sleep_fn=lambda _: None)
+
+    # Assert
+    with pytest.raises(FileNotFoundError, match="video path not found"):
+        replayer.replay_video(tmp_path / "nope.mp4")
+
+
+def test_video_dir_delegates_to_image_directory(tmp_path: Path):
+    # Arrange
+    frames_dir = tmp_path / "frames"
+    for i in range(2):
+        _write_jpg(frames_dir / f"AD{i:06d}.jpg")
+    sink = _RecordingSink()
+
+    # Act
+    emitted = FrameSourceReplayer(
+        sink, cadence=ReplayCadence(fps=10.0, realtime=False), sleep_fn=lambda _: None
+    ).replay_video(frames_dir)
+
+    # Assert
+    assert emitted == 2
+
+
+def test_video_mp4_round_trip(tmp_path: Path):
+    # Arrange — write a 5-frame 10 FPS MP4 then replay it.
+    video_path = tmp_path / "tiny.mp4"
+    _write_video(video_path, n_frames=5, fps=10.0)
+    sink = _RecordingSink()
+
+    # Act
+    emitted = FrameSourceReplayer(
+        sink, cadence=ReplayCadence(fps=10.0, realtime=False), sleep_fn=lambda _: None
+    ).replay_video(video_path)
+
+    # Assert
+    assert emitted == 5
+    assert [ts for _, ts in sink.frames] == [0, 100, 200, 300, 400]
+    assert all(b.startswith(b"\xff\xd8") for b, _ in sink.frames)
@@ -0,0 +1,192 @@
+"""Unit tests for `e2e/runner/helpers/imu_replay.py` (AZ-594 AC-3)."""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import pytest
+
+from e2e.runner.helpers.imu_replay import FcInboundEmitter, ImuReplayer, ImuSample
+
+
+@dataclass
+class _RecordingEmitter:
+    """In-memory FcInboundEmitter that captures every sample."""
+
+    samples: list[ImuSample] = field(default_factory=list)
+
+    def emit(self, sample: ImuSample) -> None:
+        self.samples.append(sample)
+
+
+@dataclass
+class _RecordingSleep:
+    sleeps: list[float] = field(default_factory=list)
+
+    def __call__(self, duration_s: float) -> None:
+        self.sleeps.append(duration_s)
+
+
+_HEADER = "timestamp_ms,ax,ay,az,gx,gy,gz,roll_deg,pitch_deg,yaw_deg,baro_m\n"
+
+
+def _write_csv(path: Path, rows: list[str], *, header: str = _HEADER) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w") as fh:
+        fh.write(header)
+        for r in rows:
+            fh.write(r + "\n")
+
+
+def test_missing_csv_raises_file_not_found(tmp_path: Path):
+    # Arrange
+    emitter = _RecordingEmitter()
+    replayer = ImuReplayer(emitter, realtime=False)
+
+    # Assert
+    with pytest.raises(FileNotFoundError, match="IMU CSV not found"):
+        replayer.replay(tmp_path / "nope.csv")
+
+
+def test_rate_hz_must_be_positive():
+    # Assert
+    with pytest.raises(ValueError, match="rate_hz must be positive"):
+        ImuReplayer(_RecordingEmitter(), rate_hz=0)
+
+
+def test_missing_required_columns_raises(tmp_path: Path):
+    # Arrange — missing `baro_m`.
+    p = tmp_path / "imu.csv"
+    _write_csv(
+        p,
+        ["100,0,0,9.8,0,0,0,0,0,0"],
+        header="timestamp_ms,ax,ay,az,gx,gy,gz,roll_deg,pitch_deg,yaw_deg\n",
+    )
+
+    # Assert
+    with pytest.raises(ValueError, match="missing required columns"):
+        ImuReplayer(_RecordingEmitter(), realtime=False).replay(p)
+
+
+def test_row_with_unparseable_value_raises(tmp_path: Path):
+    # Arrange
+    p = tmp_path / "imu.csv"
+    _write_csv(p, ["100,0,0,not-a-float,0,0,0,0,0,0,300"])
+
+    # Assert
+    with pytest.raises(ValueError, match="IMU CSV row malformed"):
+        ImuReplayer(_RecordingEmitter(), realtime=False).replay(p)
+
+
+def test_happy_path_emits_all_rows(tmp_path: Path):
+    # Arrange
+    p = tmp_path / "imu.csv"
+    _write_csv(
+        p,
+        [
+            "100,0.0,0.0,9.8,0.0,0.0,0.0,0.0,0.0,0.0,300.0",
+            "200,0.1,0.0,9.7,0.0,0.0,0.0,0.0,0.0,0.0,300.5",
+            "300,0.0,0.2,9.6,0.0,0.0,0.0,0.0,0.0,0.0,301.0",
+        ],
+    )
+    emitter = _RecordingEmitter()
+
+    # Act
+    emitted = ImuReplayer(emitter, realtime=False).replay(p)
+
+    # Assert
+    assert emitted == 3
+    assert len(emitter.samples) == 3
+    assert emitter.samples[0].timestamp_ms == 100
+    assert emitter.samples[1].accel_mss == (0.1, 0.0, 9.7)
+
+
+def test_scientific_notation_parses(tmp_path: Path):
+    # Arrange — AZ-408 fixture style float fields.
+    p = tmp_path / "imu.csv"
+    _write_csv(
+        p,
+        [
+            "100,-4.44E-16,1.23e-3,9.81,-1e-5,2e-7,0.0,1.0,2.0,3.0,300.0",
+        ],
+    )
+    emitter = _RecordingEmitter()
+
+    # Act
+    ImuReplayer(emitter, realtime=False).replay(p)
+
+    # Assert
+    s = emitter.samples[0]
+    assert s.accel_mss[0] == pytest.approx(-4.44e-16)
+    assert s.accel_mss[1] == pytest.approx(1.23e-3)
+    assert s.accel_mss[2] == pytest.approx(9.81)
+    assert s.gyro_rps == (-1e-5, 2e-7, 0.0)
+
+
+def test_attitude_radians_converted(tmp_path: Path):
+    # Arrange
+    p = tmp_path / "imu.csv"
+    _write_csv(
+        p,
+        ["100,0,0,9.8,0,0,0,90,180,270,300"],
+    )
+    emitter = _RecordingEmitter()
+
+    # Act
+    ImuReplayer(emitter, realtime=False).replay(p)
+
+    # Assert
+    roll, pitch, yaw = emitter.samples[0].attitude_rad
+    assert roll == pytest.approx(math.pi / 2)
+    assert pitch == pytest.approx(math.pi)
+    assert yaw == pytest.approx(3 * math.pi / 2)
+
+
+def test_realtime_sleeps_per_sample(tmp_path: Path):
+    # Arrange — 3 rows at 10 Hz → 3 sleeps of 0.1 s.
+    p = tmp_path / "imu.csv"
+    _write_csv(
+        p,
+        [
+            "100,0,0,9.8,0,0,0,0,0,0,300",
+            "200,0,0,9.8,0,0,0,0,0,0,300",
+            "300,0,0,9.8,0,0,0,0,0,0,300",
+        ],
+    )
+    emitter = _RecordingEmitter()
+    sleep = _RecordingSleep()
+
+    # Act
+    ImuReplayer(emitter, rate_hz=10.0, realtime=True, sleep_fn=sleep).replay(p)
+
+    # Assert
+    assert sleep.sleeps == pytest.approx([0.1, 0.1, 0.1])
+
+
+def test_non_realtime_does_not_sleep(tmp_path: Path):
+    # Arrange
+    p = tmp_path / "imu.csv"
+    _write_csv(p, ["100,0,0,9.8,0,0,0,0,0,0,300"])
+    sleep = _RecordingSleep()
+
+    # Act
+    ImuReplayer(_RecordingEmitter(), realtime=False, sleep_fn=sleep).replay(p)
+
+    # Assert
+    assert sleep.sleeps == []
+
+
+def test_empty_csv_emits_nothing(tmp_path: Path):
+    # Arrange — header only.
+    p = tmp_path / "imu.csv"
+    _write_csv(p, [])
+    emitter = _RecordingEmitter()
+
+    # Act
+    emitted = ImuReplayer(emitter, realtime=False).replay(p)
+
+    # Assert
+    assert emitted == 0
+    assert emitter.samples == []