mirror of
https://github.com/azaion/gps-denied-onboard.git
synced 2026-06-22 06:41:14 +00:00
[AZ-414] [AZ-415] [AZ-418] Test batch 71: sharp turn + multi-segment + smoothing
- AZ-414 (FT-P-07 + FT-N-02): sharp_turn_detector helper covering AC-1 (gyro_z run detection + synthetic-overlay fallback), AC-2/AC-3 (FT-N-02 during-turn label + monotonic covariance), AC-4/AC-5/AC-6 (FT-P-07 recovery lag/drift/heading); twin scenario files under positive/ and negative/. - AZ-415 (FT-P-08): multi_segment_evaluator helper + scenario. - AZ-418 (FT-P-10): smoothing_evaluator helper covering AC-1 (raw + smoothed pose pairing), AC-2 (improvement rate >= 0.80), AC-3 (mean improvement >= 5 m); scenario file. - All scenarios skip-gated on upstream frame_source_replay / imu_replay / fdr_reader stubs (auto-activate when AZ-441 + AZ-407 leftovers land). - +68 unit tests; full e2e unit suite: 393 passed. See _docs/03_implementation/batch_71_report.md and _docs/03_implementation/reviews/batch_71_review.md. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Oleksandr Bezdieniezhnykh
@@ -0,0 +1,176 @@
|
||||
"""FT-N-02 — Sharp-turn legitimate failure (AZ-414 / AC-3.2).
|
||||
|
||||
The negative twin of FT-P-07. Same detection / fixture / replay path;
|
||||
the assertion side checks behaviour DURING the turn (not recovery
|
||||
after it):
|
||||
|
||||
* AC-2: source_label ∈ ``{visual_propagated, dead_reckoned}`` for every
|
||||
inside-window frame (no ``satellite_anchored`` during the turn).
|
||||
* AC-3: ``cov_semi_major_m`` is non-decreasing across consecutive
|
||||
frames within the segment.
|
||||
|
||||
The recovery half (AC-4/5/6) is owned by FT-P-07
|
||||
(``e2e/tests/positive/test_ft_p_07_sharp_turn_recovery.py``); this file
|
||||
delegates the helper call but does not assert on the returned report.
|
||||
|
||||
Gated on the same upstream replay helpers as FT-P-07.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from runner.helpers import sharp_turn_detector as std
|
||||
|
||||
DERKACHI_DIR = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "_docs"
|
||||
/ "00_problem"
|
||||
/ "input_data"
|
||||
/ "flight_derkachi"
|
||||
)
|
||||
DERKACHI_IMU_CSV = DERKACHI_DIR / "data_imu.csv"
|
||||
DERKACHI_MP4 = DERKACHI_DIR / "flight_derkachi.mp4"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def _harness_helpers_implemented() -> bool:
|
||||
from runner.helpers import fdr_reader, imu_replay
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
try:
|
||||
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
|
||||
try:
|
||||
replayer.replay_video(Path("/tmp/non-existent.mp4"))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
imu_replay.ImuReplayer(emitter=_NullImuEmitter()).replay(Path("/tmp/non-existent.csv")) # type: ignore[arg-type]
|
||||
except NotImplementedError:
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class _NullSink:
|
||||
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class _NullImuEmitter:
|
||||
def emit(self, sample: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
@pytest.mark.traces_to("AC-3.2,AC-1,AC-2,AC-3,AC-7")
|
||||
def test_ft_n_02_sharp_turn_failure(
|
||||
fc_adapter: str,
|
||||
vio_strategy: str,
|
||||
evidence_dir, # type: ignore[no-untyped-def]
|
||||
run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
_harness_helpers_implemented: bool,
|
||||
) -> None:
|
||||
if not _harness_helpers_implemented:
|
||||
pytest.skip(
|
||||
"FT-N-02 full replay requires runner.helpers.{frame_source_replay,"
|
||||
"imu_replay,fdr_reader} — currently AZ-441 / AZ-407 leftovers. "
|
||||
"AC-2/AC-3 helper logic covered by "
|
||||
"e2e/_unit_tests/helpers/test_sharp_turn_detector.py."
|
||||
)
|
||||
|
||||
from runner.helpers import fdr_reader
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
# 1. AC-1 — identify or synthesise.
|
||||
detection = std.detect_or_synthesize(DERKACHI_IMU_CSV)
|
||||
assert detection.segments, "AC-1: at least one turn segment required"
|
||||
|
||||
# 2. Drive replay.
|
||||
FrameSourceReplayer(_resolve_frame_sink()).replay_video(DERKACHI_MP4)
|
||||
_drive_imu_replay(DERKACHI_IMU_CSV)
|
||||
|
||||
# 3. Collect samples.
|
||||
fdr_root = Path(evidence_dir).parent / f"run-{run_id}" / "fdr"
|
||||
samples: list[std.TurnFrameSample] = []
|
||||
for rec in fdr_reader.iter_records(fdr_root):
|
||||
if rec.record_type != "outbound_estimate":
|
||||
continue
|
||||
payload = rec.payload
|
||||
samples.append(
|
||||
std.TurnFrameSample(
|
||||
monotonic_ms=int(rec.monotonic_ms),
|
||||
lat_deg=float(payload["lat_deg"]), # type: ignore[arg-type]
|
||||
lon_deg=float(payload["lon_deg"]), # type: ignore[arg-type]
|
||||
source_label=str(payload["source_label"]), # type: ignore[arg-type]
|
||||
cov_semi_major_m=float(payload["cov_semi_major_m"]), # type: ignore[arg-type]
|
||||
)
|
||||
)
|
||||
|
||||
if not samples:
|
||||
pytest.fail("FT-N-02: no outbound_estimate records produced")
|
||||
|
||||
# 4. Evaluate per segment.
|
||||
n02_reports = [
|
||||
std.evaluate_ft_n_02(seg, idx, samples)
|
||||
for idx, seg in enumerate(detection.segments)
|
||||
]
|
||||
p07_reports = [
|
||||
std.evaluate_ft_p_07(seg, idx, samples)
|
||||
for idx, seg in enumerate(detection.segments)
|
||||
]
|
||||
out_csv = evidence_dir / f"ft-n-02-{fc_adapter}-{vio_strategy}.csv"
|
||||
std.write_csv_evidence(out_csv, detection, n02_reports, p07_reports)
|
||||
|
||||
# 5. NFR metrics + AC assertions (NEGATIVE twin assertions).
|
||||
for r in n02_reports:
|
||||
nfr_recorder.record_metric(
|
||||
f"ft_n_02.seg_{r.segment_index}.samples_inside",
|
||||
float(r.samples_inside),
|
||||
ac_id="AC-2",
|
||||
)
|
||||
nfr_recorder.record_metric(
|
||||
f"ft_n_02.seg_{r.segment_index}.label_violation_count",
|
||||
float(len(r.label_violations)),
|
||||
ac_id="AC-2",
|
||||
)
|
||||
nfr_recorder.record_metric(
|
||||
f"ft_n_02.seg_{r.segment_index}.cov_non_decreasing",
|
||||
1.0 if r.cov_non_decreasing else 0.0,
|
||||
ac_id="AC-3",
|
||||
)
|
||||
|
||||
nfr_recorder.record_metric(
|
||||
"ft_n_02.synthetic_overlay",
|
||||
1.0 if detection.synthetic_overlay else 0.0,
|
||||
ac_id="AC-1",
|
||||
)
|
||||
|
||||
for r in n02_reports:
|
||||
assert r.passes_label, (
|
||||
f"AC-2 (label ∈ {sorted(std.ALLOWED_DURING_TURN_LABELS)}) failed for segment "
|
||||
f"{r.segment_index}: violations={r.label_violations}, inside={r.samples_inside}"
|
||||
)
|
||||
assert r.passes_cov, (
|
||||
f"AC-3 (non-decreasing cov_semi_major_m) failed for segment "
|
||||
f"{r.segment_index}: first_decreasing_at_ms={r.first_decreasing_at_ms}"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_frame_sink(): # type: ignore[no-untyped-def]
|
||||
raise NotImplementedError(
|
||||
"frame sink resolution is owned by AZ-441 / runner.helpers.frame_source_replay"
|
||||
)
|
||||
|
||||
|
||||
def _drive_imu_replay(csv_path: Path) -> None:
|
||||
raise NotImplementedError(
|
||||
"IMU replay driver is owned by AZ-416/AZ-417 / runner.helpers.imu_replay"
|
||||
)
|
||||
@@ -0,0 +1,195 @@
|
||||
"""FT-P-07 — Sharp-turn recovery via satellite reference (AZ-414 / AC-3.2).
|
||||
|
||||
The full scenario:
|
||||
|
||||
1. Identify the sharp-turn segment in the Derkachi flight via
|
||||
``SCALED_IMU2.zgyro`` spikes (≥3 consecutive rows above the AC-3.2
|
||||
threshold). If none exist, fall back to a synthetic-gyro overlay
|
||||
(the choice is recorded in the evidence CSV per AC-1).
|
||||
2. Replay the Derkachi MP4 + IMU stream through the SUT.
|
||||
3. Collect outbound estimates with source_label + cov_semi_major_m.
|
||||
4. Per turn segment, evaluate:
|
||||
* AC-4: recovery to ``satellite_anchored`` within ≤3 frames after
|
||||
turn end (safety-budget converted to ms in the helper).
|
||||
* AC-5: drift between propagated centre at turn end and recovery
|
||||
anchor centre ≤200 m.
|
||||
* AC-6: heading delta (pre-anchor → propagated-end → recovery)
|
||||
stays in [0°, 70°].
|
||||
5. Assert FT-P-07 passes per ``(fc_adapter, vio_strategy)`` (AC-7).
|
||||
|
||||
FT-N-02 (the negative twin) is owned by
|
||||
``e2e/tests/negative/test_ft_n_02_sharp_turn_failure.py`` and shares
|
||||
the same detection + ``TurnFrameSample`` collection logic via
|
||||
``runner.helpers.sharp_turn_detector``.
|
||||
|
||||
This scenario is gated on the upstream replay helpers
|
||||
(``frame_source_replay``, ``imu_replay``, ``fdr_reader``); pure-logic
|
||||
coverage lives in ``e2e/_unit_tests/helpers/test_sharp_turn_detector.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from runner.helpers import sharp_turn_detector as std
|
||||
|
||||
DERKACHI_DIR = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "_docs"
|
||||
/ "00_problem"
|
||||
/ "input_data"
|
||||
/ "flight_derkachi"
|
||||
)
|
||||
DERKACHI_IMU_CSV = DERKACHI_DIR / "data_imu.csv"
|
||||
DERKACHI_MP4 = DERKACHI_DIR / "flight_derkachi.mp4"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def _harness_helpers_implemented() -> bool:
|
||||
"""True iff replay + IMU + FDR helpers are real."""
|
||||
from runner.helpers import fdr_reader, imu_replay
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
try:
|
||||
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
|
||||
try:
|
||||
replayer.replay_video(Path("/tmp/non-existent.mp4"))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
imu_replay.ImuReplayer(emitter=_NullImuEmitter()).replay(Path("/tmp/non-existent.csv")) # type: ignore[arg-type]
|
||||
except NotImplementedError:
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class _NullSink:
|
||||
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class _NullImuEmitter:
|
||||
def emit(self, sample: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
@pytest.mark.traces_to("AC-3.2,AC-1,AC-4,AC-5,AC-6,AC-7")
|
||||
def test_ft_p_07_sharp_turn_recovery(
|
||||
fc_adapter: str,
|
||||
vio_strategy: str,
|
||||
evidence_dir, # type: ignore[no-untyped-def]
|
||||
run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
_harness_helpers_implemented: bool,
|
||||
) -> None:
|
||||
if not _harness_helpers_implemented:
|
||||
pytest.skip(
|
||||
"FT-P-07 full replay requires runner.helpers.{frame_source_replay,"
|
||||
"imu_replay,fdr_reader} — currently AZ-441 / AZ-407 leftovers. "
|
||||
"AC-1/AC-4/AC-5/AC-6 helper logic covered by "
|
||||
"e2e/_unit_tests/helpers/test_sharp_turn_detector.py."
|
||||
)
|
||||
|
||||
from runner.helpers import fdr_reader
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
# 1. AC-1 — identify or synthesise the sharp-turn segment.
|
||||
detection = std.detect_or_synthesize(DERKACHI_IMU_CSV)
|
||||
assert detection.segments, "AC-1: at least one turn segment (natural or synthetic) required"
|
||||
|
||||
# 2. Drive replay.
|
||||
FrameSourceReplayer(_resolve_frame_sink()).replay_video(DERKACHI_MP4)
|
||||
_drive_imu_replay(DERKACHI_IMU_CSV)
|
||||
|
||||
# 3. Collect outbound estimates as TurnFrameSample.
|
||||
fdr_root = Path(evidence_dir).parent / f"run-{run_id}" / "fdr"
|
||||
samples: list[std.TurnFrameSample] = []
|
||||
for rec in fdr_reader.iter_records(fdr_root):
|
||||
if rec.record_type != "outbound_estimate":
|
||||
continue
|
||||
payload = rec.payload
|
||||
samples.append(
|
||||
std.TurnFrameSample(
|
||||
monotonic_ms=int(rec.monotonic_ms),
|
||||
lat_deg=float(payload["lat_deg"]), # type: ignore[arg-type]
|
||||
lon_deg=float(payload["lon_deg"]), # type: ignore[arg-type]
|
||||
source_label=str(payload["source_label"]), # type: ignore[arg-type]
|
||||
cov_semi_major_m=float(payload["cov_semi_major_m"]), # type: ignore[arg-type]
|
||||
)
|
||||
)
|
||||
|
||||
if not samples:
|
||||
pytest.fail("FT-P-07: no outbound_estimate records produced")
|
||||
|
||||
# 4. Evaluate per segment (recovery side only — FT-N-02 owns label/cov).
|
||||
p07_reports = [
|
||||
std.evaluate_ft_p_07(seg, idx, samples)
|
||||
for idx, seg in enumerate(detection.segments)
|
||||
]
|
||||
n02_reports = [
|
||||
std.evaluate_ft_n_02(seg, idx, samples)
|
||||
for idx, seg in enumerate(detection.segments)
|
||||
]
|
||||
out_csv = evidence_dir / f"ft-p-07-{fc_adapter}-{vio_strategy}.csv"
|
||||
std.write_csv_evidence(out_csv, detection, n02_reports, p07_reports)
|
||||
|
||||
# 5. NFR metrics + AC assertions.
|
||||
for r in p07_reports:
|
||||
if r.recovery_lag_ms is not None:
|
||||
nfr_recorder.record_metric(
|
||||
f"ft_p_07.seg_{r.segment_index}.recovery_lag_ms",
|
||||
float(r.recovery_lag_ms),
|
||||
ac_id="AC-4",
|
||||
)
|
||||
if r.drift_m is not None:
|
||||
nfr_recorder.record_metric(
|
||||
f"ft_p_07.seg_{r.segment_index}.drift_m",
|
||||
r.drift_m,
|
||||
ac_id="AC-5",
|
||||
)
|
||||
if r.heading_change_deg is not None:
|
||||
nfr_recorder.record_metric(
|
||||
f"ft_p_07.seg_{r.segment_index}.heading_change_deg",
|
||||
r.heading_change_deg,
|
||||
ac_id="AC-6",
|
||||
)
|
||||
|
||||
nfr_recorder.record_metric(
|
||||
"ft_p_07.synthetic_overlay",
|
||||
1.0 if detection.synthetic_overlay else 0.0,
|
||||
ac_id="AC-1",
|
||||
)
|
||||
|
||||
for r in p07_reports:
|
||||
assert r.passes_recovery_lag, (
|
||||
f"AC-4 (recovery ≤{std.MAX_RECOVERY_FRAMES_SAFETY_MS} ms) failed for segment "
|
||||
f"{r.segment_index}: recovery_lag_ms={r.recovery_lag_ms}"
|
||||
)
|
||||
assert r.passes_drift, (
|
||||
f"AC-5 (drift ≤{std.MAX_RECOVERY_DRIFT_M} m) failed for segment "
|
||||
f"{r.segment_index}: drift_m={r.drift_m}"
|
||||
)
|
||||
assert r.passes_heading, (
|
||||
f"AC-6 (heading delta ≤{std.MAX_HEADING_CHANGE_DEG}°) failed for segment "
|
||||
f"{r.segment_index}: heading_change_deg={r.heading_change_deg}"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_frame_sink(): # type: ignore[no-untyped-def]
|
||||
raise NotImplementedError(
|
||||
"frame sink resolution is owned by AZ-441 / runner.helpers.frame_source_replay"
|
||||
)
|
||||
|
||||
|
||||
def _drive_imu_replay(csv_path: Path) -> None:
|
||||
raise NotImplementedError(
|
||||
"IMU replay driver is owned by AZ-416/AZ-417 / runner.helpers.imu_replay"
|
||||
)
|
||||
@@ -0,0 +1,163 @@
|
||||
"""FT-P-08 — Multi-segment satellite-reference relocalisation (AC-3.3).
|
||||
|
||||
The full scenario:
|
||||
|
||||
1. Build the ``multi-segment-derkachi`` fixture (AZ-408 ``multi_segment``
|
||||
injector) via the ``multi_segment_derkachi`` pytest fixture.
|
||||
2. Replay the fixture's frames through the SUT.
|
||||
3. Read the SUT's outbound estimate stream + post-run FDR archive.
|
||||
4. For each of the ≥3 blackout windows in ``schedule.json``:
|
||||
- AC-2: assert every inside-window estimate has source_label = dead_reckoned.
|
||||
- AC-3: assert a satellite_anchored emission within 3 frames of end_ms.
|
||||
- AC-4: assert the recovery anchor is within 100 m of the last
|
||||
pre-recovery estimate.
|
||||
5. Emit ``ft-p-08-{fc_adapter}-{vio_strategy}.csv`` for evidence.
|
||||
|
||||
What this file owns:
|
||||
|
||||
* AC-1 / AC-2 / AC-3 / AC-4 / AC-5 wiring above.
|
||||
* CSV evidence emission via the AZ-415-owned ``multi_segment_evaluator``.
|
||||
|
||||
What this file does NOT own:
|
||||
|
||||
* The frame-source push → ``runner.helpers.frame_source_replay`` (stub;
|
||||
AZ-441) — skip-gated.
|
||||
* The FDR-archive iteration → ``runner.helpers.fdr_reader`` (stub;
|
||||
AZ-441) — skip-gated.
|
||||
|
||||
When both upstream helpers land, this file's runtime path activates
|
||||
automatically.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from runner.helpers import multi_segment_evaluator as mse
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def _harness_helpers_implemented() -> bool:
|
||||
"""True iff replay + FDR helpers are real."""
|
||||
from runner.helpers import fdr_reader, frame_source_replay
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
try:
|
||||
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
|
||||
try:
|
||||
replayer.replay_image_directory(Path("/tmp/non-existent"))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class _NullSink:
|
||||
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
@pytest.mark.traces_to("AC-3.3,AC-1,AC-2,AC-3,AC-4,AC-5")
|
||||
def test_ft_p_08_multi_segment_reloc(
|
||||
fc_adapter: str,
|
||||
vio_strategy: str,
|
||||
evidence_dir, # type: ignore[no-untyped-def]
|
||||
run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
multi_segment_derkachi, # type: ignore[no-untyped-def] # AZ-408 pytest fixture
|
||||
_harness_helpers_implemented: bool,
|
||||
) -> None:
|
||||
"""Full FT-P-08 scenario.
|
||||
|
||||
AC-1: blackout windows detected from schedule.json (≥3).
|
||||
AC-2: dead_reckoned inside every window.
|
||||
AC-3: recovery to satellite_anchored within ≤3 frames of end_ms.
|
||||
AC-4: trajectory continuity ≤100 m at each recovery.
|
||||
AC-5: parameterised across ``(fc_adapter, vio_strategy)``.
|
||||
"""
|
||||
if not _harness_helpers_implemented:
|
||||
pytest.skip(
|
||||
"FT-P-08 multi-segment replay requires runner.helpers.{frame_source_replay,"
|
||||
"fdr_reader} — currently AZ-441 leftover. Pure-logic ACs covered by "
|
||||
"e2e/_unit_tests/helpers/test_multi_segment_evaluator.py."
|
||||
)
|
||||
|
||||
from runner.helpers import fdr_reader
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
# 1. Load the injector's schedule.
|
||||
schedule = mse.load_schedule(multi_segment_derkachi.out_root / "schedule.json")
|
||||
if len(schedule) < mse.MIN_SEGMENTS_REQUIRED:
|
||||
pytest.fail(
|
||||
f"FT-P-08 requires ≥{mse.MIN_SEGMENTS_REQUIRED} blackout windows; "
|
||||
f"injector produced {len(schedule)}"
|
||||
)
|
||||
|
||||
# 2. Replay the fixture.
|
||||
sink = _resolve_frame_sink()
|
||||
FrameSourceReplayer(sink).replay_image_directory(multi_segment_derkachi.out_root)
|
||||
|
||||
# 3. Collect samples from the FDR archive.
|
||||
fdr_root = Path(evidence_dir).parent / f"run-{run_id}" / "fdr"
|
||||
samples: list[mse.EstimateSample] = []
|
||||
for rec in fdr_reader.iter_records(fdr_root):
|
||||
if rec.record_type == "estimate":
|
||||
payload = rec.payload
|
||||
samples.append(
|
||||
mse.EstimateSample(
|
||||
monotonic_ms=int(rec.monotonic_ms),
|
||||
lat_deg=float(payload["lat_deg"]), # type: ignore[arg-type]
|
||||
lon_deg=float(payload["lon_deg"]), # type: ignore[arg-type]
|
||||
source_label=str(payload["source_label"]), # type: ignore[arg-type]
|
||||
)
|
||||
)
|
||||
|
||||
# 4. Evaluate + emit evidence.
|
||||
report = mse.evaluate(schedule, samples)
|
||||
out_csv = evidence_dir / f"ft-p-08-{fc_adapter}-{vio_strategy}.csv"
|
||||
mse.write_csv_evidence(out_csv, report)
|
||||
|
||||
# 5. NFR metrics.
|
||||
nfr_recorder.record_metric(
|
||||
"ft_p_08.window_count", float(report.window_count), ac_id="AC-1"
|
||||
)
|
||||
nfr_recorder.record_metric(
|
||||
"ft_p_08.failed_windows", float(len(report.failed_windows)), ac_id="AC-2"
|
||||
)
|
||||
for w in report.per_window:
|
||||
if w.recovery_lag_ms is not None:
|
||||
nfr_recorder.record_metric(
|
||||
f"ft_p_08.window_{w.window_index}.recovery_lag_ms",
|
||||
float(w.recovery_lag_ms),
|
||||
ac_id="AC-3",
|
||||
)
|
||||
if w.trajectory_jump_m is not None:
|
||||
nfr_recorder.record_metric(
|
||||
f"ft_p_08.window_{w.window_index}.trajectory_jump_m",
|
||||
w.trajectory_jump_m,
|
||||
ac_id="AC-4",
|
||||
)
|
||||
|
||||
# 6. AC assertions.
|
||||
assert report.passes, (
|
||||
f"FT-P-08 failed: {len(report.failed_windows)} of {report.window_count} "
|
||||
f"windows failed. Per-window: "
|
||||
+ ", ".join(
|
||||
f"#{w.window_index}(label={w.passes_label},rec={w.passes_recovery},"
|
||||
f"jump={w.passes_jump})"
|
||||
for w in report.per_window
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _resolve_frame_sink(): # type: ignore[no-untyped-def]
|
||||
raise NotImplementedError(
|
||||
"frame sink resolution is owned by AZ-441 / runner.helpers.frame_source_replay"
|
||||
)
|
||||
@@ -0,0 +1,210 @@
|
||||
"""FT-P-10 — GTSAM smoothing-loop look-back accuracy (AC-4.5 revised).
|
||||
|
||||
The full scenario:
|
||||
|
||||
1. Replay Derkachi end-to-end through the SUT.
|
||||
2. Read the post-run FDR archive for per-past-keyframe pose records;
|
||||
per AC-NEW-3 the SUT emits two records per past keyframe:
|
||||
``pose_kind = "raw"`` (single-shot at first emission) and
|
||||
``pose_kind = "smoothed"`` (iSAM2-converged at smoother window exit).
|
||||
3. Load Derkachi ``data_imu.csv`` and extract ``GLOBAL_POSITION_INT``
|
||||
GT poses (10 Hz).
|
||||
4. Pair raw + smoothed per keyframe; compute distance(raw, GT) and
|
||||
distance(smoothed, GT); aggregate.
|
||||
5. Assert AC-2 (improvement_rate ≥ 0.80) AND AC-3 (mean_improvement ≥ 5 m).
|
||||
|
||||
What this file owns:
|
||||
|
||||
* AC-1 / AC-2 / AC-3 / AC-4 wiring above.
|
||||
* Per-strategy reporting (the spec calls out that vins_mono ≥ okvis2 ≥
|
||||
klt_ransac is expected even if all pass).
|
||||
|
||||
What this file does NOT own:
|
||||
|
||||
* The MP4 replay → ``runner.helpers.frame_source_replay`` (stub) — gated.
|
||||
* The IMU CSV replay → ``runner.helpers.imu_replay`` (stub) — gated.
|
||||
* The FDR-archive iteration → ``runner.helpers.fdr_reader`` (stub) — gated.
|
||||
|
||||
When the upstream helpers land, this file's runtime path activates
|
||||
automatically.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from runner.helpers import smoothing_evaluator as se
|
||||
|
||||
DERKACHI_DIR = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "_docs"
|
||||
/ "00_problem"
|
||||
/ "input_data"
|
||||
/ "flight_derkachi"
|
||||
)
|
||||
DERKACHI_IMU_CSV = DERKACHI_DIR / "data_imu.csv"
|
||||
DERKACHI_MP4 = DERKACHI_DIR / "flight_derkachi.mp4"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def _harness_helpers_implemented() -> bool:
|
||||
"""True iff replay + IMU + FDR helpers are real."""
|
||||
from runner.helpers import fdr_reader, frame_source_replay, imu_replay
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
try:
|
||||
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
|
||||
try:
|
||||
replayer.replay_video(Path("/tmp/non-existent.mp4"))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
imu_replay.ImuReplayer(emitter=_NullImuEmitter()).replay(Path("/tmp/non-existent.csv")) # type: ignore[arg-type]
|
||||
except NotImplementedError:
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class _NullSink:
|
||||
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class _NullImuEmitter:
|
||||
def emit(self, sample: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _load_derkachi_gt_track() -> list[se.GtPose]:
|
||||
"""Read GLOBAL_POSITION_INT poses from data_imu.csv.
|
||||
|
||||
lat / lon are stored as decimal degrees (e.g. 50.0809634), NOT 1e-7
|
||||
int32. The column names confirm this: ``GLOBAL_POSITION_INT.lat`` /
|
||||
``GLOBAL_POSITION_INT.lon`` per the CSV header.
|
||||
"""
|
||||
track: list[se.GtPose] = []
|
||||
with DERKACHI_IMU_CSV.open() as fh:
|
||||
reader = csv.DictReader(fh)
|
||||
for row in reader:
|
||||
ts = row.get("timestamp(ms)", "").strip()
|
||||
if not ts:
|
||||
continue
|
||||
track.append(
|
||||
se.GtPose(
|
||||
monotonic_ms=int(float(ts)),
|
||||
lat_deg=float(row["GLOBAL_POSITION_INT.lat"]),
|
||||
lon_deg=float(row["GLOBAL_POSITION_INT.lon"]),
|
||||
)
|
||||
)
|
||||
return track
|
||||
|
||||
|
||||
@pytest.mark.traces_to("AC-4.5,AC-1,AC-2,AC-3,AC-4")
|
||||
def test_ft_p_10_smoothing_lookback(
|
||||
fc_adapter: str,
|
||||
vio_strategy: str,
|
||||
evidence_dir, # type: ignore[no-untyped-def]
|
||||
run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
_harness_helpers_implemented: bool,
|
||||
) -> None:
|
||||
"""Full FT-P-10 scenario.
|
||||
|
||||
AC-1: FDR contains raw + smoothed per past keyframe — verified at
|
||||
the pairing step (orphan = excluded).
|
||||
AC-2: improvement_rate ≥ 0.80.
|
||||
AC-3: mean_improvement_m ≥ 5 m.
|
||||
AC-4: parameterised across ``(fc_adapter, vio_strategy)``.
|
||||
"""
|
||||
if not _harness_helpers_implemented:
|
||||
pytest.skip(
|
||||
"FT-P-10 full replay requires runner.helpers.{frame_source_replay,"
|
||||
"imu_replay,fdr_reader} — currently AZ-441 / AZ-407 leftovers. "
|
||||
"Pure-logic ACs covered by e2e/_unit_tests/helpers/test_smoothing_evaluator.py."
|
||||
)
|
||||
|
||||
from runner.helpers import fdr_reader, imu_replay
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
# 1. Drive replay.
|
||||
sink = _resolve_frame_sink()
|
||||
emitter = _resolve_fc_inbound_emitter(fc_adapter)
|
||||
FrameSourceReplayer(sink).replay_video(DERKACHI_MP4)
|
||||
imu_replay.ImuReplayer(emitter).replay(DERKACHI_IMU_CSV)
|
||||
|
||||
# 2. Collect raw + smoothed pose records from the FDR archive.
|
||||
fdr_root = Path(evidence_dir).parent / f"run-{run_id}" / "fdr"
|
||||
records: list[se.KeyframePoseRecord] = []
|
||||
for rec in fdr_reader.iter_records(fdr_root):
|
||||
if rec.record_type == "keyframe_pose":
|
||||
payload = rec.payload
|
||||
records.append(
|
||||
se.KeyframePoseRecord(
|
||||
keyframe_id=int(payload["keyframe_id"]), # type: ignore[arg-type]
|
||||
pose_kind=str(payload["pose_kind"]), # type: ignore[arg-type]
|
||||
monotonic_ms=int(rec.monotonic_ms),
|
||||
lat_deg=float(payload["lat_deg"]), # type: ignore[arg-type]
|
||||
lon_deg=float(payload["lon_deg"]), # type: ignore[arg-type]
|
||||
)
|
||||
)
|
||||
|
||||
if not records:
|
||||
pytest.fail(
|
||||
"FT-P-10: SUT did not emit any keyframe_pose FDR records "
|
||||
"(AC-1 / AC-NEW-3 requires raw + smoothed per past keyframe)."
|
||||
)
|
||||
|
||||
# 3. Load Derkachi GT track.
|
||||
gt_track = _load_derkachi_gt_track()
|
||||
if not gt_track:
|
||||
pytest.fail(f"FT-P-10: empty GT track loaded from {DERKACHI_IMU_CSV}")
|
||||
|
||||
# 4. Evaluate + emit evidence.
|
||||
report = se.evaluate(records, gt_track)
|
||||
out_csv = evidence_dir / f"ft-p-10-{fc_adapter}-{vio_strategy}.csv"
|
||||
se.write_csv_evidence(out_csv, report)
|
||||
|
||||
# 5. NFR metrics (per-strategy comparability per AC-4).
|
||||
nfr_recorder.record_metric(
|
||||
"ft_p_10.improvement_rate", report.improvement_rate, ac_id="AC-2"
|
||||
)
|
||||
nfr_recorder.record_metric(
|
||||
"ft_p_10.mean_improvement_m", report.mean_improvement_m, ac_id="AC-3"
|
||||
)
|
||||
nfr_recorder.record_metric(
|
||||
"ft_p_10.median_improvement_m", report.median_improvement_m, ac_id="AC-3"
|
||||
)
|
||||
nfr_recorder.record_metric(
|
||||
"ft_p_10.pair_count", float(report.pair_count), ac_id="AC-1"
|
||||
)
|
||||
|
||||
# 6. AC assertions.
|
||||
assert report.passes_rate, (
|
||||
f"AC-2 (improvement rate ≥{se.IMPROVEMENT_RATE_REQUIRED:.0%}) failed: "
|
||||
f"{report.improvement_rate:.4f} over {report.pair_count} keyframes"
|
||||
)
|
||||
assert report.passes_mean, (
|
||||
f"AC-3 (mean improvement ≥{se.MEAN_IMPROVEMENT_M_REQUIRED} m) failed: "
|
||||
f"mean={report.mean_improvement_m:.3f} m, median={report.median_improvement_m:.3f} m"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_frame_sink(): # type: ignore[no-untyped-def]
|
||||
raise NotImplementedError(
|
||||
"frame sink resolution is owned by AZ-441 / runner.helpers.frame_source_replay"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_fc_inbound_emitter(fc_adapter: str): # type: ignore[no-untyped-def]
|
||||
raise NotImplementedError(
|
||||
"FC inbound emitter resolution is owned by AZ-416/AZ-417 / runner.helpers.imu_replay"
|
||||
)
|
||||
Reference in New Issue
Block a user