mirror of
https://github.com/azaion/gps-denied-onboard.git
synced 2026-06-22 22:31:12 +00:00
[AZ-424] [AZ-425] [AZ-426] Implement negatives set (FT-N-01/03/04)
Adds three pure-logic evaluators + scenarios + unit tests covering the project's failure-mode robustness ladder (AC-3.1, AC-3.4, AC-3.5, AC-NEW-8): * outlier_tolerance_evaluator (AZ-424 / FT-N-01): per-event 50 m drift bound + 3-frame covariance-monotonic window over the AZ-408 outlier injector's medium-density manifest. * outage_request_evaluator (AZ-425 / FT-N-03): detects 3+ consecutive missing-frame windows; validates OPERATOR_RELOC_REQUEST STATUSTEXT arrives at 2 s ±500 ms, dead_reckoned label during outage, and no FC EKF divergence. * blackout_spoof_evaluator (AZ-426 / FT-N-04): eight-AC ladder across the 5 s / 15 s / 35 s sub-windows — switch latency, spoof rejection, monotonic covariance, honest horiz_accuracy, STATUSTEXT 1-2 Hz, 35 s escalation thresholds, and recovery gate. Each scenario is skip-gated on the AZ-441 / AZ-407 / AZ-416 replay / SITL / mavproxy helpers; unit tests (14 + 18 + 29 = 61) cover the AC logic today. Full e2e unit-test suite: 527 passed (+67). Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -0,0 +1,170 @@
|
||||
"""FT-N-01 — 350 m outlier injection tolerance (AZ-424 / AC-3.1).
|
||||
|
||||
Replays the Derkachi flight with the AZ-408 ``outlier`` injector at
|
||||
``--density medium`` and verifies AC-1 / AC-2 / AC-3 via
|
||||
``runner.helpers.outlier_tolerance_evaluator``.
|
||||
|
||||
Gated on the same upstream replay helpers as FT-N-02 / FT-P-07
|
||||
(``frame_source_replay``, ``fdr_reader``, ``imu_replay``). When those
|
||||
helpers are still stubbed (current state under AZ-441 / AZ-407
|
||||
leftovers), the scenario test skips while
|
||||
``e2e/_unit_tests/helpers/test_outlier_tolerance_evaluator.py`` covers
|
||||
the pure-logic AC-2 / AC-3 invariants.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from fixtures.injectors.outlier import OutlierInjectionReport
|
||||
from runner.helpers import outlier_tolerance_evaluator as ote
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def _harness_helpers_implemented() -> bool:
|
||||
from runner.helpers import fdr_reader, imu_replay
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
try:
|
||||
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
|
||||
try:
|
||||
replayer.replay_video(Path("/tmp/non-existent.mp4"))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
imu_replay.ImuReplayer(emitter=_NullImuEmitter()).replay(Path("/tmp/non-existent.csv")) # type: ignore[arg-type]
|
||||
except NotImplementedError:
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class _NullSink:
|
||||
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class _NullImuEmitter:
|
||||
def emit(self, sample: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"outlier_injection_derkachi",
|
||||
[{"density": "medium", "seed": 0}],
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.traces_to("AC-3.1,AC-1,AC-2,AC-3,AC-4")
|
||||
def test_ft_n_01_outlier_tolerance(
|
||||
fc_adapter: str,
|
||||
vio_strategy: str,
|
||||
outlier_injection_derkachi: OutlierInjectionReport,
|
||||
evidence_dir, # type: ignore[no-untyped-def]
|
||||
run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
_harness_helpers_implemented: bool,
|
||||
) -> None:
|
||||
if not _harness_helpers_implemented:
|
||||
pytest.skip(
|
||||
"FT-N-01 full replay requires runner.helpers.{frame_source_replay,"
|
||||
"fdr_reader,imu_replay} — currently AZ-441 / AZ-407 leftovers. "
|
||||
"AC-1/AC-2/AC-3 helper logic covered by "
|
||||
"e2e/_unit_tests/helpers/test_outlier_tolerance_evaluator.py."
|
||||
)
|
||||
|
||||
from runner.helpers import fdr_reader
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
# 1. AC-1 — load injection plan (outlier event frames + offsets).
|
||||
manifest_path = outlier_injection_derkachi.out_root / "manifest.csv"
|
||||
events = ote.load_outlier_manifest(manifest_path)
|
||||
assert len(events) >= ote.MIN_OUTLIER_COUNT, (
|
||||
f"AC-1: medium-density injection must produce ≥{ote.MIN_OUTLIER_COUNT} "
|
||||
f"outliers (got {len(events)} from {manifest_path})"
|
||||
)
|
||||
|
||||
# 2. Drive replay against the injected frames directory.
|
||||
FrameSourceReplayer(_resolve_frame_sink()).replay_video(
|
||||
outlier_injection_derkachi.out_root / "frames"
|
||||
)
|
||||
|
||||
# 3. Collect outbound estimates + GT from FDR + tile cache.
|
||||
fdr_root = Path(evidence_dir).parent / f"run-{run_id}" / "fdr"
|
||||
estimates: list[ote.OutboundEstimate] = []
|
||||
for rec in fdr_reader.iter_records(fdr_root):
|
||||
if rec.record_type != "outbound_estimate":
|
||||
continue
|
||||
payload = rec.payload
|
||||
estimates.append(
|
||||
ote.OutboundEstimate(
|
||||
frame_idx=int(payload["frame_idx"]), # type: ignore[arg-type]
|
||||
monotonic_ms=int(rec.monotonic_ms),
|
||||
lat_deg=float(payload["lat_deg"]), # type: ignore[arg-type]
|
||||
lon_deg=float(payload["lon_deg"]), # type: ignore[arg-type]
|
||||
cov_semi_major_m=float(payload["cov_semi_major_m"]), # type: ignore[arg-type]
|
||||
source_label=str(payload["source_label"]), # type: ignore[arg-type]
|
||||
)
|
||||
)
|
||||
gt: list[ote.GtPose] = _resolve_gt_per_frame(outlier_injection_derkachi)
|
||||
|
||||
if not estimates:
|
||||
pytest.fail("FT-N-01: no outbound_estimate records produced")
|
||||
|
||||
# 4. Evaluate per outlier event.
|
||||
report = ote.evaluate(events, estimates, gt)
|
||||
out_csv = evidence_dir / f"ft-n-01-{fc_adapter}-{vio_strategy}.csv"
|
||||
ote.write_csv_evidence(out_csv, report)
|
||||
|
||||
# 5. NFR + AC assertions.
|
||||
nfr_recorder.record_metric(
|
||||
"ft_n_01.total_outliers", float(report.total_outliers), ac_id="AC-1"
|
||||
)
|
||||
nfr_recorder.record_metric(
|
||||
"ft_n_01.failed_event_count", float(report.failed_event_count), ac_id="AC-2"
|
||||
)
|
||||
for e in report.events:
|
||||
if e.drift_m is not None:
|
||||
nfr_recorder.record_metric(
|
||||
f"ft_n_01.event_{e.frame_idx}.drift_m", e.drift_m, ac_id="AC-2"
|
||||
)
|
||||
nfr_recorder.record_metric(
|
||||
f"ft_n_01.event_{e.frame_idx}.cov_non_decreasing",
|
||||
1.0 if e.cov_non_decreasing else 0.0,
|
||||
ac_id="AC-3",
|
||||
)
|
||||
|
||||
assert report.passes_count, (
|
||||
f"AC-1: ≥{ote.MIN_OUTLIER_COUNT} outliers required; "
|
||||
f"got {report.total_outliers}"
|
||||
)
|
||||
for e in report.events:
|
||||
assert e.passes_drift, (
|
||||
f"AC-2 (drift ≤ {ote.DRIFT_BUDGET_M} m) failed at frame "
|
||||
f"{e.frame_idx}: drift_m={e.drift_m}, "
|
||||
f"error_before={e.error_before_m}, error_after={e.error_after_m}"
|
||||
)
|
||||
assert e.passes_covariance, (
|
||||
f"AC-3 (cov_semi_major_m non-decreasing across window) failed at "
|
||||
f"frame {e.frame_idx}: "
|
||||
f"cov_before={e.cov_before}, cov_outlier={e.cov_outlier}, "
|
||||
f"cov_after={e.cov_after}"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_frame_sink(): # type: ignore[no-untyped-def]
|
||||
raise NotImplementedError(
|
||||
"frame sink resolution is owned by AZ-441 / runner.helpers.frame_source_replay"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_gt_per_frame(report: OutlierInjectionReport) -> list[ote.GtPose]:
|
||||
raise NotImplementedError(
|
||||
"Per-frame GT resolution is owned by AZ-407 / runner.helpers.tile_cache_gt"
|
||||
)
|
||||
Reference in New Issue
Block a user