mirror of
https://github.com/azaion/gps-denied-onboard.git
synced 2026-06-22 15:21:14 +00:00
[AZ-424] [AZ-425] [AZ-426] Implement negatives set (FT-N-01/03/04)
Adds three pure-logic evaluators + scenarios + unit tests covering the project's failure-mode robustness ladder (AC-3.1, AC-3.4, AC-3.5, AC-NEW-8): * outlier_tolerance_evaluator (AZ-424 / FT-N-01): per-event 50 m drift bound + 3-frame covariance-monotonic window over the AZ-408 outlier injector's medium-density manifest. * outage_request_evaluator (AZ-425 / FT-N-03): detects 3+ consecutive missing-frame windows; validates OPERATOR_RELOC_REQUEST STATUSTEXT arrives at 2 s ±500 ms, dead_reckoned label during outage, and no FC EKF divergence. * blackout_spoof_evaluator (AZ-426 / FT-N-04): eight-AC ladder across the 5 s / 15 s / 35 s sub-windows — switch latency, spoof rejection, monotonic covariance, honest horiz_accuracy, STATUSTEXT 1-2 Hz, 35 s escalation thresholds, and recovery gate. Each scenario is skip-gated on the AZ-441 / AZ-407 / AZ-416 replay / SITL / mavproxy helpers; unit tests (14 + 18 + 29 = 61) cover the AC logic today. Full e2e unit-test suite: 527 passed (+67). Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Oleksandr Bezdieniezhnykh
@@ -0,0 +1,170 @@
|
||||
"""FT-N-01 — 350 m outlier injection tolerance (AZ-424 / AC-3.1).
|
||||
|
||||
Replays the Derkachi flight with the AZ-408 ``outlier`` injector at
|
||||
``--density medium`` and verifies AC-1 / AC-2 / AC-3 via
|
||||
``runner.helpers.outlier_tolerance_evaluator``.
|
||||
|
||||
Gated on the same upstream replay helpers as FT-N-02 / FT-P-07
|
||||
(``frame_source_replay``, ``fdr_reader``, ``imu_replay``). When those
|
||||
helpers are still stubbed (current state under AZ-441 / AZ-407
|
||||
leftovers), the scenario test skips while
|
||||
``e2e/_unit_tests/helpers/test_outlier_tolerance_evaluator.py`` covers
|
||||
the pure-logic AC-2 / AC-3 invariants.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from fixtures.injectors.outlier import OutlierInjectionReport
|
||||
from runner.helpers import outlier_tolerance_evaluator as ote
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def _harness_helpers_implemented() -> bool:
|
||||
from runner.helpers import fdr_reader, imu_replay
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
try:
|
||||
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
|
||||
try:
|
||||
replayer.replay_video(Path("/tmp/non-existent.mp4"))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
imu_replay.ImuReplayer(emitter=_NullImuEmitter()).replay(Path("/tmp/non-existent.csv")) # type: ignore[arg-type]
|
||||
except NotImplementedError:
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class _NullSink:
|
||||
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class _NullImuEmitter:
|
||||
def emit(self, sample: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"outlier_injection_derkachi",
|
||||
[{"density": "medium", "seed": 0}],
|
||||
indirect=True,
|
||||
)
|
||||
@pytest.mark.traces_to("AC-3.1,AC-1,AC-2,AC-3,AC-4")
|
||||
def test_ft_n_01_outlier_tolerance(
|
||||
fc_adapter: str,
|
||||
vio_strategy: str,
|
||||
outlier_injection_derkachi: OutlierInjectionReport,
|
||||
evidence_dir, # type: ignore[no-untyped-def]
|
||||
run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
_harness_helpers_implemented: bool,
|
||||
) -> None:
|
||||
if not _harness_helpers_implemented:
|
||||
pytest.skip(
|
||||
"FT-N-01 full replay requires runner.helpers.{frame_source_replay,"
|
||||
"fdr_reader,imu_replay} — currently AZ-441 / AZ-407 leftovers. "
|
||||
"AC-1/AC-2/AC-3 helper logic covered by "
|
||||
"e2e/_unit_tests/helpers/test_outlier_tolerance_evaluator.py."
|
||||
)
|
||||
|
||||
from runner.helpers import fdr_reader
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
# 1. AC-1 — load injection plan (outlier event frames + offsets).
|
||||
manifest_path = outlier_injection_derkachi.out_root / "manifest.csv"
|
||||
events = ote.load_outlier_manifest(manifest_path)
|
||||
assert len(events) >= ote.MIN_OUTLIER_COUNT, (
|
||||
f"AC-1: medium-density injection must produce ≥{ote.MIN_OUTLIER_COUNT} "
|
||||
f"outliers (got {len(events)} from {manifest_path})"
|
||||
)
|
||||
|
||||
# 2. Drive replay against the injected frames directory.
|
||||
FrameSourceReplayer(_resolve_frame_sink()).replay_video(
|
||||
outlier_injection_derkachi.out_root / "frames"
|
||||
)
|
||||
|
||||
# 3. Collect outbound estimates + GT from FDR + tile cache.
|
||||
fdr_root = Path(evidence_dir).parent / f"run-{run_id}" / "fdr"
|
||||
estimates: list[ote.OutboundEstimate] = []
|
||||
for rec in fdr_reader.iter_records(fdr_root):
|
||||
if rec.record_type != "outbound_estimate":
|
||||
continue
|
||||
payload = rec.payload
|
||||
estimates.append(
|
||||
ote.OutboundEstimate(
|
||||
frame_idx=int(payload["frame_idx"]), # type: ignore[arg-type]
|
||||
monotonic_ms=int(rec.monotonic_ms),
|
||||
lat_deg=float(payload["lat_deg"]), # type: ignore[arg-type]
|
||||
lon_deg=float(payload["lon_deg"]), # type: ignore[arg-type]
|
||||
cov_semi_major_m=float(payload["cov_semi_major_m"]), # type: ignore[arg-type]
|
||||
source_label=str(payload["source_label"]), # type: ignore[arg-type]
|
||||
)
|
||||
)
|
||||
gt: list[ote.GtPose] = _resolve_gt_per_frame(outlier_injection_derkachi)
|
||||
|
||||
if not estimates:
|
||||
pytest.fail("FT-N-01: no outbound_estimate records produced")
|
||||
|
||||
# 4. Evaluate per outlier event.
|
||||
report = ote.evaluate(events, estimates, gt)
|
||||
out_csv = evidence_dir / f"ft-n-01-{fc_adapter}-{vio_strategy}.csv"
|
||||
ote.write_csv_evidence(out_csv, report)
|
||||
|
||||
# 5. NFR + AC assertions.
|
||||
nfr_recorder.record_metric(
|
||||
"ft_n_01.total_outliers", float(report.total_outliers), ac_id="AC-1"
|
||||
)
|
||||
nfr_recorder.record_metric(
|
||||
"ft_n_01.failed_event_count", float(report.failed_event_count), ac_id="AC-2"
|
||||
)
|
||||
for e in report.events:
|
||||
if e.drift_m is not None:
|
||||
nfr_recorder.record_metric(
|
||||
f"ft_n_01.event_{e.frame_idx}.drift_m", e.drift_m, ac_id="AC-2"
|
||||
)
|
||||
nfr_recorder.record_metric(
|
||||
f"ft_n_01.event_{e.frame_idx}.cov_non_decreasing",
|
||||
1.0 if e.cov_non_decreasing else 0.0,
|
||||
ac_id="AC-3",
|
||||
)
|
||||
|
||||
assert report.passes_count, (
|
||||
f"AC-1: ≥{ote.MIN_OUTLIER_COUNT} outliers required; "
|
||||
f"got {report.total_outliers}"
|
||||
)
|
||||
for e in report.events:
|
||||
assert e.passes_drift, (
|
||||
f"AC-2 (drift ≤ {ote.DRIFT_BUDGET_M} m) failed at frame "
|
||||
f"{e.frame_idx}: drift_m={e.drift_m}, "
|
||||
f"error_before={e.error_before_m}, error_after={e.error_after_m}"
|
||||
)
|
||||
assert e.passes_covariance, (
|
||||
f"AC-3 (cov_semi_major_m non-decreasing across window) failed at "
|
||||
f"frame {e.frame_idx}: "
|
||||
f"cov_before={e.cov_before}, cov_outlier={e.cov_outlier}, "
|
||||
f"cov_after={e.cov_after}"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_frame_sink(): # type: ignore[no-untyped-def]
|
||||
raise NotImplementedError(
|
||||
"frame sink resolution is owned by AZ-441 / runner.helpers.frame_source_replay"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_gt_per_frame(report: OutlierInjectionReport) -> list[ote.GtPose]:
|
||||
raise NotImplementedError(
|
||||
"Per-frame GT resolution is owned by AZ-407 / runner.helpers.tile_cache_gt"
|
||||
)
|
||||
@@ -0,0 +1,201 @@
|
||||
"""FT-N-03 — Extended outage triggers operator re-loc request (AZ-425 / AC-3.4).
|
||||
|
||||
Replays the Derkachi flight with a 3-consecutive-frame failure injector
|
||||
(a thin extension of the AZ-408 outlier injector that emits all-zero
|
||||
frames instead of crops) and verifies AC-1..AC-4 via
|
||||
``runner.helpers.outage_request_evaluator``.
|
||||
|
||||
Gated on the same upstream replay helpers as FT-N-01 / FT-N-02 / FT-P-07
|
||||
(``frame_source_replay``, ``fdr_reader``, ``imu_replay``, mavproxy
|
||||
``.tlog`` capture, SITL state read). When those helpers are still
|
||||
stubbed, the scenario test skips while
|
||||
``e2e/_unit_tests/helpers/test_outage_request_evaluator.py`` covers the
|
||||
AC-1..AC-4 evaluator logic.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from runner.helpers import outage_request_evaluator as ore
|
||||
|
||||
DERKACHI_DIR = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "_docs"
|
||||
/ "00_problem"
|
||||
/ "input_data"
|
||||
/ "flight_derkachi"
|
||||
)
|
||||
DERKACHI_MP4 = DERKACHI_DIR / "flight_derkachi.mp4"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def _harness_helpers_implemented() -> bool:
|
||||
from runner.helpers import fdr_reader, mavproxy_tlog_reader, sitl_observer
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
try:
|
||||
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
|
||||
try:
|
||||
replayer.replay_video(Path("/tmp/non-existent.mp4"))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
list(mavproxy_tlog_reader.iter_messages(Path("/tmp/non-existent.tlog")))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
sitl_observer.read_ekf_divergence_events() # type: ignore[attr-defined]
|
||||
except (AttributeError, NotImplementedError):
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class _NullSink:
|
||||
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
@pytest.mark.traces_to("AC-3.4,AC-1,AC-2,AC-3,AC-4,AC-5")
|
||||
def test_ft_n_03_outage_reloc(
|
||||
fc_adapter: str,
|
||||
vio_strategy: str,
|
||||
evidence_dir, # type: ignore[no-untyped-def]
|
||||
run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
_harness_helpers_implemented: bool,
|
||||
) -> None:
|
||||
if not _harness_helpers_implemented:
|
||||
pytest.skip(
|
||||
"FT-N-03 full replay requires runner.helpers.{frame_source_replay,"
|
||||
"fdr_reader,mavproxy_tlog_reader,sitl_observer} — currently "
|
||||
"AZ-441 / AZ-407 / AZ-416 leftovers. AC-1..AC-4 evaluator logic "
|
||||
"covered by e2e/_unit_tests/helpers/test_outage_request_evaluator.py."
|
||||
)
|
||||
|
||||
from runner.helpers import fdr_reader, mavproxy_tlog_reader, sitl_observer
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
# 1. Build / locate the 3-frame outage injection fixture.
|
||||
injected_frames_dir = _resolve_outage_injection_frames()
|
||||
|
||||
# 2. Drive replay.
|
||||
FrameSourceReplayer(_resolve_frame_sink()).replay_video(injected_frames_dir)
|
||||
|
||||
# 3. Collect outbound estimates + STATUSTEXT + EKF events.
|
||||
fdr_root = Path(evidence_dir).parent / f"run-{run_id}" / "fdr"
|
||||
estimates: list[ore.OutboundEstimateSample] = []
|
||||
expected_frame_indices: list[int] = []
|
||||
for rec in fdr_reader.iter_records(fdr_root):
|
||||
if rec.record_type == "frame_received":
|
||||
expected_frame_indices.append(int(rec.payload["frame_idx"])) # type: ignore[arg-type]
|
||||
elif rec.record_type == "outbound_estimate":
|
||||
payload = rec.payload
|
||||
estimates.append(
|
||||
ore.OutboundEstimateSample(
|
||||
frame_idx=int(payload["frame_idx"]), # type: ignore[arg-type]
|
||||
monotonic_ms=int(rec.monotonic_ms),
|
||||
source_label=str(payload["source_label"]), # type: ignore[arg-type]
|
||||
)
|
||||
)
|
||||
|
||||
tlog_path = Path(evidence_dir).parent / f"run-{run_id}" / "mavproxy.tlog"
|
||||
statustexts = [
|
||||
ore.StatustextSample(
|
||||
monotonic_ms=int(m.timestamp_us // 1000),
|
||||
text=str(m.fields.get("text", "")),
|
||||
)
|
||||
for m in mavproxy_tlog_reader.iter_messages(tlog_path)
|
||||
if m.msg_type == "STATUSTEXT"
|
||||
]
|
||||
ekf_events = [
|
||||
ore.EkfDivergenceEvent(
|
||||
monotonic_ms=int(ev.monotonic_ms), reason=str(ev.reason)
|
||||
)
|
||||
for ev in sitl_observer.read_ekf_divergence_events() # type: ignore[attr-defined]
|
||||
]
|
||||
|
||||
# 4. Evaluate.
|
||||
reports = ore.evaluate(
|
||||
expected_frame_indices,
|
||||
estimates,
|
||||
statustexts,
|
||||
ekf_events,
|
||||
frame_period_ms=_resolve_frame_period_ms(),
|
||||
)
|
||||
out_csv = evidence_dir / f"ft-n-03-{fc_adapter}-{vio_strategy}.csv"
|
||||
ore.write_csv_evidence(out_csv, reports)
|
||||
|
||||
# 5. NFR metrics + AC assertions.
|
||||
assert reports, "FT-N-03: at least one outage window must be detected (AC-1)"
|
||||
for r in reports:
|
||||
nfr_recorder.record_metric(
|
||||
f"ft_n_03.window_{r.window.first_missing_frame_idx}.length_frames",
|
||||
float(r.window.length_frames),
|
||||
ac_id="AC-1",
|
||||
)
|
||||
if r.statustext_offset_ms is not None:
|
||||
nfr_recorder.record_metric(
|
||||
f"ft_n_03.window_{r.window.first_missing_frame_idx}.statustext_offset_ms",
|
||||
float(r.statustext_offset_ms),
|
||||
ac_id="AC-2",
|
||||
)
|
||||
nfr_recorder.record_metric(
|
||||
f"ft_n_03.window_{r.window.first_missing_frame_idx}.dead_reckoned_count",
|
||||
float(r.dead_reckoned_count),
|
||||
ac_id="AC-3",
|
||||
)
|
||||
nfr_recorder.record_metric(
|
||||
f"ft_n_03.window_{r.window.first_missing_frame_idx}.ekf_divergence_count",
|
||||
float(r.ekf_divergence_count),
|
||||
ac_id="AC-4",
|
||||
)
|
||||
|
||||
for r in reports:
|
||||
assert r.passes_min_length, (
|
||||
f"AC-1: outage window {r.window.first_missing_frame_idx}-"
|
||||
f"{r.window.last_missing_frame_idx} is shorter than "
|
||||
f"{ore.MIN_OUTAGE_FRAMES} frames"
|
||||
)
|
||||
assert r.passes_statustext, (
|
||||
f"AC-2: '{ore.STATUSTEXT_REGEX}' STATUSTEXT not within "
|
||||
f"{int(ore.OUTAGE_THRESHOLD_S * 1000)} ±{int(ore.TOLERANCE_S * 1000)} ms "
|
||||
f"of outage onset at frame {r.window.first_missing_frame_idx} "
|
||||
f"(observed offset={r.statustext_offset_ms} ms)"
|
||||
)
|
||||
assert r.passes_dead_reckoned, (
|
||||
f"AC-3: no `dead_reckoned` estimate emitted during outage "
|
||||
f"window starting at frame {r.window.first_missing_frame_idx}"
|
||||
)
|
||||
assert r.passes_ekf, (
|
||||
f"AC-4: EKF divergence event(s) observed during outage "
|
||||
f"window starting at frame {r.window.first_missing_frame_idx} "
|
||||
f"(count={r.ekf_divergence_count})"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_outage_injection_frames() -> Path:
|
||||
raise NotImplementedError(
|
||||
"3-frame outage injector is owned by AZ-408 extension / "
|
||||
"fixtures/injectors/outlier.py (--all-zero variant)"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_frame_sink(): # type: ignore[no-untyped-def]
|
||||
raise NotImplementedError(
|
||||
"frame sink resolution is owned by AZ-441 / runner.helpers.frame_source_replay"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_frame_period_ms() -> int:
|
||||
raise NotImplementedError(
|
||||
"Frame period resolution is owned by AZ-441 / runner.helpers.frame_source_replay"
|
||||
)
|
||||
@@ -0,0 +1,267 @@
|
||||
"""FT-N-04 — Visual blackout + spoofed GPS combined failsafe (AZ-426 / AC-3.5, AC-NEW-8).
|
||||
|
||||
Three sub-cases (5 s / 15 s / 35 s) at the ladder of windows
|
||||
prescribed by AC-3.5 + AC-NEW-8, replayed via the AZ-408
|
||||
``blackout_spoof`` injector + the FC-inbound spoof proxy, and
|
||||
validated by ``runner.helpers.blackout_spoof_evaluator``.
|
||||
|
||||
Gated on the same upstream replay helpers as the other negative
|
||||
scenarios (``frame_source_replay``, ``fdr_reader``,
|
||||
``mavproxy_tlog_reader``, ``sitl_observer``, ``fc_proxy`` runtime
|
||||
binding). When those helpers are still stubbed the scenario test
|
||||
skips while
|
||||
``e2e/_unit_tests/helpers/test_blackout_spoof_evaluator.py`` covers
|
||||
the AC-1..AC-8 evaluator logic.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from fixtures.injectors.blackout_spoof import BlackoutSpoofReport
|
||||
from runner.helpers import blackout_spoof_evaluator as bse
|
||||
|
||||
_WINDOW_LADDER_S = (5.0, 15.0, 35.0)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def _harness_helpers_implemented() -> bool:
|
||||
from runner.helpers import fdr_reader, mavproxy_tlog_reader, sitl_observer
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
try:
|
||||
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
|
||||
try:
|
||||
replayer.replay_video(Path("/tmp/non-existent.mp4"))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
list(mavproxy_tlog_reader.iter_messages(Path("/tmp/non-existent.tlog")))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
sitl_observer.read_gps_health_samples() # type: ignore[attr-defined]
|
||||
sitl_observer.read_consistency_check_events() # type: ignore[attr-defined]
|
||||
except (AttributeError, NotImplementedError):
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class _NullSink:
|
||||
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"blackout_spoof_derkachi",
|
||||
[{"window_seconds": s, "seed": 0} for s in _WINDOW_LADDER_S],
|
||||
indirect=True,
|
||||
ids=[f"{int(s)}s" for s in _WINDOW_LADDER_S],
|
||||
)
|
||||
@pytest.mark.traces_to(
|
||||
"AC-3.5,AC-NEW-8,AC-1,AC-2,AC-3,AC-4,AC-5,AC-6,AC-7,AC-8,AC-9"
|
||||
)
|
||||
def test_ft_n_04_blackout_spoof(
|
||||
fc_adapter: str,
|
||||
vio_strategy: str,
|
||||
blackout_spoof_derkachi: BlackoutSpoofReport,
|
||||
evidence_dir, # type: ignore[no-untyped-def]
|
||||
run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
_harness_helpers_implemented: bool,
|
||||
) -> None:
|
||||
if not _harness_helpers_implemented:
|
||||
pytest.skip(
|
||||
"FT-N-04 full replay requires runner.helpers.{frame_source_replay,"
|
||||
"fdr_reader,mavproxy_tlog_reader,sitl_observer,fc_proxy} — currently "
|
||||
"AZ-441 / AZ-407 / AZ-416 leftovers. AC-1..AC-8 evaluator logic "
|
||||
"covered by e2e/_unit_tests/helpers/test_blackout_spoof_evaluator.py."
|
||||
)
|
||||
|
||||
from runner.helpers import fdr_reader, mavproxy_tlog_reader, sitl_observer
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
schedule = blackout_spoof_derkachi.schedule
|
||||
window = bse.BlackoutWindow(
|
||||
onset_monotonic_ms=schedule.window_start_ms,
|
||||
end_monotonic_ms=schedule.window_end_ms,
|
||||
)
|
||||
is_35s = abs(window.duration_s - 35.0) < 0.5
|
||||
|
||||
# 1. Drive replay (frames + paired fc-proxy spoof injection).
|
||||
FrameSourceReplayer(_resolve_frame_sink()).replay_video(
|
||||
blackout_spoof_derkachi.out_root / "frames"
|
||||
)
|
||||
_drive_fc_proxy(blackout_spoof_derkachi.out_root / "schedule.json")
|
||||
|
||||
# 2. Collect FDR estimates + spoof-rejected events.
|
||||
fdr_root = Path(evidence_dir).parent / f"run-{run_id}" / "fdr"
|
||||
estimates: list[bse.OutboundEstimateSample] = []
|
||||
spoof_events: list[bse.SpoofRejectedEvent] = []
|
||||
for rec in fdr_reader.iter_records(fdr_root):
|
||||
if rec.record_type == "outbound_estimate":
|
||||
p = rec.payload
|
||||
estimates.append(
|
||||
bse.OutboundEstimateSample(
|
||||
monotonic_ms=int(rec.monotonic_ms),
|
||||
source_label=str(p["source_label"]), # type: ignore[arg-type]
|
||||
cov_semi_major_m=float(p["cov_semi_major_m"]), # type: ignore[arg-type]
|
||||
horiz_accuracy=float(p.get("horiz_accuracy", p["cov_semi_major_m"])), # type: ignore[arg-type]
|
||||
fix_type=int(p.get("fix_type", -1)), # type: ignore[arg-type]
|
||||
)
|
||||
)
|
||||
elif rec.record_type == "spoof_rejected":
|
||||
spoof_events.append(
|
||||
bse.SpoofRejectedEvent(
|
||||
monotonic_ms=int(rec.monotonic_ms),
|
||||
reason=str(rec.payload.get("reason", "")), # type: ignore[arg-type]
|
||||
)
|
||||
)
|
||||
|
||||
# 3. Collect STATUSTEXTs from mavproxy tlog.
|
||||
tlog_path = Path(evidence_dir).parent / f"run-{run_id}" / "mavproxy.tlog"
|
||||
statustexts = [
|
||||
bse.StatustextSample(
|
||||
monotonic_ms=int(m.timestamp_us // 1000),
|
||||
text=str(m.fields.get("text", "")),
|
||||
)
|
||||
for m in mavproxy_tlog_reader.iter_messages(tlog_path)
|
||||
if m.msg_type == "STATUSTEXT"
|
||||
]
|
||||
|
||||
# 4. Collect FC-side GPS health + consistency-check events (recovery gate).
|
||||
gps_health = [
|
||||
bse.GpsHealthSample(
|
||||
monotonic_ms=int(s.monotonic_ms),
|
||||
healthy=bool(s.healthy),
|
||||
spoofed=bool(s.spoofed),
|
||||
)
|
||||
for s in sitl_observer.read_gps_health_samples() # type: ignore[attr-defined]
|
||||
]
|
||||
consistency = [
|
||||
bse.ConsistencyCheckEvent(
|
||||
monotonic_ms=int(c.monotonic_ms), passed=bool(c.passed)
|
||||
)
|
||||
for c in sitl_observer.read_consistency_check_events() # type: ignore[attr-defined]
|
||||
]
|
||||
|
||||
# 5. Evaluate.
|
||||
report = bse.evaluate(
|
||||
window,
|
||||
estimates=estimates,
|
||||
statustexts=statustexts,
|
||||
spoof_events=spoof_events,
|
||||
gps_health=gps_health,
|
||||
consistency_checks=consistency,
|
||||
frame_period_ms=_resolve_frame_period_ms(),
|
||||
is_35s_window=is_35s,
|
||||
)
|
||||
out_csv = (
|
||||
evidence_dir
|
||||
/ f"ft-n-04-{int(window.duration_s)}s-{fc_adapter}-{vio_strategy}.csv"
|
||||
)
|
||||
bse.write_csv_evidence(out_csv, report)
|
||||
|
||||
# 6. NFR metrics + AC assertions.
|
||||
nfr_recorder.record_metric(
|
||||
f"ft_n_04.{int(window.duration_s)}s.switch_latency_ms",
|
||||
float(report.switch_latency.first_dead_reckoned_offset_ms or 0),
|
||||
ac_id="AC-1",
|
||||
)
|
||||
nfr_recorder.record_metric(
|
||||
f"ft_n_04.{int(window.duration_s)}s.spoof_rejected_count",
|
||||
float(report.spoof_rejection.spoof_rejected_count),
|
||||
ac_id="AC-2",
|
||||
)
|
||||
nfr_recorder.record_metric(
|
||||
f"ft_n_04.{int(window.duration_s)}s.honest_accuracy_violation_count",
|
||||
float(report.honest_accuracy.violation_count),
|
||||
ac_id="AC-4",
|
||||
)
|
||||
if report.statustext_rate.observed_hz is not None:
|
||||
nfr_recorder.record_metric(
|
||||
f"ft_n_04.{int(window.duration_s)}s.statustext_imu_only_hz",
|
||||
report.statustext_rate.observed_hz,
|
||||
ac_id="AC-5",
|
||||
)
|
||||
if is_35s:
|
||||
nfr_recorder.record_metric(
|
||||
"ft_n_04.35s.cov2d_at_ms",
|
||||
float(report.escalation.cov2d_crossed_at_ms or 0),
|
||||
ac_id="AC-6",
|
||||
)
|
||||
nfr_recorder.record_metric(
|
||||
"ft_n_04.35s.failsafe_trigger_at_ms",
|
||||
float(report.escalation.cov500_or_30s_crossed_at_ms or 0),
|
||||
ac_id="AC-7",
|
||||
)
|
||||
|
||||
assert report.switch_latency.passes, (
|
||||
f"AC-1: dead_reckoned label not within ≤{bse.SWITCH_LATENCY_MS} ms / "
|
||||
f"1 frame of blackout onset; "
|
||||
f"offset={report.switch_latency.first_dead_reckoned_offset_ms} ms, "
|
||||
f"frame_period={report.switch_latency.frame_period_ms} ms"
|
||||
)
|
||||
assert report.spoof_rejection.passes, (
|
||||
f"AC-2: spoof rejection failed; "
|
||||
f"rejected_count={report.spoof_rejection.spoof_rejected_count}, "
|
||||
f"re_anchored_count={report.spoof_rejection.satellite_anchored_inside_window}"
|
||||
)
|
||||
assert report.covariance_monotonic.passes, (
|
||||
f"AC-3: cov_semi_major_m decreased at "
|
||||
f"{report.covariance_monotonic.first_decreasing_at_ms} ms"
|
||||
)
|
||||
assert report.honest_accuracy.passes, (
|
||||
f"AC-4: horiz_accuracy under-reporting "
|
||||
f"({report.honest_accuracy.violation_count} violations of "
|
||||
f"{report.honest_accuracy.sample_count} samples)"
|
||||
)
|
||||
assert report.statustext_rate.passes, (
|
||||
f"AC-5: VISUAL_BLACKOUT_IMU_ONLY rate "
|
||||
f"{report.statustext_rate.observed_hz} Hz outside "
|
||||
f"[{bse.STATUSTEXT_RATE_MIN_HZ}, {bse.STATUSTEXT_RATE_MAX_HZ}] Hz"
|
||||
)
|
||||
if is_35s:
|
||||
assert report.escalation.passes_ac6, (
|
||||
f"AC-6: fix_type not degraded after cov crossed "
|
||||
f"{bse.ESCALATION_COV_2D_M} m at "
|
||||
f"{report.escalation.cov2d_crossed_at_ms} ms"
|
||||
)
|
||||
assert report.escalation.passes_ac7, (
|
||||
f"AC-7: failsafe escalation incomplete; "
|
||||
f"horiz_999={report.escalation.horiz_accuracy_999}, "
|
||||
f"failsafe_statustext_offset_ms="
|
||||
f"{report.escalation.failsafe_statustext_offset_ms}"
|
||||
)
|
||||
assert report.recovery_gate.passes, (
|
||||
f"AC-8: recovery gate failed; "
|
||||
f"recovery_at_ms={report.recovery_gate.recovery_at_ms}, "
|
||||
f"stable_period_s={report.recovery_gate.stable_period_s}, "
|
||||
f"consistency_check_passed={report.recovery_gate.consistency_check_passed}"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_frame_sink(): # type: ignore[no-untyped-def]
|
||||
raise NotImplementedError(
|
||||
"frame sink resolution is owned by AZ-441 / runner.helpers.frame_source_replay"
|
||||
)
|
||||
|
||||
|
||||
def _drive_fc_proxy(schedule_path: Path) -> None:
|
||||
raise NotImplementedError(
|
||||
"FC-inbound spoof proxy driver is owned by AZ-441 / runner.helpers.fc_proxy_runtime"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_frame_period_ms() -> int:
|
||||
raise NotImplementedError(
|
||||
"Frame period resolution is owned by AZ-441 / runner.helpers.frame_source_replay"
|
||||
)
|
||||
Reference in New Issue
Block a user