azaion/gps-denied-onboard
1
0
Fork 0
mirror of https://github.com/azaion/gps-denied-onboard.git synced 2026-06-22 18:21:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

[AZ-424] [AZ-425] [AZ-426] Implement negatives set (FT-N-01/03/04)

Browse Source 
Adds three pure-logic evaluators + scenarios + unit tests covering the
project's failure-mode robustness ladder (AC-3.1, AC-3.4, AC-3.5,
AC-NEW-8):

* outlier_tolerance_evaluator (AZ-424 / FT-N-01): per-event 50 m drift
  bound + 3-frame covariance-monotonic window over the AZ-408 outlier
  injector's medium-density manifest.
* outage_request_evaluator (AZ-425 / FT-N-03): detects 3+ consecutive
  missing-frame windows; validates OPERATOR_RELOC_REQUEST STATUSTEXT
  arrives at 2 s ±500 ms, dead_reckoned label during outage, and no
  FC EKF divergence.
* blackout_spoof_evaluator (AZ-426 / FT-N-04): eight-AC ladder across
  the 5 s / 15 s / 35 s sub-windows — switch latency, spoof rejection,
  monotonic covariance, honest horiz_accuracy, STATUSTEXT 1-2 Hz,
  35 s escalation thresholds, and recovery gate.

Each scenario is skip-gated on the AZ-441 / AZ-407 / AZ-416 replay /
SITL / mavproxy helpers; unit tests (14 + 18 + 29 = 61) cover the
AC logic today. Full e2e unit-test suite: 527 passed (+67).

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Oleksandr Bezdieniezhnykh
2026-05-17 08:26:16 +03:00
parent a644debdb7
commit 2d6d44af5d
16 changed files with 3343 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
e2e/runner/helpers/blackout_spoof_evaluator.py
+557
View File
@@ -0,0 +1,557 @@
"""Blackout-spoof evaluation for FT-N-04 (AZ-426 / AC-3.5 + AC-NEW-8).
Three-window ladder (5 s / 15 s / 35 s) with the
``blackout_spoof.py`` injector + FC-inbound spoof proxy. The
evaluator validates per AZ-426:
* AC-1: switch latency — within ≤1 frame OR ≤``SWITCH_LATENCY_MS``
(whichever is shorter), the first outbound estimate after blackout
onset carries ``source_label = dead_reckoned``.
* AC-2: spoof rejection — at least one FDR ``spoof-rejected`` event
is observed during the blackout window AND zero spoofed GPS records
are consumed into the estimator (label never returns to
``satellite_anchored`` during the window).
* AC-3: monotonic covariance — ``cov_semi_major_m`` is non-decreasing
across consecutive emissions inside the window.
* AC-4: honest horiz_accuracy —
``horiz_accuracy ≥ HONEST_ACCURACY_RATIO × cov_semi_major_m``
for every emission.
* AC-5: STATUSTEXT 1-2 Hz —
``VISUAL_BLACKOUT_IMU_ONLY`` STATUSTEXT rate is in
``[STATUSTEXT_RATE_MIN_HZ, STATUSTEXT_RATE_MAX_HZ]`` throughout the
window.
* AC-6 (35 s only): when 95 % covariance crosses
``ESCALATION_COV_2D_M``, fix_type degrades to ≤``ESCALATION_FIX_TYPE_2D``.
* AC-7 (35 s only): when 95 % covariance crosses
``ESCALATION_COV_FAILSAFE_M`` OR window duration exceeds
``ESCALATION_DURATION_FAILSAFE_S``, ``horiz_accuracy ==
HORIZ_ACCURACY_FAILSAFE`` AND ``VISUAL_BLACKOUT_FAILSAFE``
STATUSTEXT is emitted within ≤``ESCALATION_LATENCY_MS`` of the
crossing.
* AC-8: recovery gate — after blackout end, label only returns to
``satellite_anchored`` once both (a) FC GPS-health is stable +
non-spoofed for ≥``RECOVERY_STABLE_S`` AND (b) a
visual/satellite consistency check succeeds.
Public-boundary discipline: does NOT import any
``src/gps_denied_onboard`` symbol.
"""
from __future__ import annotations
import csv
from dataclasses import dataclass
from pathlib import Path
from typing import Iterable, Sequence
# AC-1
SWITCH_LATENCY_MS = 400
# AC-2
DEAD_RECKONED_LABEL = "dead_reckoned"
SATELLITE_ANCHORED_LABEL = "satellite_anchored"
# AC-4
HONEST_ACCURACY_RATIO = 0.95
# AC-5
STATUSTEXT_IMU_ONLY = "VISUAL_BLACKOUT_IMU_ONLY"
STATUSTEXT_RATE_MIN_HZ = 1.0
STATUSTEXT_RATE_MAX_HZ = 2.0
# AC-6 / AC-7
STATUSTEXT_FAILSAFE = "VISUAL_BLACKOUT_FAILSAFE"
ESCALATION_COV_2D_M = 100.0
ESCALATION_COV_FAILSAFE_M = 500.0
ESCALATION_DURATION_FAILSAFE_S = 30.0
ESCALATION_FIX_TYPE_2D = 2 # MAVLink GPS_FIX_TYPE_2D
HORIZ_ACCURACY_FAILSAFE = 999.0
ESCALATION_LATENCY_MS = 500
# AC-8
RECOVERY_STABLE_S = 10.0
@dataclass(frozen=True)
class BlackoutWindow:
"""The injector-emitted window the evaluator is bound to."""
onset_monotonic_ms: int
end_monotonic_ms: int
@property
def duration_s(self) -> float:
return (self.end_monotonic_ms - self.onset_monotonic_ms) / 1000.0
@dataclass(frozen=True)
class OutboundEstimateSample:
"""One outbound estimate with fields used by FT-N-04 ACs."""
monotonic_ms: int
source_label: str
cov_semi_major_m: float
horiz_accuracy: float # AP GPS_INPUT.horiz_accuracy (m)
fix_type: int # MAVLink GPS fix type (0..6); -1 if unavailable
@dataclass(frozen=True)
class StatustextSample:
monotonic_ms: int
text: str
@dataclass(frozen=True)
class SpoofRejectedEvent:
"""One FDR `spoof-rejected` event."""
monotonic_ms: int
reason: str
@dataclass(frozen=True)
class GpsHealthSample:
"""FC-side GPS health sample (post-blackout, for recovery gate)."""
monotonic_ms: int
healthy: bool
spoofed: bool
@dataclass(frozen=True)
class ConsistencyCheckEvent:
"""Visual/satellite consistency check outcome (post-blackout)."""
monotonic_ms: int
passed: bool
@dataclass(frozen=True)
class SwitchLatencyReport:
"""AC-1 result."""
first_dead_reckoned_offset_ms: int | None # ms after window onset
frame_period_ms: int
passes: bool
@dataclass(frozen=True)
class SpoofRejectionReport:
"""AC-2 result."""
spoof_rejected_count: int
satellite_anchored_inside_window: int
passes: bool
@dataclass(frozen=True)
class CovarianceMonotonicReport:
"""AC-3 result."""
first_decreasing_at_ms: int | None
sample_count: int
passes: bool
@dataclass(frozen=True)
class HonestAccuracyReport:
"""AC-4 result."""
violation_count: int
sample_count: int
passes: bool
@dataclass(frozen=True)
class StatustextRateReport:
"""AC-5 result for VISUAL_BLACKOUT_IMU_ONLY."""
observed_hz: float | None
count: int
passes: bool
@dataclass(frozen=True)
class EscalationReport:
"""AC-6 + AC-7 result (35 s window only — other windows return passes=True)."""
cov2d_crossed: bool
cov2d_crossed_at_ms: int | None
fix_type_degraded: bool # AC-6 satisfied
cov500_or_30s_crossed: bool
cov500_or_30s_crossed_at_ms: int | None
horiz_accuracy_999: bool # AC-7 part 1
failsafe_statustext_offset_ms: int | None
failsafe_statustext_in_time: bool # AC-7 part 2
passes_ac6: bool
passes_ac7: bool
@property
def passes(self) -> bool:
return self.passes_ac6 and self.passes_ac7
@dataclass(frozen=True)
class RecoveryGateReport:
"""AC-8 result."""
recovery_at_ms: int | None
stable_period_s: float | None
consistency_check_passed: bool
passes: bool
@dataclass(frozen=True)
class BlackoutSpoofReport:
"""Aggregate FT-N-04 result for one window."""
window: BlackoutWindow
switch_latency: SwitchLatencyReport
spoof_rejection: SpoofRejectionReport
covariance_monotonic: CovarianceMonotonicReport
honest_accuracy: HonestAccuracyReport
statustext_rate: StatustextRateReport
escalation: EscalationReport
recovery_gate: RecoveryGateReport
@property
def passes(self) -> bool:
return all(
(
self.switch_latency.passes,
self.spoof_rejection.passes,
self.covariance_monotonic.passes,
self.honest_accuracy.passes,
self.statustext_rate.passes,
self.escalation.passes,
self.recovery_gate.passes,
)
)
def _inside_window(window: BlackoutWindow, t_ms: int) -> bool:
return window.onset_monotonic_ms <= t_ms <= window.end_monotonic_ms
def _samples_inside_window(
window: BlackoutWindow, samples: Iterable[OutboundEstimateSample]
) -> list[OutboundEstimateSample]:
return [s for s in samples if _inside_window(window, s.monotonic_ms)]
def evaluate_switch_latency(
window: BlackoutWindow,
estimates: Sequence[OutboundEstimateSample],
frame_period_ms: int,
) -> SwitchLatencyReport:
"""AC-1: dead_reckoned label within ≤1 frame OR ≤SWITCH_LATENCY_MS."""
budget_ms = min(SWITCH_LATENCY_MS, frame_period_ms)
offset: int | None = None
for s in estimates:
if s.monotonic_ms < window.onset_monotonic_ms:
continue
if s.source_label == DEAD_RECKONED_LABEL:
offset = s.monotonic_ms - window.onset_monotonic_ms
break
return SwitchLatencyReport(
first_dead_reckoned_offset_ms=offset,
frame_period_ms=frame_period_ms,
passes=offset is not None and offset <= budget_ms,
)
def evaluate_spoof_rejection(
window: BlackoutWindow,
estimates: Sequence[OutboundEstimateSample],
spoof_events: Sequence[SpoofRejectedEvent],
) -> SpoofRejectionReport:
"""AC-2: spoof-rejected events present AND no satellite_anchored re-entry."""
rejected = sum(
1 for ev in spoof_events if _inside_window(window, ev.monotonic_ms)
)
inside = _samples_inside_window(window, estimates)
re_anchored = sum(1 for s in inside if s.source_label == SATELLITE_ANCHORED_LABEL)
return SpoofRejectionReport(
spoof_rejected_count=rejected,
satellite_anchored_inside_window=re_anchored,
passes=rejected >= 1 and re_anchored == 0,
)
def evaluate_covariance_monotonic(
window: BlackoutWindow, estimates: Sequence[OutboundEstimateSample]
) -> CovarianceMonotonicReport:
"""AC-3: cov_semi_major_m non-decreasing across consecutive emissions."""
inside = _samples_inside_window(window, estimates)
first_dec: int | None = None
for i in range(1, len(inside)):
if inside[i].cov_semi_major_m < inside[i - 1].cov_semi_major_m:
first_dec = inside[i].monotonic_ms
break
return CovarianceMonotonicReport(
first_decreasing_at_ms=first_dec,
sample_count=len(inside),
passes=first_dec is None and len(inside) >= 1,
)
def evaluate_honest_accuracy(
window: BlackoutWindow, estimates: Sequence[OutboundEstimateSample]
) -> HonestAccuracyReport:
"""AC-4: horiz_accuracy ≥ HONEST_ACCURACY_RATIO × cov_semi_major_m."""
inside = _samples_inside_window(window, estimates)
violations = sum(
1
for s in inside
if s.horiz_accuracy < HONEST_ACCURACY_RATIO * s.cov_semi_major_m
)
return HonestAccuracyReport(
violation_count=violations,
sample_count=len(inside),
passes=violations == 0 and len(inside) >= 1,
)
def evaluate_statustext_rate(
window: BlackoutWindow, statustexts: Sequence[StatustextSample]
) -> StatustextRateReport:
"""AC-5: VISUAL_BLACKOUT_IMU_ONLY rate ∈ [1, 2] Hz."""
inside = [
st
for st in statustexts
if STATUSTEXT_IMU_ONLY in st.text and _inside_window(window, st.monotonic_ms)
]
duration_s = window.duration_s
if duration_s <= 0 or not inside:
return StatustextRateReport(observed_hz=None, count=len(inside), passes=False)
rate = len(inside) / duration_s
return StatustextRateReport(
observed_hz=rate,
count=len(inside),
passes=STATUSTEXT_RATE_MIN_HZ <= rate <= STATUSTEXT_RATE_MAX_HZ,
)
def _first_cov_crossing_ms(
window: BlackoutWindow,
estimates: Sequence[OutboundEstimateSample],
threshold_m: float,
) -> int | None:
for s in _samples_inside_window(window, estimates):
if s.cov_semi_major_m >= threshold_m:
return s.monotonic_ms
return None
def evaluate_escalation(
window: BlackoutWindow,
estimates: Sequence[OutboundEstimateSample],
statustexts: Sequence[StatustextSample],
*,
is_35s_window: bool,
) -> EscalationReport:
"""AC-6 + AC-7: applies only to the 35 s sub-case.
For non-35 s windows the report is vacuously passing — those windows
are not expected to cross either escalation threshold and any
incidental crossing is treated as informational only.
"""
cov2d_at = _first_cov_crossing_ms(window, estimates, ESCALATION_COV_2D_M)
cov500_at = _first_cov_crossing_ms(window, estimates, ESCALATION_COV_FAILSAFE_M)
duration_breach_at: int | None = None
if window.duration_s >= ESCALATION_DURATION_FAILSAFE_S:
duration_breach_at = (
window.onset_monotonic_ms
+ int(ESCALATION_DURATION_FAILSAFE_S * 1000)
)
failsafe_trigger_at: int | None = None
if cov500_at is not None and duration_breach_at is not None:
failsafe_trigger_at = min(cov500_at, duration_breach_at)
else:
failsafe_trigger_at = cov500_at if cov500_at is not None else duration_breach_at
if not is_35s_window:
return EscalationReport(
cov2d_crossed=cov2d_at is not None,
cov2d_crossed_at_ms=cov2d_at,
fix_type_degraded=True,
cov500_or_30s_crossed=failsafe_trigger_at is not None,
cov500_or_30s_crossed_at_ms=failsafe_trigger_at,
horiz_accuracy_999=True,
failsafe_statustext_offset_ms=None,
failsafe_statustext_in_time=True,
passes_ac6=True,
passes_ac7=True,
)
# AC-6: any sample at/after cov2d_at must have fix_type ≤ ESCALATION_FIX_TYPE_2D.
fix_degraded = True
if cov2d_at is not None:
post = [s for s in _samples_inside_window(window, estimates) if s.monotonic_ms >= cov2d_at]
if post and any(s.fix_type > ESCALATION_FIX_TYPE_2D for s in post):
fix_degraded = False
passes_ac6 = cov2d_at is None or fix_degraded
# AC-7: post-trigger samples must have horiz_accuracy == 999 AND
# VISUAL_BLACKOUT_FAILSAFE STATUSTEXT must arrive within ≤500 ms of trigger.
horiz_999 = True
failsafe_offset: int | None = None
failsafe_in_time = True
if failsafe_trigger_at is not None:
post = [s for s in _samples_inside_window(window, estimates) if s.monotonic_ms >= failsafe_trigger_at]
if post and any(s.horiz_accuracy != HORIZ_ACCURACY_FAILSAFE for s in post):
horiz_999 = False
for st in statustexts:
if STATUSTEXT_FAILSAFE not in st.text:
continue
if st.monotonic_ms < failsafe_trigger_at:
continue
offset = st.monotonic_ms - failsafe_trigger_at
if failsafe_offset is None or offset < failsafe_offset:
failsafe_offset = offset
failsafe_in_time = (
failsafe_offset is not None and failsafe_offset <= ESCALATION_LATENCY_MS
)
passes_ac7 = failsafe_trigger_at is None or (horiz_999 and failsafe_in_time)
return EscalationReport(
cov2d_crossed=cov2d_at is not None,
cov2d_crossed_at_ms=cov2d_at,
fix_type_degraded=fix_degraded,
cov500_or_30s_crossed=failsafe_trigger_at is not None,
cov500_or_30s_crossed_at_ms=failsafe_trigger_at,
horiz_accuracy_999=horiz_999,
failsafe_statustext_offset_ms=failsafe_offset,
failsafe_statustext_in_time=failsafe_in_time,
passes_ac6=passes_ac6,
passes_ac7=passes_ac7,
)
def evaluate_recovery_gate(
window: BlackoutWindow,
estimates: Sequence[OutboundEstimateSample],
gps_health: Sequence[GpsHealthSample],
consistency_checks: Sequence[ConsistencyCheckEvent],
) -> RecoveryGateReport:
"""AC-8: recovery only after ≥10 s healthy/non-spoofed FC GPS AND a consistency check pass."""
# First post-window satellite_anchored sample marks the (claimed) recovery moment.
recovery_at: int | None = None
for s in estimates:
if (
s.monotonic_ms > window.end_monotonic_ms
and s.source_label == SATELLITE_ANCHORED_LABEL
):
recovery_at = s.monotonic_ms
break
if recovery_at is None:
# No recovery attempted — vacuously passing for this gate; the
# caller can still flag it via window-level coverage.
return RecoveryGateReport(
recovery_at_ms=None,
stable_period_s=None,
consistency_check_passed=False,
passes=True,
)
# (a) Continuous healthy/non-spoofed FC GPS for ≥RECOVERY_STABLE_S BEFORE recovery_at.
cutoff_ms = recovery_at - int(RECOVERY_STABLE_S * 1000)
relevant = [
h for h in gps_health
if window.end_monotonic_ms <= h.monotonic_ms <= recovery_at
]
stable = all(h.healthy and not h.spoofed for h in relevant) and len(relevant) >= 1
earliest_relevant = relevant[0].monotonic_ms if relevant else recovery_at
stable_period_s = (recovery_at - earliest_relevant) / 1000.0
has_enough_window = earliest_relevant <= cutoff_ms
# (b) Consistency check pass occurred between window-end and recovery_at.
consistency_passed = any(
c.passed and window.end_monotonic_ms <= c.monotonic_ms <= recovery_at
for c in consistency_checks
)
return RecoveryGateReport(
recovery_at_ms=recovery_at,
stable_period_s=stable_period_s,
consistency_check_passed=consistency_passed,
passes=stable and has_enough_window and consistency_passed,
)
def evaluate(
window: BlackoutWindow,
*,
estimates: Sequence[OutboundEstimateSample],
statustexts: Sequence[StatustextSample],
spoof_events: Sequence[SpoofRejectedEvent],
gps_health: Sequence[GpsHealthSample],
consistency_checks: Sequence[ConsistencyCheckEvent],
frame_period_ms: int,
is_35s_window: bool,
) -> BlackoutSpoofReport:
"""Run every AC-1..AC-8 check for a single window."""
return BlackoutSpoofReport(
window=window,
switch_latency=evaluate_switch_latency(window, estimates, frame_period_ms),
spoof_rejection=evaluate_spoof_rejection(window, estimates, spoof_events),
covariance_monotonic=evaluate_covariance_monotonic(window, estimates),
honest_accuracy=evaluate_honest_accuracy(window, estimates),
statustext_rate=evaluate_statustext_rate(window, statustexts),
escalation=evaluate_escalation(
window, estimates, statustexts, is_35s_window=is_35s_window
),
recovery_gate=evaluate_recovery_gate(
window, estimates, gps_health, consistency_checks
),
)
def write_csv_evidence(out_path: Path, report: BlackoutSpoofReport) -> Path:
"""Write FT-N-04 aggregate evidence — one row of per-AC summary."""
out_path.parent.mkdir(parents=True, exist_ok=True)
with out_path.open("w", newline="") as fh:
writer = csv.writer(fh)
writer.writerow(
[
"window_duration_s",
"ac1_switch_latency_ms",
"ac1_passes",
"ac2_spoof_rejected_count",
"ac2_re_anchored_count",
"ac2_passes",
"ac3_first_decreasing_at_ms",
"ac3_passes",
"ac4_violation_count",
"ac4_passes",
"ac5_observed_hz",
"ac5_passes",
"ac6_cov2d_at_ms",
"ac6_passes",
"ac7_failsafe_trigger_at_ms",
"ac7_passes",
"ac8_recovery_at_ms",
"ac8_passes",
"passes",
]
)
r = report
writer.writerow(
[
f"{r.window.duration_s:.3f}",
"" if r.switch_latency.first_dead_reckoned_offset_ms is None else r.switch_latency.first_dead_reckoned_offset_ms,
"true" if r.switch_latency.passes else "false",
r.spoof_rejection.spoof_rejected_count,
r.spoof_rejection.satellite_anchored_inside_window,
"true" if r.spoof_rejection.passes else "false",
"" if r.covariance_monotonic.first_decreasing_at_ms is None else r.covariance_monotonic.first_decreasing_at_ms,
"true" if r.covariance_monotonic.passes else "false",
r.honest_accuracy.violation_count,
"true" if r.honest_accuracy.passes else "false",
"" if r.statustext_rate.observed_hz is None else f"{r.statustext_rate.observed_hz:.3f}",
"true" if r.statustext_rate.passes else "false",
"" if r.escalation.cov2d_crossed_at_ms is None else r.escalation.cov2d_crossed_at_ms,
"true" if r.escalation.passes_ac6 else "false",
"" if r.escalation.cov500_or_30s_crossed_at_ms is None else r.escalation.cov500_or_30s_crossed_at_ms,
"true" if r.escalation.passes_ac7 else "false",
"" if r.recovery_gate.recovery_at_ms is None else r.recovery_gate.recovery_at_ms,
"true" if r.recovery_gate.passes else "false",
"true" if r.passes else "false",
]
)
return out_path
e2e/runner/helpers/outage_request_evaluator.py
+293
View File
@@ -0,0 +1,293 @@
"""Outage-request evaluation for FT-N-03 (AZ-425 / AC-3.4).
Detects sustained no-estimate outage windows from an outbound-estimate
stream, then evaluates:
* AC-1: outage onset — ≥``MIN_OUTAGE_FRAMES`` consecutive missing frames.
* AC-2: STATUSTEXT containing ``OPERATOR_RELOC_REQUEST`` is emitted
within ``[OUTAGE_THRESHOLD_S − TOLERANCE_S, OUTAGE_THRESHOLD_S +
TOLERANCE_S]`` of outage onset.
* AC-3: during the outage window, the outbound stream emits at least
one estimate carrying ``source_label = dead_reckoned`` (IMU-extrapolated
propagation continues).
* AC-4: FC-side SITL state shows NO EKF divergence event during the
outage.
A "no-estimate frame" is a frame_idx in the expected sequence with no
matching outbound-estimate record. Frame indices are expected to be
monotonic; ``expected_frame_indices`` is supplied by the caller so the
evaluator does not have to know the replay's total frame count.
Public-boundary discipline: does NOT import any
``src/gps_denied_onboard`` symbol.
"""
from __future__ import annotations
import csv
from dataclasses import dataclass
from pathlib import Path
from typing import Iterable, Sequence
MIN_OUTAGE_FRAMES = 3 # AC-1
OUTAGE_THRESHOLD_S = 2.0 # AC-2
TOLERANCE_S = 0.5 # AC-2 ±500 ms window
STATUSTEXT_REGEX = "OPERATOR_RELOC_REQUEST" # AC-2 exact substring
DEAD_RECKONED_LABEL = "dead_reckoned" # AC-3
@dataclass(frozen=True)
class OutboundEstimateSample:
"""One outbound estimate keyed by frame index + monotonic time."""
frame_idx: int
monotonic_ms: int
source_label: str
@dataclass(frozen=True)
class StatustextSample:
"""One STATUSTEXT message captured from mavproxy tlog."""
monotonic_ms: int
text: str
@dataclass(frozen=True)
class EkfDivergenceEvent:
"""One EKF-divergence event observed via SITL state read."""
monotonic_ms: int
reason: str
@dataclass(frozen=True)
class OutageWindow:
"""One detected outage window — contiguous run of missing frames."""
first_missing_frame_idx: int
last_missing_frame_idx: int
onset_monotonic_ms: int
end_monotonic_ms: int
@property
def length_frames(self) -> int:
return self.last_missing_frame_idx - self.first_missing_frame_idx + 1
@property
def duration_ms(self) -> int:
return self.end_monotonic_ms - self.onset_monotonic_ms
@dataclass(frozen=True)
class OutageReport:
"""AC-1 / AC-2 / AC-3 / AC-4 evaluation for one outage window."""
window: OutageWindow
passes_min_length: bool # AC-1
statustext_offset_ms: int | None # AC-2: ms after onset, None if absent
passes_statustext: bool # AC-2
dead_reckoned_count: int # AC-3 supporting metric
passes_dead_reckoned: bool # AC-3
ekf_divergence_count: int # AC-4 supporting metric
passes_ekf: bool # AC-4
@property
def passes(self) -> bool:
return (
self.passes_min_length
and self.passes_statustext
and self.passes_dead_reckoned
and self.passes_ekf
)
def detect_outage_windows(
expected_frame_indices: Sequence[int],
estimates: Sequence[OutboundEstimateSample],
frame_period_ms: int,
replay_start_monotonic_ms: int = 0,
) -> list[OutageWindow]:
"""Detect contiguous outage windows.
A frame index in ``expected_frame_indices`` with no matching estimate
counts as missing. Runs of consecutive missing frames of length
≥``MIN_OUTAGE_FRAMES`` become outage windows.
``frame_period_ms`` is the nominal inter-frame interval; onset/end
timestamps are derived as
``replay_start_monotonic_ms + frame_idx * frame_period_ms``. The
timing fields are estimates — when actual capture timestamps are
available the caller should pass them via ``estimates`` and rely on
those for downstream timing checks.
"""
present = {e.frame_idx for e in estimates}
windows: list[OutageWindow] = []
run_start: int | None = None
prev_idx: int | None = None
for idx in expected_frame_indices:
if idx not in present:
if run_start is None:
run_start = idx
prev_idx = idx
else:
if run_start is not None and prev_idx is not None:
run_length = prev_idx - run_start + 1
if run_length >= MIN_OUTAGE_FRAMES:
windows.append(
OutageWindow(
first_missing_frame_idx=run_start,
last_missing_frame_idx=prev_idx,
onset_monotonic_ms=replay_start_monotonic_ms
+ run_start * frame_period_ms,
end_monotonic_ms=replay_start_monotonic_ms
+ (prev_idx + 1) * frame_period_ms,
)
)
run_start = None
prev_idx = None
# Trailing run.
if run_start is not None and prev_idx is not None:
run_length = prev_idx - run_start + 1
if run_length >= MIN_OUTAGE_FRAMES:
windows.append(
OutageWindow(
first_missing_frame_idx=run_start,
last_missing_frame_idx=prev_idx,
onset_monotonic_ms=replay_start_monotonic_ms
+ run_start * frame_period_ms,
end_monotonic_ms=replay_start_monotonic_ms
+ (prev_idx + 1) * frame_period_ms,
)
)
return windows
def _first_statustext_offset_ms(
window: OutageWindow,
statustexts: Iterable[StatustextSample],
) -> int | None:
"""Return ms-offset of first OPERATOR_RELOC_REQUEST after onset, or None."""
best: int | None = None
for st in statustexts:
if STATUSTEXT_REGEX not in st.text:
continue
if st.monotonic_ms < window.onset_monotonic_ms:
continue
offset = st.monotonic_ms - window.onset_monotonic_ms
if best is None or offset < best:
best = offset
return best
def _dead_reckoned_during_window(
window: OutageWindow,
estimates: Iterable[OutboundEstimateSample],
) -> int:
count = 0
for e in estimates:
if (
e.source_label == DEAD_RECKONED_LABEL
and window.onset_monotonic_ms <= e.monotonic_ms <= window.end_monotonic_ms
):
count += 1
return count
def _ekf_divergence_during_window(
window: OutageWindow,
events: Iterable[EkfDivergenceEvent],
) -> int:
count = 0
for ev in events:
if window.onset_monotonic_ms <= ev.monotonic_ms <= window.end_monotonic_ms:
count += 1
return count
def evaluate_window(
window: OutageWindow,
estimates: Sequence[OutboundEstimateSample],
statustexts: Sequence[StatustextSample],
ekf_events: Sequence[EkfDivergenceEvent],
) -> OutageReport:
"""Compute AC-1..AC-4 evaluation for a single outage window."""
offset = _first_statustext_offset_ms(window, statustexts)
threshold_ms = int(OUTAGE_THRESHOLD_S * 1000)
tolerance_ms = int(TOLERANCE_S * 1000)
passes_statustext = (
offset is not None
and (threshold_ms - tolerance_ms) <= offset <= (threshold_ms + tolerance_ms)
)
dr_count = _dead_reckoned_during_window(window, estimates)
ekf_count = _ekf_divergence_during_window(window, ekf_events)
return OutageReport(
window=window,
passes_min_length=window.length_frames >= MIN_OUTAGE_FRAMES,
statustext_offset_ms=offset,
passes_statustext=passes_statustext,
dead_reckoned_count=dr_count,
passes_dead_reckoned=dr_count >= 1,
ekf_divergence_count=ekf_count,
passes_ekf=ekf_count == 0,
)
def evaluate(
expected_frame_indices: Sequence[int],
estimates: Sequence[OutboundEstimateSample],
statustexts: Sequence[StatustextSample],
ekf_events: Sequence[EkfDivergenceEvent],
frame_period_ms: int,
replay_start_monotonic_ms: int = 0,
) -> list[OutageReport]:
"""Detect outage windows and evaluate each."""
windows = detect_outage_windows(
expected_frame_indices,
estimates,
frame_period_ms=frame_period_ms,
replay_start_monotonic_ms=replay_start_monotonic_ms,
)
return [evaluate_window(w, estimates, statustexts, ekf_events) for w in windows]
def write_csv_evidence(out_path: Path, reports: Sequence[OutageReport]) -> Path:
out_path.parent.mkdir(parents=True, exist_ok=True)
with out_path.open("w", newline="") as fh:
writer = csv.writer(fh)
writer.writerow(
[
"first_missing_frame",
"last_missing_frame",
"length_frames",
"onset_ms",
"duration_ms",
"statustext_offset_ms",
"dead_reckoned_count",
"ekf_divergence_count",
"passes_min_length",
"passes_statustext",
"passes_dead_reckoned",
"passes_ekf",
"passes",
]
)
for r in reports:
writer.writerow(
[
r.window.first_missing_frame_idx,
r.window.last_missing_frame_idx,
r.window.length_frames,
r.window.onset_monotonic_ms,
r.window.duration_ms,
"" if r.statustext_offset_ms is None else r.statustext_offset_ms,
r.dead_reckoned_count,
r.ekf_divergence_count,
"true" if r.passes_min_length else "false",
"true" if r.passes_statustext else "false",
"true" if r.passes_dead_reckoned else "false",
"true" if r.passes_ekf else "false",
"true" if r.passes else "false",
]
)
return out_path
e2e/runner/helpers/outlier_tolerance_evaluator.py
+261
View File
@@ -0,0 +1,261 @@
"""Outlier-tolerance evaluation for FT-N-01 (AZ-424 / AC-3.1).
Consumes the AZ-408 ``outlier`` injector's ``manifest.csv`` (which
frames were replaced + the geodesic offset) and the SUT's outbound
estimate stream, and validates:
* AC-1: at least ``MIN_OUTLIER_COUNT`` outlier frames were injected
over the replay.
* AC-2: for every outlier event,
``error_after_outlier ≤ error_before_outlier + DRIFT_BUDGET_M``.
* AC-3: ``cov_semi_major_m`` is non-decreasing across the 3-frame
window centred on the outlier (frame before, outlier, frame after).
The injector's ``geodesic_offset_m`` column verifies the
RESTRICT-CAM-1 / AC-3.1 threshold (>350 m) per-row — the AC-1 count
check here is a coarser invariant that does not duplicate the
per-row geodesic gate.
Public-boundary discipline: does NOT import any
``src/gps_denied_onboard`` symbol.
"""
from __future__ import annotations
import csv
from dataclasses import dataclass
from pathlib import Path
from typing import Sequence
from .geo import distance_m
DRIFT_BUDGET_M = 50.0 # AC-2
COVARIANCE_WINDOW_FRAMES = 3 # AC-3: 1 before + 1 outlier + 1 after
MIN_OUTLIER_COUNT = 10 # AC-1: ~10 over Derkachi 8-min replay
@dataclass(frozen=True)
class GtPose:
"""One ground-truth pose for a video frame, keyed by frame index."""
frame_idx: int
lat_deg: float
lon_deg: float
@dataclass(frozen=True)
class OutboundEstimate:
"""One outbound estimate with covariance + label, keyed by frame index."""
frame_idx: int
monotonic_ms: int
lat_deg: float
lon_deg: float
cov_semi_major_m: float
source_label: str
@dataclass(frozen=True)
class OutlierEvent:
"""One row from the injector's manifest.csv."""
frame_idx: int
geodesic_offset_m: float
src_jpeg_path: str
@dataclass(frozen=True)
class OutlierEventReport:
"""AC-2 + AC-3 evaluation for one outlier event."""
frame_idx: int
error_before_m: float | None
error_outlier_m: float | None
error_after_m: float | None
drift_m: float | None # error_after - error_before; AC-2 budget
cov_before: float | None
cov_outlier: float | None
cov_after: float | None
cov_non_decreasing: bool
@property
def passes_drift(self) -> bool:
return (
self.drift_m is not None
and self.drift_m <= DRIFT_BUDGET_M
)
@property
def passes_covariance(self) -> bool:
return self.cov_non_decreasing
@property
def passes(self) -> bool:
return self.passes_drift and self.passes_covariance
@dataclass(frozen=True)
class OutlierToleranceReport:
"""Aggregate report for all outlier events in the replay."""
events: tuple[OutlierEventReport, ...]
total_outliers: int
@property
def passes_count(self) -> bool:
return self.total_outliers >= MIN_OUTLIER_COUNT
@property
def failed_event_count(self) -> int:
return sum(1 for e in self.events if not e.passes)
@property
def passes(self) -> bool:
return self.passes_count and self.failed_event_count == 0
def load_outlier_manifest(manifest_path: Path) -> list[OutlierEvent]:
"""Read ``outlier/manifest.csv`` into typed events.
Schema (AZ-408): ``frame_idx, src_jpeg_path, replacement_tile_x,
replacement_tile_y, geodesic_offset_m, seed``.
"""
if not manifest_path.exists():
raise FileNotFoundError(
f"outlier manifest not found: {manifest_path} — run the "
"outlier injector first (AZ-408 / runner/helpers/injector_fixtures)"
)
events: list[OutlierEvent] = []
with manifest_path.open() as fh:
reader = csv.DictReader(fh)
required = {"frame_idx", "src_jpeg_path", "geodesic_offset_m"}
missing = required - set(reader.fieldnames or [])
if missing:
raise ValueError(
f"outlier manifest {manifest_path} missing required columns: "
f"{sorted(missing)}"
)
for row in reader:
events.append(
OutlierEvent(
frame_idx=int(row["frame_idx"]),
geodesic_offset_m=float(row["geodesic_offset_m"]),
src_jpeg_path=row["src_jpeg_path"],
)
)
return events
def _index_by_frame(estimates: Sequence[OutboundEstimate]) -> dict[int, OutboundEstimate]:
by_frame: dict[int, OutboundEstimate] = {}
for e in estimates:
by_frame[e.frame_idx] = e
return by_frame
def _index_gt(gt: Sequence[GtPose]) -> dict[int, GtPose]:
by_frame: dict[int, GtPose] = {}
for g in gt:
by_frame[g.frame_idx] = g
return by_frame
def _error_m(est: OutboundEstimate | None, gt: GtPose | None) -> float | None:
if est is None or gt is None:
return None
return distance_m(gt.lat_deg, gt.lon_deg, est.lat_deg, est.lon_deg)
def evaluate_event(
event: OutlierEvent,
estimates_by_frame: dict[int, OutboundEstimate],
gt_by_frame: dict[int, GtPose],
) -> OutlierEventReport:
"""Compute the AC-2 + AC-3 report for one outlier event."""
before = estimates_by_frame.get(event.frame_idx - 1)
outlier = estimates_by_frame.get(event.frame_idx)
after = estimates_by_frame.get(event.frame_idx + 1)
gt_before = gt_by_frame.get(event.frame_idx - 1)
gt_outlier = gt_by_frame.get(event.frame_idx)
gt_after = gt_by_frame.get(event.frame_idx + 1)
err_before = _error_m(before, gt_before)
err_outlier = _error_m(outlier, gt_outlier)
err_after = _error_m(after, gt_after)
drift: float | None = None
if err_before is not None and err_after is not None:
drift = err_after - err_before
cov_before = before.cov_semi_major_m if before is not None else None
cov_outlier = outlier.cov_semi_major_m if outlier is not None else None
cov_after = after.cov_semi_major_m if after is not None else None
covs = [c for c in (cov_before, cov_outlier, cov_after) if c is not None]
cov_non_decreasing = all(covs[i + 1] >= covs[i] for i in range(len(covs) - 1))
return OutlierEventReport(
frame_idx=event.frame_idx,
error_before_m=err_before,
error_outlier_m=err_outlier,
error_after_m=err_after,
drift_m=drift,
cov_before=cov_before,
cov_outlier=cov_outlier,
cov_after=cov_after,
cov_non_decreasing=cov_non_decreasing,
)
def evaluate(
events: Sequence[OutlierEvent],
estimates: Sequence[OutboundEstimate],
gt: Sequence[GtPose],
) -> OutlierToleranceReport:
"""Aggregate report across all outlier events."""
by_frame = _index_by_frame(estimates)
gt_idx = _index_gt(gt)
reports = tuple(evaluate_event(ev, by_frame, gt_idx) for ev in events)
return OutlierToleranceReport(events=reports, total_outliers=len(events))
def write_csv_evidence(out_path: Path, report: OutlierToleranceReport) -> Path:
"""Write per-event FT-N-01 evidence CSV."""
out_path.parent.mkdir(parents=True, exist_ok=True)
with out_path.open("w", newline="") as fh:
writer = csv.writer(fh)
writer.writerow(
[
"frame_idx",
"error_before_m",
"error_outlier_m",
"error_after_m",
"drift_m",
"cov_before",
"cov_outlier",
"cov_after",
"cov_non_decreasing",
"passes_drift",
"passes_covariance",
"passes",
]
)
for e in report.events:
writer.writerow(
[
e.frame_idx,
"" if e.error_before_m is None else f"{e.error_before_m:.3f}",
"" if e.error_outlier_m is None else f"{e.error_outlier_m:.3f}",
"" if e.error_after_m is None else f"{e.error_after_m:.3f}",
"" if e.drift_m is None else f"{e.drift_m:.3f}",
"" if e.cov_before is None else f"{e.cov_before:.3f}",
"" if e.cov_outlier is None else f"{e.cov_outlier:.3f}",
"" if e.cov_after is None else f"{e.cov_after:.3f}",
"true" if e.cov_non_decreasing else "false",
"true" if e.passes_drift else "false",
"true" if e.passes_covariance else "false",
"true" if e.passes else "false",
]
)
return out_path