mirror of
https://github.com/azaion/gps-denied-onboard.git
synced 2026-06-22 22:01:14 +00:00
[AZ-432] [AZ-433] [AZ-434] [AZ-435] Add NFT-RES-01..04 resilience scenarios
Batch 86: 4 NFT-RES blackbox scenarios + 4 helper evaluators + 74 unit tests + directory-layout registration. * AZ-432 NFT-RES-01: 30 s IMU-only fallback drift bound (AC-3.5 + AC-NEW-7); two sub-cases (no_imu ≤100m, good_imu_combined_factor ≤50m). * AZ-433 NFT-RES-02: companion mid-flight reboot (AC-5.2 + AC-5.3); resume ≤30s + first-emission accuracy ≤100m. * AZ-434 NFT-RES-03: 100-iteration Monte Carlo envelope (AC-NEW-4); iteration-count + master-seed determinism + envelope ratio ≥0.95. Canonical-param by default; E2E_NFT_RES_03_FULL_MATRIX=1 unlocks matrix. * AZ-435 NFT-RES-04: 35s blackout+spoof escalation ladder (AC-NEW-8); AC-1 (cov-2d→fix-degrade ≤500ms) + AC-2 (failsafe→999+STATUSTEXT ≤500ms) + AC-ORDER (strict ordering). Verdict: PASS_WITH_WARNINGS (0 Critical, 0 High, 0 Medium, 5 Low). F5 documents intentional threshold duplication with blackout_spoof evaluator (prevents contract drift between FT-N-04 and NFT-RES-04). Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Oleksandr Bezdieniezhnykh
@@ -0,0 +1,228 @@
|
||||
"""NFT-RES-01 — 30 s IMU-only fallback drift bound (AZ-432 / AC-3.5, AC-NEW-7).
|
||||
|
||||
Tier-1 OR Tier-2. Two sub-cases run sequentially per
|
||||
``(fc_adapter, vio_strategy)``:
|
||||
|
||||
* sub-case (a) ``no_imu`` — SUT runs with IMU input disabled
|
||||
(``E2E_NFT_RES_01_DISABLE_IMU=1`` env propagated to the SUT, OR
|
||||
empty IMU stream from the FC inbound proxy). Drift budget = 100 m.
|
||||
* sub-case (b) ``good_imu_combined_factor`` — SUT default config.
|
||||
Drift budget = 50 m.
|
||||
|
||||
Each sub-case injects a 30 s pure-vision-blackout window (no spoof)
|
||||
via ``fixtures/injectors/blackout_spoof.py --no-spoof`` and measures
|
||||
the SUT's outbound estimate vs ground truth at blackout end. Drift
|
||||
is Vincenty distance.
|
||||
|
||||
Production dependency surfaced to AZ-595: the
|
||||
``E2E_NFT_RES_01_FIXTURE`` env var names a JSON file (absolute path
|
||||
or relative to ``E2E_SITL_REPLAY_DIR``) with shape:
|
||||
|
||||
{
|
||||
"window": {"onset_monotonic_ms": <int>, "end_monotonic_ms": <int>},
|
||||
"sub_cases": [
|
||||
{
|
||||
"subcase": "no_imu",
|
||||
"estimates": [{"monotonic_ms": <int>, "lat_deg": <f>, "lon_deg": <f>}, ...],
|
||||
"ground_truth": [{"monotonic_ms": <int>, "lat_deg": <f>, "lon_deg": <f>}, ...]
|
||||
},
|
||||
{
|
||||
"subcase": "good_imu_combined_factor",
|
||||
...
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Both sub-cases must be present; partial fixtures fail the test.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from runner.helpers import imu_fallback_drift_evaluator as ife
|
||||
|
||||
NFT_RES_01_FIXTURE_ENV_VAR = "E2E_NFT_RES_01_FIXTURE"
|
||||
NFT_RES_01_DEFAULT_FIXTURE_NAME = "nft_res_01_imu_fallback.json"
|
||||
|
||||
|
||||
@pytest.mark.scenario_id("nft-res-01")
|
||||
@pytest.mark.traces_to("AC-3.5,AC-NEW-7,AC-1,AC-2,AC-3,AC-4")
|
||||
def test_nft_res_01_imu_only_fallback(
|
||||
fc_adapter: str,
|
||||
vio_strategy: str,
|
||||
evidence_dir, # type: ignore[no-untyped-def]
|
||||
run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
sitl_replay_ready: bool,
|
||||
) -> None:
|
||||
"""AC-1 (window) + AC-2 (no-IMU drift) + AC-3 (good-IMU drift)."""
|
||||
if not sitl_replay_ready:
|
||||
pytest.skip(
|
||||
"NFT-RES-01 requires `E2E_SITL_REPLAY_DIR` to point at a "
|
||||
"prepared SITL replay fixture (AZ-595) carrying both sub-case "
|
||||
"estimates + GT for a 30 s blackout window. Pure-logic drift "
|
||||
"evaluation covered by "
|
||||
"e2e/_unit_tests/helpers/test_imu_fallback_drift_evaluator.py."
|
||||
)
|
||||
|
||||
fixture_path = _resolve_fixture_path()
|
||||
if not fixture_path.is_file():
|
||||
pytest.fail(
|
||||
f"NFT-RES-01: fixture not found at {fixture_path}. "
|
||||
f"`{NFT_RES_01_FIXTURE_ENV_VAR}` env var must point at a JSON "
|
||||
"file with the schema documented in the scenario docstring "
|
||||
"(window + sub-cases for no_imu + good_imu_combined_factor). "
|
||||
"Production dependency: AZ-595."
|
||||
)
|
||||
|
||||
payload = json.loads(fixture_path.read_text())
|
||||
window, sub_cases = _parse_payload(payload, fixture_path)
|
||||
if not window.window_in_spec:
|
||||
pytest.fail(
|
||||
f"NFT-RES-01: AC-1 violated — fixture window duration "
|
||||
f"{window.duration_s:.2f}s outside [28, 32]s nominal-30s "
|
||||
f"tolerance. Fixture path: {fixture_path}."
|
||||
)
|
||||
|
||||
sub_case_names = {name for name, _, _ in sub_cases}
|
||||
if sub_case_names != set(ife.ALLOWED_SUBCASES):
|
||||
pytest.fail(
|
||||
f"NFT-RES-01: fixture must contain both sub-cases "
|
||||
f"{ife.ALLOWED_SUBCASES}; got {sorted(sub_case_names)}. "
|
||||
f"Fixture path: {fixture_path}."
|
||||
)
|
||||
|
||||
report = ife.evaluate(window, sub_cases=sub_cases)
|
||||
out_csv = (
|
||||
evidence_dir
|
||||
/ "nft-res-01"
|
||||
/ f"{fc_adapter}-{vio_strategy}.csv"
|
||||
)
|
||||
ife.write_csv_evidence(out_csv, report)
|
||||
|
||||
nfr_recorder.record_metric(
|
||||
"nft_res_01.window_duration_s",
|
||||
float(report.window.duration_s),
|
||||
ac_id="AC-1",
|
||||
)
|
||||
no_imu = report.by_subcase(ife.SUBCASE_NO_IMU)
|
||||
good_imu = report.by_subcase(ife.SUBCASE_GOOD_IMU)
|
||||
if no_imu.drift_m is not None:
|
||||
nfr_recorder.record_metric(
|
||||
"nft_res_01.no_imu_drift_m", float(no_imu.drift_m), ac_id="AC-2"
|
||||
)
|
||||
if good_imu.drift_m is not None:
|
||||
nfr_recorder.record_metric(
|
||||
"nft_res_01.good_imu_drift_m", float(good_imu.drift_m), ac_id="AC-3"
|
||||
)
|
||||
|
||||
assert report.passes_window, (
|
||||
f"AC-1: 30 s window not injected; observed {report.window.duration_s:.2f}s "
|
||||
f"(tolerance ±{ife.WINDOW_TOLERANCE_S}s)"
|
||||
)
|
||||
assert no_imu.passes, (
|
||||
f"AC-2: no-IMU drift {no_imu.drift_m} m > budget {no_imu.budget_m} m "
|
||||
f"(estimate_end={no_imu.estimate_at_end}, gt_end={no_imu.gt_at_end})"
|
||||
)
|
||||
assert good_imu.passes, (
|
||||
f"AC-3: good-IMU drift {good_imu.drift_m} m > budget {good_imu.budget_m} m "
|
||||
f"(estimate_end={good_imu.estimate_at_end}, gt_end={good_imu.gt_at_end})"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_fixture_path() -> Path:
|
||||
raw = os.environ.get(NFT_RES_01_FIXTURE_ENV_VAR, "").strip()
|
||||
from runner.helpers import sitl_observer
|
||||
|
||||
root = sitl_observer.replay_dir()
|
||||
if not raw:
|
||||
if root is None:
|
||||
return Path(f"<{NFT_RES_01_FIXTURE_ENV_VAR}-unset>")
|
||||
return root / NFT_RES_01_DEFAULT_FIXTURE_NAME
|
||||
path = Path(raw)
|
||||
if not path.is_absolute() and root is not None:
|
||||
path = root / path
|
||||
return path
|
||||
|
||||
|
||||
def _parse_payload(
|
||||
payload: object, fixture_path: Path
|
||||
) -> tuple[
|
||||
ife.BlackoutWindow,
|
||||
list[tuple[str, list[ife.PositionSample], list[ife.PositionSample]]],
|
||||
]:
|
||||
if not isinstance(payload, dict):
|
||||
pytest.fail(
|
||||
f"NFT-RES-01: fixture {fixture_path} must be a JSON object; "
|
||||
f"got top-level type={type(payload).__name__}"
|
||||
)
|
||||
win_raw = payload.get("window")
|
||||
if not isinstance(win_raw, dict):
|
||||
pytest.fail(
|
||||
f"NFT-RES-01: fixture {fixture_path} missing 'window' object"
|
||||
)
|
||||
try:
|
||||
window = ife.BlackoutWindow(
|
||||
onset_monotonic_ms=int(win_raw["onset_monotonic_ms"]),
|
||||
end_monotonic_ms=int(win_raw["end_monotonic_ms"]),
|
||||
)
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
pytest.fail(
|
||||
f"NFT-RES-01: fixture {fixture_path} 'window' shape invalid: {exc}"
|
||||
)
|
||||
|
||||
subs_raw = payload.get("sub_cases")
|
||||
if not isinstance(subs_raw, list) or not subs_raw:
|
||||
pytest.fail(
|
||||
f"NFT-RES-01: fixture {fixture_path} 'sub_cases' must be a "
|
||||
f"non-empty list"
|
||||
)
|
||||
|
||||
parsed: list[tuple[str, list[ife.PositionSample], list[ife.PositionSample]]] = []
|
||||
for idx, entry in enumerate(subs_raw):
|
||||
if not isinstance(entry, dict):
|
||||
pytest.fail(
|
||||
f"NFT-RES-01: sub_cases[{idx}] in {fixture_path} must be "
|
||||
f"an object; got {type(entry).__name__}"
|
||||
)
|
||||
name = str(entry.get("subcase", ""))
|
||||
if name not in ife.ALLOWED_SUBCASES:
|
||||
pytest.fail(
|
||||
f"NFT-RES-01: sub_cases[{idx}].subcase {name!r} not in "
|
||||
f"{ife.ALLOWED_SUBCASES}"
|
||||
)
|
||||
estimates = _parse_samples(entry.get("estimates"), fixture_path, f"sub_cases[{idx}].estimates")
|
||||
ground_truth = _parse_samples(entry.get("ground_truth"), fixture_path, f"sub_cases[{idx}].ground_truth")
|
||||
parsed.append((name, estimates, ground_truth))
|
||||
return window, parsed
|
||||
|
||||
|
||||
def _parse_samples(raw: object, fixture_path: Path, where: str) -> list[ife.PositionSample]:
|
||||
if not isinstance(raw, list):
|
||||
pytest.fail(
|
||||
f"NFT-RES-01: {where} in {fixture_path} must be a list of objects"
|
||||
)
|
||||
out: list[ife.PositionSample] = []
|
||||
for j, entry in enumerate(raw):
|
||||
if not isinstance(entry, dict):
|
||||
pytest.fail(
|
||||
f"NFT-RES-01: {where}[{j}] in {fixture_path} must be an object"
|
||||
)
|
||||
try:
|
||||
out.append(
|
||||
ife.PositionSample(
|
||||
monotonic_ms=int(entry["monotonic_ms"]),
|
||||
lat_deg=float(entry["lat_deg"]),
|
||||
lon_deg=float(entry["lon_deg"]),
|
||||
)
|
||||
)
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
pytest.fail(
|
||||
f"NFT-RES-01: {where}[{j}] in {fixture_path} shape invalid: {exc}"
|
||||
)
|
||||
return out
|
||||
@@ -0,0 +1,207 @@
|
||||
"""NFT-RES-02 — Companion mid-flight reboot recovery (AZ-433 / AC-5.2 + AC-5.3).
|
||||
|
||||
Tier-1 OR Tier-2. Mid-Derkachi-replay restart (Docker on Tier-1,
|
||||
systemd on Tier-2). Asserts:
|
||||
|
||||
* AC-1 — process restarts within ≤5 s of the restart command.
|
||||
* AC-2 — first post-restart outbound emission within ≤30 s of the
|
||||
restart command.
|
||||
* AC-3 — that first emission is within ≤100 m of GT.
|
||||
|
||||
The runner harness owns the actual restart command + observation of the
|
||||
process-up timestamp + capture of the first post-restart emission. The
|
||||
scenario consumes a fixture that encodes the captured timestamps + the
|
||||
first-emission estimate + GT-at-that-timestamp.
|
||||
|
||||
Production dependency surfaced to AZ-595 / AZ-444: the
|
||||
``E2E_NFT_RES_02_FIXTURE`` env var names a JSON file with shape:
|
||||
|
||||
{
|
||||
"restart_command_monotonic_ms": <int>,
|
||||
"process_restarted_monotonic_ms": <int | null>,
|
||||
"first_post_restart_emission_monotonic_ms": <int | null>,
|
||||
"first_post_restart_estimate": {"monotonic_ms": <int>, "lat_deg": <f>, "lon_deg": <f>} | null,
|
||||
"ground_truth_at_first_emission": {"monotonic_ms": <int>, "lat_deg": <f>, "lon_deg": <f>} | null
|
||||
}
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from runner.helpers import companion_reboot_evaluator as cre
|
||||
|
||||
NFT_RES_02_FIXTURE_ENV_VAR = "E2E_NFT_RES_02_FIXTURE"
|
||||
NFT_RES_02_DEFAULT_FIXTURE_NAME = "nft_res_02_companion_reboot.json"
|
||||
|
||||
|
||||
@pytest.mark.scenario_id("nft-res-02")
|
||||
@pytest.mark.traces_to("AC-5.2,AC-5.3,AC-1,AC-2,AC-3,AC-4")
|
||||
def test_nft_res_02_companion_reboot(
|
||||
fc_adapter: str,
|
||||
vio_strategy: str,
|
||||
evidence_dir, # type: ignore[no-untyped-def]
|
||||
run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
sitl_replay_ready: bool,
|
||||
) -> None:
|
||||
"""AC-1 (restart trigger ≤5 s) + AC-2 (resume ≤30 s) + AC-3 (accuracy ≤100 m)."""
|
||||
if not sitl_replay_ready:
|
||||
pytest.skip(
|
||||
"NFT-RES-02 requires `E2E_SITL_REPLAY_DIR` to point at a "
|
||||
"prepared SITL replay fixture (AZ-595 + AZ-444) carrying "
|
||||
"restart-command + first-post-restart-emission timestamps + "
|
||||
"GT-at-emission. Pure-logic AC verdicts covered by "
|
||||
"e2e/_unit_tests/helpers/test_companion_reboot_evaluator.py."
|
||||
)
|
||||
|
||||
fixture_path = _resolve_fixture_path()
|
||||
if not fixture_path.is_file():
|
||||
pytest.fail(
|
||||
f"NFT-RES-02: fixture not found at {fixture_path}. "
|
||||
f"`{NFT_RES_02_FIXTURE_ENV_VAR}` env var must point at a JSON "
|
||||
"file with the schema documented in the scenario docstring. "
|
||||
"Production dependencies: AZ-595 (capture builder) + AZ-444 "
|
||||
"(Tier-2 systemd restart orchestration)."
|
||||
)
|
||||
|
||||
evidence = _parse_fixture(fixture_path)
|
||||
report = cre.evaluate(evidence)
|
||||
out_csv = (
|
||||
evidence_dir
|
||||
/ "nft-res-02"
|
||||
/ f"{fc_adapter}-{vio_strategy}.csv"
|
||||
)
|
||||
cre.write_csv_evidence(out_csv, report)
|
||||
|
||||
if report.restart_trigger_latency_s is not None:
|
||||
nfr_recorder.record_metric(
|
||||
"nft_res_02.restart_trigger_latency_s",
|
||||
float(report.restart_trigger_latency_s),
|
||||
ac_id="AC-1",
|
||||
)
|
||||
if report.resume_time_s is not None:
|
||||
nfr_recorder.record_metric(
|
||||
"nft_res_02.resume_time_s",
|
||||
float(report.resume_time_s),
|
||||
ac_id="AC-2",
|
||||
)
|
||||
if report.first_emission_accuracy_m is not None:
|
||||
nfr_recorder.record_metric(
|
||||
"nft_res_02.first_emission_accuracy_m",
|
||||
float(report.first_emission_accuracy_m),
|
||||
ac_id="AC-3",
|
||||
)
|
||||
|
||||
assert report.passes_restart_trigger, (
|
||||
f"AC-1: restart trigger latency = {report.restart_trigger_latency_s} s > budget "
|
||||
f"{report.restart_trigger_budget_s} s (process_restarted_ms="
|
||||
f"{evidence.process_restarted_monotonic_ms})"
|
||||
)
|
||||
assert report.passes_resume_time, (
|
||||
f"AC-2: resume time = {report.resume_time_s} s > budget "
|
||||
f"{report.resume_budget_s} s (first_emission_ms="
|
||||
f"{evidence.first_post_restart_emission_monotonic_ms})"
|
||||
)
|
||||
assert report.passes_first_emission_accuracy, (
|
||||
f"AC-3: first-emission accuracy = {report.first_emission_accuracy_m} m > "
|
||||
f"budget {report.accuracy_budget_m} m "
|
||||
f"(estimate={evidence.first_post_restart_estimate}, "
|
||||
f"gt={evidence.ground_truth_at_first_emission})"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_fixture_path() -> Path:
|
||||
raw = os.environ.get(NFT_RES_02_FIXTURE_ENV_VAR, "").strip()
|
||||
from runner.helpers import sitl_observer
|
||||
|
||||
root = sitl_observer.replay_dir()
|
||||
if not raw:
|
||||
if root is None:
|
||||
return Path(f"<{NFT_RES_02_FIXTURE_ENV_VAR}-unset>")
|
||||
return root / NFT_RES_02_DEFAULT_FIXTURE_NAME
|
||||
path = Path(raw)
|
||||
if not path.is_absolute() and root is not None:
|
||||
path = root / path
|
||||
return path
|
||||
|
||||
|
||||
def _parse_fixture(fixture_path: Path) -> cre.RestartEvidence:
|
||||
payload = json.loads(fixture_path.read_text())
|
||||
if not isinstance(payload, dict):
|
||||
pytest.fail(
|
||||
f"NFT-RES-02: fixture {fixture_path} must be a JSON object; "
|
||||
f"got top-level type={type(payload).__name__}"
|
||||
)
|
||||
try:
|
||||
command_ms = int(payload["restart_command_monotonic_ms"])
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
pytest.fail(
|
||||
f"NFT-RES-02: fixture {fixture_path} missing/invalid "
|
||||
f"'restart_command_monotonic_ms': {exc}"
|
||||
)
|
||||
|
||||
process_ms = _maybe_int(
|
||||
payload.get("process_restarted_monotonic_ms"),
|
||||
fixture_path,
|
||||
"process_restarted_monotonic_ms",
|
||||
)
|
||||
first_emission_ms = _maybe_int(
|
||||
payload.get("first_post_restart_emission_monotonic_ms"),
|
||||
fixture_path,
|
||||
"first_post_restart_emission_monotonic_ms",
|
||||
)
|
||||
estimate = _maybe_geofix(
|
||||
payload.get("first_post_restart_estimate"),
|
||||
fixture_path,
|
||||
"first_post_restart_estimate",
|
||||
)
|
||||
gt = _maybe_geofix(
|
||||
payload.get("ground_truth_at_first_emission"),
|
||||
fixture_path,
|
||||
"ground_truth_at_first_emission",
|
||||
)
|
||||
|
||||
return cre.RestartEvidence(
|
||||
restart_command_monotonic_ms=command_ms,
|
||||
process_restarted_monotonic_ms=process_ms,
|
||||
first_post_restart_emission_monotonic_ms=first_emission_ms,
|
||||
first_post_restart_estimate=estimate,
|
||||
ground_truth_at_first_emission=gt,
|
||||
)
|
||||
|
||||
|
||||
def _maybe_int(raw: object, fixture_path: Path, where: str) -> int | None:
|
||||
if raw is None:
|
||||
return None
|
||||
try:
|
||||
return int(raw)
|
||||
except (TypeError, ValueError) as exc:
|
||||
pytest.fail(
|
||||
f"NFT-RES-02: {where} in {fixture_path} must be int or null: {exc}"
|
||||
)
|
||||
return None # unreachable; pytest.fail raises
|
||||
|
||||
|
||||
def _maybe_geofix(raw: object, fixture_path: Path, where: str) -> cre.GeoFix | None:
|
||||
if raw is None:
|
||||
return None
|
||||
if not isinstance(raw, dict):
|
||||
pytest.fail(
|
||||
f"NFT-RES-02: {where} in {fixture_path} must be object or null"
|
||||
)
|
||||
try:
|
||||
return cre.GeoFix(
|
||||
monotonic_ms=int(raw["monotonic_ms"]),
|
||||
lat_deg=float(raw["lat_deg"]),
|
||||
lon_deg=float(raw["lon_deg"]),
|
||||
)
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
pytest.fail(
|
||||
f"NFT-RES-02: {where} in {fixture_path} shape invalid: {exc}"
|
||||
)
|
||||
return None # unreachable; pytest.fail raises
|
||||
@@ -0,0 +1,235 @@
|
||||
"""NFT-RES-03 — 100-iteration Monte Carlo statistical envelope (AZ-434 / AC-NEW-4).
|
||||
|
||||
Tier-1 OR Tier-2. The runner orchestrates 100 Derkachi replays with
|
||||
seeded perturbations (gain noise, IMU bias, frame-drop, outlier
|
||||
injection) and supplies this scenario with a captured fixture
|
||||
containing per-iteration per-frame ``(error_m, cov_semi_major_m)``
|
||||
pairs. The scenario validates:
|
||||
|
||||
* AC-1 — iteration_count ≥ 100.
|
||||
* AC-2 — same master_seed yields bit-identical iteration outcomes
|
||||
(verified by re-evaluating the same fixture twice and comparing
|
||||
``determinism_fingerprint``).
|
||||
* AC-3 — global aggregate envelope:
|
||||
``count(error_m ≤ 1.96 × cov_semi_major_m) / total ≥ 0.95``.
|
||||
* AC-4 — parameterization: SHOULD run only one canonical
|
||||
parameterization per CI invocation by default; full-matrix mode
|
||||
gated behind ``E2E_NFT_RES_03_FULL_MATRIX=1``. The scenario uses
|
||||
``fc_adapter`` + ``vio_strategy`` fixtures so the harness param
|
||||
matrix decides which combinations to run.
|
||||
|
||||
Production dependency surfaced to AZ-595: the
|
||||
``E2E_NFT_RES_03_FIXTURE`` env var names a JSON file with shape:
|
||||
|
||||
{
|
||||
"master_seed": <int>,
|
||||
"iterations": [
|
||||
{
|
||||
"iteration_id": "iter-001",
|
||||
"iteration_seed": <int>,
|
||||
"samples": [{"error_m": <f>, "cov_semi_major_m": <f>}, ...]
|
||||
},
|
||||
...
|
||||
]
|
||||
}
|
||||
|
||||
The harness MAY emit the fixture with a single canonical parameterization
|
||||
per CI invocation by default — ``E2E_NFT_RES_03_FULL_MATRIX=1``
|
||||
unlocks the full 100 × N_params expansion.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from runner.helpers import monte_carlo_envelope_evaluator as mce
|
||||
|
||||
NFT_RES_03_FIXTURE_ENV_VAR = "E2E_NFT_RES_03_FIXTURE"
|
||||
NFT_RES_03_DEFAULT_FIXTURE_NAME = "nft_res_03_monte_carlo.json"
|
||||
NFT_RES_03_FULL_MATRIX_ENV_VAR = "E2E_NFT_RES_03_FULL_MATRIX"
|
||||
NFT_RES_03_CANONICAL_FC = "ardupilot"
|
||||
NFT_RES_03_CANONICAL_VIO = "okvis2"
|
||||
|
||||
|
||||
@pytest.mark.scenario_id("nft-res-03")
|
||||
@pytest.mark.traces_to("AC-NEW-4,AC-1,AC-2,AC-3,AC-4")
|
||||
def test_nft_res_03_monte_carlo(
|
||||
fc_adapter: str,
|
||||
vio_strategy: str,
|
||||
evidence_dir, # type: ignore[no-untyped-def]
|
||||
run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
sitl_replay_ready: bool,
|
||||
) -> None:
|
||||
"""AC-1 (iteration count) + AC-2 (determinism) + AC-3 (envelope) + AC-4 (param)."""
|
||||
if not _full_matrix_enabled() and (
|
||||
fc_adapter != NFT_RES_03_CANONICAL_FC
|
||||
or vio_strategy != NFT_RES_03_CANONICAL_VIO
|
||||
):
|
||||
pytest.skip(
|
||||
f"NFT-RES-03 AC-4: by default runs only canonical "
|
||||
f"({NFT_RES_03_CANONICAL_FC}, {NFT_RES_03_CANONICAL_VIO}); "
|
||||
f"set {NFT_RES_03_FULL_MATRIX_ENV_VAR}=1 to enable the "
|
||||
f"100 × N_params full-matrix expansion."
|
||||
)
|
||||
|
||||
if not sitl_replay_ready:
|
||||
pytest.skip(
|
||||
"NFT-RES-03 requires `E2E_SITL_REPLAY_DIR` to point at a "
|
||||
"prepared SITL replay fixture (AZ-595) carrying N≥100 "
|
||||
"Monte Carlo iterations. Pure-logic AC-1 + AC-2 + AC-3 "
|
||||
"covered by "
|
||||
"e2e/_unit_tests/helpers/test_monte_carlo_envelope_evaluator.py."
|
||||
)
|
||||
|
||||
fixture_path = _resolve_fixture_path()
|
||||
if not fixture_path.is_file():
|
||||
pytest.fail(
|
||||
f"NFT-RES-03: fixture not found at {fixture_path}. "
|
||||
f"`{NFT_RES_03_FIXTURE_ENV_VAR}` env var must point at a JSON "
|
||||
"file with the schema documented in the scenario docstring. "
|
||||
"Production dependency: AZ-595."
|
||||
)
|
||||
|
||||
payload = json.loads(fixture_path.read_text())
|
||||
master_seed, iterations = _parse_payload(payload, fixture_path)
|
||||
|
||||
report1 = mce.evaluate(iterations, master_seed=master_seed)
|
||||
report2 = mce.evaluate(iterations, master_seed=master_seed)
|
||||
fingerprint = mce.determinism_fingerprint(report1)
|
||||
fingerprint2 = mce.determinism_fingerprint(report2)
|
||||
|
||||
out_base = (
|
||||
evidence_dir
|
||||
/ "nft-res-03"
|
||||
/ f"{fc_adapter}-{vio_strategy}"
|
||||
)
|
||||
mce.write_csv_evidence(out_base.with_suffix(".csv"), report1)
|
||||
mce.write_per_iteration_csv(
|
||||
out_base.with_name(out_base.name + "-per-iter").with_suffix(".csv"),
|
||||
report1,
|
||||
)
|
||||
|
||||
nfr_recorder.record_metric(
|
||||
"nft_res_03.iteration_count", float(report1.iteration_count), ac_id="AC-1"
|
||||
)
|
||||
nfr_recorder.record_metric(
|
||||
"nft_res_03.total_samples", float(report1.total_samples)
|
||||
)
|
||||
if report1.envelope_ratio is not None:
|
||||
nfr_recorder.record_metric(
|
||||
"nft_res_03.envelope_ratio", float(report1.envelope_ratio), ac_id="AC-3"
|
||||
)
|
||||
nfr_recorder.record_metric(
|
||||
"nft_res_03.master_seed", float(report1.master_seed)
|
||||
)
|
||||
|
||||
assert report1.passes_iteration_count, (
|
||||
f"AC-1: iteration_count={report1.iteration_count} < required "
|
||||
f"{report1.min_iteration_count}"
|
||||
)
|
||||
assert fingerprint == fingerprint2, (
|
||||
f"AC-2: determinism fingerprint differs across two evaluations of the "
|
||||
f"same fixture: {fingerprint} vs {fingerprint2}"
|
||||
)
|
||||
assert report1.passes_envelope, (
|
||||
f"AC-3: envelope ratio = {report1.envelope_ratio} < budget "
|
||||
f"{report1.envelope_ratio_budget} "
|
||||
f"(covered={report1.covered_samples}/{report1.total_samples})"
|
||||
)
|
||||
|
||||
|
||||
def _full_matrix_enabled() -> bool:
|
||||
return os.environ.get(NFT_RES_03_FULL_MATRIX_ENV_VAR, "").strip() in {"1", "true", "yes"}
|
||||
|
||||
|
||||
def _resolve_fixture_path() -> Path:
|
||||
raw = os.environ.get(NFT_RES_03_FIXTURE_ENV_VAR, "").strip()
|
||||
from runner.helpers import sitl_observer
|
||||
|
||||
root = sitl_observer.replay_dir()
|
||||
if not raw:
|
||||
if root is None:
|
||||
return Path(f"<{NFT_RES_03_FIXTURE_ENV_VAR}-unset>")
|
||||
return root / NFT_RES_03_DEFAULT_FIXTURE_NAME
|
||||
path = Path(raw)
|
||||
if not path.is_absolute() and root is not None:
|
||||
path = root / path
|
||||
return path
|
||||
|
||||
|
||||
def _parse_payload(
|
||||
payload: object, fixture_path: Path
|
||||
) -> tuple[int, list[mce.IterationOutcome]]:
|
||||
if not isinstance(payload, dict):
|
||||
pytest.fail(
|
||||
f"NFT-RES-03: fixture {fixture_path} must be a JSON object; "
|
||||
f"got top-level type={type(payload).__name__}"
|
||||
)
|
||||
try:
|
||||
master_seed = int(payload["master_seed"])
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
pytest.fail(
|
||||
f"NFT-RES-03: fixture {fixture_path} missing/invalid "
|
||||
f"'master_seed': {exc}"
|
||||
)
|
||||
|
||||
raw_iters = payload.get("iterations")
|
||||
if not isinstance(raw_iters, list) or not raw_iters:
|
||||
pytest.fail(
|
||||
f"NFT-RES-03: fixture {fixture_path} 'iterations' must be a "
|
||||
f"non-empty list"
|
||||
)
|
||||
|
||||
parsed: list[mce.IterationOutcome] = []
|
||||
for idx, entry in enumerate(raw_iters):
|
||||
if not isinstance(entry, dict):
|
||||
pytest.fail(
|
||||
f"NFT-RES-03: iterations[{idx}] in {fixture_path} must be "
|
||||
f"an object; got {type(entry).__name__}"
|
||||
)
|
||||
iter_id = str(entry.get("iteration_id") or f"iter-{idx:03d}")
|
||||
try:
|
||||
seed = int(entry["iteration_seed"])
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
pytest.fail(
|
||||
f"NFT-RES-03: iterations[{idx}].iteration_seed in "
|
||||
f"{fixture_path} must be int: {exc}"
|
||||
)
|
||||
raw_samples = entry.get("samples")
|
||||
if not isinstance(raw_samples, list):
|
||||
pytest.fail(
|
||||
f"NFT-RES-03: iterations[{idx}].samples in {fixture_path} "
|
||||
f"must be a list of objects"
|
||||
)
|
||||
samples: list[mce.FrameSample] = []
|
||||
for j, s in enumerate(raw_samples):
|
||||
if not isinstance(s, dict):
|
||||
pytest.fail(
|
||||
f"NFT-RES-03: iterations[{idx}].samples[{j}] in "
|
||||
f"{fixture_path} must be an object"
|
||||
)
|
||||
try:
|
||||
samples.append(
|
||||
mce.FrameSample(
|
||||
error_m=float(s["error_m"]),
|
||||
cov_semi_major_m=float(s["cov_semi_major_m"]),
|
||||
)
|
||||
)
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
pytest.fail(
|
||||
f"NFT-RES-03: iterations[{idx}].samples[{j}] in "
|
||||
f"{fixture_path} shape invalid: {exc}"
|
||||
)
|
||||
parsed.append(
|
||||
mce.IterationOutcome(
|
||||
iteration_id=iter_id,
|
||||
iteration_seed=seed,
|
||||
samples=tuple(samples),
|
||||
)
|
||||
)
|
||||
return master_seed, parsed
|
||||
@@ -0,0 +1,227 @@
|
||||
"""NFT-RES-04 — 35 s blackout + spoof full escalation ladder (AZ-435 / AC-NEW-8 escalation).
|
||||
|
||||
Tier-1 OR Tier-2. Sibling of FT-N-04 — same 35 s window with spoof,
|
||||
but asserts the *full* escalation ladder fires in observable order
|
||||
under tight latency budgets:
|
||||
|
||||
* AC-1 — 100 m covariance → fix-type degrade within ≤500 ms.
|
||||
* AC-2 — 500 m covariance OR 30 s elapsed → horiz_accuracy=999.0
|
||||
AND ``VISUAL_BLACKOUT_FAILSAFE`` STATUSTEXT within ≤500 ms.
|
||||
* AC-ORDER — AC-1 crossing strictly precedes the AC-2 trigger.
|
||||
* AC-3 — parameterized over (fc_adapter, vio_strategy).
|
||||
|
||||
The runner consumes the same Derkachi replay + blackout-spoof
|
||||
injector fixture as FT-N-04 (``E2E_SITL_REPLAY_DIR``), so the
|
||||
``E2E_NFT_RES_04_FIXTURE`` env var defaults to the same payload.
|
||||
This avoids duplicating the 35 s captured trace just for the
|
||||
resilience-tier assertions.
|
||||
|
||||
Production dependency surfaced to AZ-595: the fixture JSON has shape:
|
||||
|
||||
{
|
||||
"window": {"onset_monotonic_ms": <int>, "end_monotonic_ms": <int>},
|
||||
"estimates": [
|
||||
{"monotonic_ms": <int>, "cov_semi_major_m": <f>,
|
||||
"horiz_accuracy": <f>, "fix_type": <int>}, ...
|
||||
],
|
||||
"statustexts": [
|
||||
{"monotonic_ms": <int>, "text": <str>}, ...
|
||||
]
|
||||
}
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from runner.helpers import escalation_ladder_evaluator as ele
|
||||
|
||||
NFT_RES_04_FIXTURE_ENV_VAR = "E2E_NFT_RES_04_FIXTURE"
|
||||
NFT_RES_04_DEFAULT_FIXTURE_NAME = "nft_res_04_blackout_escalation.json"
|
||||
|
||||
|
||||
@pytest.mark.scenario_id("nft-res-04")
|
||||
@pytest.mark.traces_to("AC-NEW-8,AC-1,AC-2,AC-3")
|
||||
def test_nft_res_04_blackout_escalation(
|
||||
fc_adapter: str,
|
||||
vio_strategy: str,
|
||||
evidence_dir, # type: ignore[no-untyped-def]
|
||||
run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
sitl_replay_ready: bool,
|
||||
) -> None:
|
||||
"""AC-1 + AC-2 + AC-ORDER for the 35 s spoof+blackout window."""
|
||||
if not sitl_replay_ready:
|
||||
pytest.skip(
|
||||
"NFT-RES-04 requires `E2E_SITL_REPLAY_DIR` to point at a "
|
||||
"prepared SITL replay fixture (AZ-595) carrying the 35 s "
|
||||
"spoof+blackout window with cov_semi_major_m, horiz_accuracy, "
|
||||
"fix_type, and STATUSTEXT samples. Pure-logic AC-1/AC-2/AC-ORDER "
|
||||
"covered by "
|
||||
"e2e/_unit_tests/helpers/test_escalation_ladder_evaluator.py."
|
||||
)
|
||||
|
||||
fixture_path = _resolve_fixture_path()
|
||||
if not fixture_path.is_file():
|
||||
pytest.fail(
|
||||
f"NFT-RES-04: fixture not found at {fixture_path}. "
|
||||
f"`{NFT_RES_04_FIXTURE_ENV_VAR}` env var must point at a JSON "
|
||||
"file with the schema documented in the scenario docstring. "
|
||||
"Production dependency: AZ-595."
|
||||
)
|
||||
|
||||
payload = json.loads(fixture_path.read_text())
|
||||
window, estimates, statustexts = _parse_payload(payload, fixture_path)
|
||||
if not window.is_35s:
|
||||
pytest.fail(
|
||||
f"NFT-RES-04: window duration {window.duration_s:.2f}s outside "
|
||||
f"35±2s — the resilience-tier scenario only meaningfully covers "
|
||||
f"the 35 s sub-case; other sub-cases are owned by FT-N-04 "
|
||||
f"({fixture_path})."
|
||||
)
|
||||
|
||||
report = ele.evaluate(window, estimates=estimates, statustexts=statustexts)
|
||||
out_csv = (
|
||||
evidence_dir
|
||||
/ "nft-res-04"
|
||||
/ f"{fc_adapter}-{vio_strategy}.csv"
|
||||
)
|
||||
ele.write_csv_evidence(out_csv, report)
|
||||
|
||||
if report.fix_degrade.latency_ms is not None:
|
||||
nfr_recorder.record_metric(
|
||||
"nft_res_04.cov2d_to_fix_degrade_latency_ms",
|
||||
float(report.fix_degrade.latency_ms),
|
||||
ac_id="AC-1",
|
||||
)
|
||||
if report.failsafe.horiz_999_latency_ms is not None:
|
||||
nfr_recorder.record_metric(
|
||||
"nft_res_04.failsafe_to_horiz999_latency_ms",
|
||||
float(report.failsafe.horiz_999_latency_ms),
|
||||
ac_id="AC-2",
|
||||
)
|
||||
if report.failsafe.statustext_latency_ms is not None:
|
||||
nfr_recorder.record_metric(
|
||||
"nft_res_04.failsafe_to_statustext_latency_ms",
|
||||
float(report.failsafe.statustext_latency_ms),
|
||||
ac_id="AC-2",
|
||||
)
|
||||
|
||||
assert report.fix_degrade.passes, (
|
||||
f"AC-1: cov-2d → fix-degrade latency = "
|
||||
f"{report.fix_degrade.latency_ms} ms (budget {report.fix_degrade.budget_ms} ms); "
|
||||
f"cov2d_at_ms={report.fix_degrade.cov2d_crossed_at_ms}, "
|
||||
f"fix_degraded_at_ms={report.fix_degrade.fix_degraded_at_ms}"
|
||||
)
|
||||
assert report.failsafe.passes, (
|
||||
f"AC-2: failsafe escalation incomplete; "
|
||||
f"trigger_at_ms={report.failsafe.failsafe_trigger_at_ms}, "
|
||||
f"horiz_999_latency_ms={report.failsafe.horiz_999_latency_ms}, "
|
||||
f"statustext_latency_ms={report.failsafe.statustext_latency_ms}, "
|
||||
f"budget {report.failsafe.budget_ms} ms"
|
||||
)
|
||||
assert report.ordering.passes, (
|
||||
f"AC-ORDER: cov-2d crossing must strictly precede failsafe trigger; "
|
||||
f"cov2d_at_ms={report.ordering.cov2d_at_ms}, "
|
||||
f"failsafe_trigger_at_ms={report.ordering.failsafe_trigger_at_ms}"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_fixture_path() -> Path:
|
||||
raw = os.environ.get(NFT_RES_04_FIXTURE_ENV_VAR, "").strip()
|
||||
from runner.helpers import sitl_observer
|
||||
|
||||
root = sitl_observer.replay_dir()
|
||||
if not raw:
|
||||
if root is None:
|
||||
return Path(f"<{NFT_RES_04_FIXTURE_ENV_VAR}-unset>")
|
||||
return root / NFT_RES_04_DEFAULT_FIXTURE_NAME
|
||||
path = Path(raw)
|
||||
if not path.is_absolute() and root is not None:
|
||||
path = root / path
|
||||
return path
|
||||
|
||||
|
||||
def _parse_payload(
|
||||
payload: object, fixture_path: Path
|
||||
) -> tuple[
|
||||
ele.BlackoutWindow,
|
||||
list[ele.EstimateSample],
|
||||
list[ele.StatustextSample],
|
||||
]:
|
||||
if not isinstance(payload, dict):
|
||||
pytest.fail(
|
||||
f"NFT-RES-04: fixture {fixture_path} must be a JSON object; "
|
||||
f"got top-level type={type(payload).__name__}"
|
||||
)
|
||||
win_raw = payload.get("window")
|
||||
if not isinstance(win_raw, dict):
|
||||
pytest.fail(
|
||||
f"NFT-RES-04: fixture {fixture_path} missing 'window' object"
|
||||
)
|
||||
try:
|
||||
window = ele.BlackoutWindow(
|
||||
onset_monotonic_ms=int(win_raw["onset_monotonic_ms"]),
|
||||
end_monotonic_ms=int(win_raw["end_monotonic_ms"]),
|
||||
)
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
pytest.fail(
|
||||
f"NFT-RES-04: fixture {fixture_path} 'window' shape invalid: {exc}"
|
||||
)
|
||||
|
||||
raw_estimates = payload.get("estimates")
|
||||
if not isinstance(raw_estimates, list):
|
||||
pytest.fail(
|
||||
f"NFT-RES-04: fixture {fixture_path} 'estimates' must be a list"
|
||||
)
|
||||
estimates: list[ele.EstimateSample] = []
|
||||
for idx, entry in enumerate(raw_estimates):
|
||||
if not isinstance(entry, dict):
|
||||
pytest.fail(
|
||||
f"NFT-RES-04: estimates[{idx}] in {fixture_path} must be "
|
||||
f"an object; got {type(entry).__name__}"
|
||||
)
|
||||
try:
|
||||
estimates.append(
|
||||
ele.EstimateSample(
|
||||
monotonic_ms=int(entry["monotonic_ms"]),
|
||||
cov_semi_major_m=float(entry["cov_semi_major_m"]),
|
||||
horiz_accuracy=float(entry["horiz_accuracy"]),
|
||||
fix_type=int(entry["fix_type"]),
|
||||
)
|
||||
)
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
pytest.fail(
|
||||
f"NFT-RES-04: estimates[{idx}] in {fixture_path} shape invalid: {exc}"
|
||||
)
|
||||
|
||||
raw_st = payload.get("statustexts", [])
|
||||
if not isinstance(raw_st, list):
|
||||
pytest.fail(
|
||||
f"NFT-RES-04: fixture {fixture_path} 'statustexts' must be a list "
|
||||
"(may be empty)"
|
||||
)
|
||||
statustexts: list[ele.StatustextSample] = []
|
||||
for idx, entry in enumerate(raw_st):
|
||||
if not isinstance(entry, dict):
|
||||
pytest.fail(
|
||||
f"NFT-RES-04: statustexts[{idx}] in {fixture_path} must be "
|
||||
f"an object"
|
||||
)
|
||||
try:
|
||||
statustexts.append(
|
||||
ele.StatustextSample(
|
||||
monotonic_ms=int(entry["monotonic_ms"]),
|
||||
text=str(entry["text"]),
|
||||
)
|
||||
)
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
pytest.fail(
|
||||
f"NFT-RES-04: statustexts[{idx}] in {fixture_path} shape invalid: {exc}"
|
||||
)
|
||||
|
||||
return window, estimates, statustexts
|
||||
Reference in New Issue
Block a user