azaion/gps-denied-onboard
1
0
Fork 0
mirror of https://github.com/azaion/gps-denied-onboard.git synced 2026-06-22 19:21:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

[AZ-436] [AZ-437] [AZ-438] [AZ-439] Add NFT-SEC-01..05 security scenarios

Browse Source 
Batch 87: 6 NFT-SEC blackbox scenarios + 5 helper evaluators + 75 unit
tests + cumulative review batches 85-87.

* AZ-436 NFT-SEC-01: cache-poisoning safety budget (AC-NEW-9); aggregate
  false_trust_count ≤ N×1e-6; zero-tolerance default. Canonical-only by
  default; E2E_NFT_SEC_01_RELEASE_GATE=1 unlocks full matrix.
* AZ-437 NFT-SEC-02 + NFT-SEC-05: shared egress-observation evaluator
  (AC-NEW-10); SEC-02 = 0 packets to non-e2e-net over 5min replay;
  SEC-05 = DNS-blackhole sidecar healthy + lookup fails + UDP-53 silent.
* AZ-438 NFT-SEC-03: AP-only signing rejection (AC-NEW-11); 3 sub-cases
  (unsigned/wrong-key/replayed) each reject ≤500ms + no position drift.
* AZ-439 NFT-SEC-04: probe (always-run) = no-crash + deterministic
  decode outcome; ASan-fuzz (release-gate) = 0 findings ≥4h; AC-3
  corpus floor informational only per spec.

Verdict per-batch: PASS_WITH_WARNINGS (5 Low). Cumulative review for
batches 85-87 (K=3 window) also PASS_WITH_WARNINGS with 5 cross-batch
findings — recommends hygiene PBIs for write_csv_evidence duplication
(13 helpers) and _resolve_fixture_path duplication (13 scenarios), plus
new tickets for AZ-595 fixture builder + DNS-blackhole sidecar service.

Also adds _docs/LESSONS.md documenting the Jira transition-ID lesson
(always call getTransitionsForJiraIssue first, never memorize numeric
IDs across sessions).

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Oleksandr Bezdieniezhnykh
2026-05-17 17:33:22 +03:00
parent de19e716d8
commit c56d4584e6
21 changed files with 3510 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
e2e/tests/security/test_nft_sec_01_cache_poisoning.py
+237
View File
@@ -0,0 +1,237 @@
"""NFT-SEC-01 — Cache-poisoning safety probability ≤ 1e-6 / flight (AZ-436 / AC-NEW-9).
Tier-1 OR Tier-2. ``N`` synthetic micro-flights (~60 s each) carry ``[1 %,
5 %]`` of crafted poisoned tiles (signing-cert mismatch, freshness
violation, voting disagreement). The SUT MUST reject or downgrade them
via three independent defense layers and emit zero false-trust events.
Default CI runs N=1000 with a single canonical parameterization to keep
total runtime bounded; the full release-gate run is N=10000 across
``(fc_adapter × vio_strategy)`` and is gated behind
``E2E_NFT_SEC_01_RELEASE_GATE=1``.
Production dependencies surfaced to the cumulative review window:
* **AZ-595**: emit ``nft_sec_01_cache_poisoning.json`` containing
per-flight tile-cache slates + runner-collected false-trust events
+ per-flight ``rejection_reasons`` counter — see fixture JSON shape
in the docstring of ``_parse_payload``.
* **SUT**: outbound ``source_label`` MUST carry the ``tile_id`` so the
runner can match a ``satellite_anchored`` frame back to a poisoned
tile; otherwise false-trust events cannot be detected reliably.
Pure aggregate-budget logic is fully covered by
``e2e/_unit_tests/helpers/test_cache_poisoning_evaluator.py``; the
scenario test only validates the fixture parser, the AC assertions, and
the conftest skip-rules.
"""
from __future__ import annotations
import json
import os
from pathlib import Path
import pytest
from runner.helpers import cache_poisoning_evaluator as cpe
NFT_SEC_01_FIXTURE_ENV_VAR = "E2E_NFT_SEC_01_FIXTURE"
NFT_SEC_01_DEFAULT_FIXTURE_NAME = "nft_sec_01_cache_poisoning.json"
NFT_SEC_01_RELEASE_GATE_ENV_VAR = "E2E_NFT_SEC_01_RELEASE_GATE"
NFT_SEC_01_CI_MIN_FLIGHTS = 1000
@pytest.mark.scenario_id("nft-sec-01")
@pytest.mark.traces_to("AC-NEW-9,AC-1,AC-2,AC-3,AC-4")
def test_nft_sec_01_cache_poisoning(
fc_adapter: str,
vio_strategy: str,
evidence_dir, # type: ignore[no-untyped-def]
run_id: str,
nfr_recorder, # type: ignore[no-untyped-def]
sitl_replay_ready: bool,
) -> None:
"""Aggregate false-trust count ≤ N × 1e-6 (zero-tolerance default)."""
release_gate = _release_gate_enabled()
if not release_gate and not _is_canonical_param(fc_adapter, vio_strategy):
pytest.skip(
"NFT-SEC-01 default CI run uses a single canonical "
"parameterization (ardupilot, okvis2) to keep N=1000 × 4 "
"Monte Carlo cost bounded. Set "
f"`{NFT_SEC_01_RELEASE_GATE_ENV_VAR}=1` for the full matrix."
)
if not sitl_replay_ready:
pytest.skip(
"NFT-SEC-01 requires `E2E_SITL_REPLAY_DIR` to point at a "
"prepared SITL replay fixture (AZ-595) carrying the N "
"synthetic flights with crafted poisoned tiles. Pure "
"aggregate-budget logic covered by "
"e2e/_unit_tests/helpers/test_cache_poisoning_evaluator.py."
)
fixture_path = _resolve_fixture_path()
if not fixture_path.is_file():
pytest.fail(
f"NFT-SEC-01: fixture not found at {fixture_path}. "
f"`{NFT_SEC_01_FIXTURE_ENV_VAR}` env var must point at a JSON "
"file with the schema documented in the scenario docstring. "
"Production dependency: AZ-595."
)
payload = json.loads(fixture_path.read_text())
flights = _parse_payload(payload, fixture_path)
if len(flights) < NFT_SEC_01_CI_MIN_FLIGHTS and not release_gate:
pytest.fail(
f"NFT-SEC-01 AC-1: fixture provides only {len(flights)} flights "
f"but the CI default requires ≥{NFT_SEC_01_CI_MIN_FLIGHTS}. "
f"Set `{NFT_SEC_01_RELEASE_GATE_ENV_VAR}=1` to allow shorter runs "
"for debugging."
)
report = cpe.evaluate(flights)
out_csv = (
evidence_dir
/ "nft-sec-01"
/ f"{fc_adapter}-{vio_strategy}.csv"
)
cpe.write_csv_evidence(out_csv, report)
nfr_recorder.record_metric(
"nft_sec_01.flight_count",
float(report.flight_count),
ac_id="AC-1",
)
nfr_recorder.record_metric(
"nft_sec_01.total_false_trust",
float(report.total_false_trust),
ac_id="AC-3",
)
nfr_recorder.record_metric(
"nft_sec_01.budget",
report.budget,
ac_id="AC-3",
)
assert report.passes_ratio, (
"AC-2: poison ratio outside [1%, 5%] in flights: "
f"{list(report.flights_with_bad_poison_ratio)[:10]}"
)
assert report.passes_layer_coverage, (
"AC-2: at least one defense layer absent from flight: "
f"{list(report.flights_missing_defense_layers)[:10]}"
)
assert report.passes_rejection_reason_vocabulary, (
"AC-2 evidence: unknown rejection_reason vocabulary in flights: "
f"{list(report.flights_with_unknown_rejection_reasons)[:10]}"
)
assert report.passes_budget, (
f"AC-3: total_false_trust = {report.total_false_trust} "
f"(budget {report.budget:g} expected events at N={report.flight_count}; "
"zero-tolerance default — see Mode B Fact #103)."
)
def _release_gate_enabled() -> bool:
return os.environ.get(NFT_SEC_01_RELEASE_GATE_ENV_VAR, "").strip().lower() in (
"1",
"true",
"yes",
)
def _is_canonical_param(fc_adapter: str, vio_strategy: str) -> bool:
return fc_adapter == "ardupilot" and vio_strategy == "okvis2"
def _resolve_fixture_path() -> Path:
raw = os.environ.get(NFT_SEC_01_FIXTURE_ENV_VAR, "").strip()
from runner.helpers import sitl_observer
root = sitl_observer.replay_dir()
if not raw:
if root is None:
return Path(f"<{NFT_SEC_01_FIXTURE_ENV_VAR}-unset>")
return root / NFT_SEC_01_DEFAULT_FIXTURE_NAME
path = Path(raw)
if not path.is_absolute() and root is not None:
path = root / path
return path
def _parse_payload(
payload: object, fixture_path: Path
) -> list[cpe.FlightOutcome]:
"""Parse the fixture into typed ``FlightOutcome`` records.
Expected shape:
{
"flights": [
{
"flight_id": "<str>",
"total_tile_count": <int>,
"poisoned_tiles": [
{"tile_id": "<str>", "defense_layer": "<str>"}, ...
],
"false_trust_events": [
{"flight_id": "<str>", "tile_id": "<str>",
"monotonic_ms": <int>, "defense_layer": "<str>"}, ...
],
"rejection_reasons": {"<reason>": <int>, ...}
}, ...
]
}
"""
if not isinstance(payload, dict):
pytest.fail(
f"NFT-SEC-01: fixture {fixture_path} must be a JSON object; "
f"got top-level type={type(payload).__name__}"
)
raw_flights = payload.get("flights")
if not isinstance(raw_flights, list):
pytest.fail(
f"NFT-SEC-01: fixture {fixture_path} 'flights' must be a list"
)
flights: list[cpe.FlightOutcome] = []
for idx, entry in enumerate(raw_flights):
if not isinstance(entry, dict):
pytest.fail(
f"NFT-SEC-01: flights[{idx}] in {fixture_path} must be "
f"an object; got {type(entry).__name__}"
)
try:
poisoned = tuple(
cpe.PoisonedTileSpec(
tile_id=str(p["tile_id"]),
defense_layer=str(p["defense_layer"]),
)
for p in entry.get("poisoned_tiles", [])
)
false_trust = tuple(
cpe.FalseTrustEvent(
flight_id=str(e.get("flight_id", entry["flight_id"])),
tile_id=str(e["tile_id"]),
monotonic_ms=int(e["monotonic_ms"]),
defense_layer=str(e["defense_layer"]),
)
for e in entry.get("false_trust_events", [])
)
rejection_reasons = {
str(k): int(v)
for k, v in (entry.get("rejection_reasons") or {}).items()
}
flights.append(
cpe.FlightOutcome(
flight_id=str(entry["flight_id"]),
total_tile_count=int(entry["total_tile_count"]),
poisoned_tiles=poisoned,
false_trust_events=false_trust,
rejection_reasons=rejection_reasons,
)
)
except (KeyError, TypeError, ValueError) as exc:
pytest.fail(
f"NFT-SEC-01: flights[{idx}] in {fixture_path} shape invalid: {exc}"
)
return flights
e2e/tests/security/test_nft_sec_02_no_egress.py
+146
View File
@@ -0,0 +1,146 @@
"""NFT-SEC-02 — No-egress contract (AZ-437 / AC-NEW-10).
Tier-1 OR Tier-2. Over a 5-min Derkachi replay against
``e2e-net.internal: true``, ``docker network inspect e2e-net`` MUST show
zero packets from the SUT container to any non-``e2e-net`` destination.
The egress-counter snapshot pair is sourced from the SITL replay
fixture (AZ-595) since the live ``docker network inspect`` call requires
a running e2e-runner container with Docker-API access — which only
exists inside the harness, not on the developer workstation. The
scenario test therefore behaves identically to the other fixture-
consumer NFTs: skip cleanly without fixtures; parse + verdict + record
when fixtures are present.
Production dependency surfaced to AZ-595: fixture JSON shape
{
"window_label": "<str>",
"before": {"egress_packets_to_internal_net": <int>,
"egress_packets_to_other_destinations": <int>,
"udp53_egress_packets": <int>},
"after": {... same shape ...}
}
"""
from __future__ import annotations
import json
import os
from pathlib import Path
import pytest
from runner.helpers import egress_observer as eo
NFT_SEC_02_FIXTURE_ENV_VAR = "E2E_NFT_SEC_02_FIXTURE"
NFT_SEC_02_DEFAULT_FIXTURE_NAME = "nft_sec_02_no_egress.json"
@pytest.mark.scenario_id("nft-sec-02")
@pytest.mark.traces_to("AC-NEW-10,AC-1,AC-4")
def test_nft_sec_02_no_egress(
fc_adapter: str,
vio_strategy: str,
evidence_dir, # type: ignore[no-untyped-def]
run_id: str,
nfr_recorder, # type: ignore[no-untyped-def]
sitl_replay_ready: bool,
) -> None:
"""AC-1: 0 packets to non-e2e-net during the 5-min replay window."""
if not sitl_replay_ready:
pytest.skip(
"NFT-SEC-02 requires `E2E_SITL_REPLAY_DIR` to point at a "
"prepared SITL replay fixture (AZ-595) carrying the Docker "
"network-stats before/after snapshots. Pure delta-verdict "
"logic covered by "
"e2e/_unit_tests/helpers/test_egress_observer.py."
)
fixture_path = _resolve_fixture_path()
if not fixture_path.is_file():
pytest.fail(
f"NFT-SEC-02: fixture not found at {fixture_path}. "
f"`{NFT_SEC_02_FIXTURE_ENV_VAR}` env var must point at a JSON "
"file with the schema documented in the scenario docstring. "
"Production dependency: AZ-595."
)
payload = json.loads(fixture_path.read_text())
before, after, window_label = _parse_payload(payload, fixture_path)
report = eo.evaluate_no_egress(before, after, window_label=window_label)
out_csv = (
evidence_dir
/ "nft-sec-02"
/ f"{fc_adapter}-{vio_strategy}.csv"
)
eo.write_no_egress_csv_evidence(out_csv, report)
nfr_recorder.record_metric(
"nft_sec_02.egress_packets_to_other_destinations_delta",
float(report.delta_other_destinations),
ac_id="AC-1",
)
nfr_recorder.record_metric(
"nft_sec_02.egress_packets_to_internal_net_delta",
float(report.delta_internal),
ac_id="AC-1",
)
assert report.passes, (
f"AC-1: SUT container egressed {report.delta_other_destinations} "
f"packets to non-e2e-net destinations during window "
f"'{report.window_label}' (budget = 0). "
f"before={report.before.egress_packets_to_other_destinations}, "
f"after={report.after.egress_packets_to_other_destinations}"
)
def _resolve_fixture_path() -> Path:
raw = os.environ.get(NFT_SEC_02_FIXTURE_ENV_VAR, "").strip()
from runner.helpers import sitl_observer
root = sitl_observer.replay_dir()
if not raw:
if root is None:
return Path(f"<{NFT_SEC_02_FIXTURE_ENV_VAR}-unset>")
return root / NFT_SEC_02_DEFAULT_FIXTURE_NAME
path = Path(raw)
if not path.is_absolute() and root is not None:
path = root / path
return path
def _parse_payload(
payload: object, fixture_path: Path
) -> tuple[eo.EgressCounterSnapshot, eo.EgressCounterSnapshot, str]:
if not isinstance(payload, dict):
pytest.fail(
f"NFT-SEC-02: fixture {fixture_path} must be a JSON object; "
f"got top-level type={type(payload).__name__}"
)
window_label = str(payload.get("window_label", "5min-derkachi-replay"))
try:
before = _parse_snapshot(payload["before"], fixture_path, "before")
after = _parse_snapshot(payload["after"], fixture_path, "after")
except (KeyError, TypeError, ValueError) as exc:
pytest.fail(
f"NFT-SEC-02: fixture {fixture_path} snapshot shape invalid: {exc}"
)
return before, after, window_label
def _parse_snapshot(
raw: object, fixture_path: Path, label: str
) -> eo.EgressCounterSnapshot:
if not isinstance(raw, dict):
pytest.fail(
f"NFT-SEC-02: fixture {fixture_path} '{label}' must be an object"
)
return eo.EgressCounterSnapshot(
egress_packets_to_internal_net=int(raw["egress_packets_to_internal_net"]),
egress_packets_to_other_destinations=int(
raw["egress_packets_to_other_destinations"]
),
udp53_egress_packets=int(raw.get("udp53_egress_packets", 0)),
)
e2e/tests/security/test_nft_sec_03_mavlink_signing.py
+262
View File
@@ -0,0 +1,262 @@
"""NFT-SEC-03 — AP rejects unsigned / wrong-key / replayed messages (AZ-438 / AC-NEW-11).
AP-only. Three sub-cases (sent in order; the runner pauses between
each):
* (a) unsigned ``GPS_INPUT``;
* (b) signed-with-wrong-key ``GPS_INPUT``;
* (c) replayed-from-tlog signed ``GPS_INPUT`` (counter-replay attack).
For each: AP MUST emit ``BAD_SIGNATURE`` (or one of the documented
equivalent rejection STATUSTEXTs) within ≤500 ms; AP's
``GLOBAL_POSITION_INT`` must NOT update from the injected message
(``position_drift_m ≤ 1 m`` tolerance).
iNav is N/A — MSP has no signing layer; the test skips when
``fc_adapter == 'inav'`` (AC-1). vio_strategy parameterization (AC-5)
runs the AP probe under each strategy because the conftest matrix
already enforces it; the SUT's VIO is irrelevant to the AP-side
rejection but the parameterization keeps evidence symmetric across the
test matrix.
Production dependencies surfaced to AZ-595 / SUT:
* fixture JSON shape (below) is sourced from a ``ap-only`` SITL replay
with the three injection timestamps + AP STATUSTEXT capture + AP
``GLOBAL_POSITION_INT`` capture;
* AP build MUST have MAVLink 2.0 signing enabled (per FT-P-09-AP /
AZ-416 handshake); otherwise the rejection STATUSTEXT is never
emitted and every sub-case fails on AC-2 — a fail-safe outcome,
but the test will be noisy until the handshake fixture is wired.
Fixture JSON shape::
{
"injections": [
{"sub_case": "unsigned"|"wrong_key"|"replayed",
"injected_at_ms": <int>}, ...
],
"statustexts": [{"monotonic_ms": <int>, "text": <str>}, ...],
"positions": [{"monotonic_ms": <int>,
"lat_e7": <int>, "lon_e7": <int>}, ...]
}
"""
from __future__ import annotations
import json
import os
from pathlib import Path
import pytest
from runner.helpers import mavlink_signing_evaluator as mse
NFT_SEC_03_FIXTURE_ENV_VAR = "E2E_NFT_SEC_03_FIXTURE"
NFT_SEC_03_DEFAULT_FIXTURE_NAME = "nft_sec_03_mavlink_signing.json"
@pytest.mark.scenario_id("nft-sec-03")
@pytest.mark.traces_to("AC-NEW-11,AC-1,AC-2,AC-3,AC-4,AC-5")
def test_nft_sec_03_mavlink_signing(
fc_adapter: str,
vio_strategy: str,
evidence_dir, # type: ignore[no-untyped-def]
run_id: str,
nfr_recorder, # type: ignore[no-untyped-def]
sitl_replay_ready: bool,
) -> None:
"""AP rejects all three injection sub-cases within ≤500 ms; no position drift."""
if fc_adapter == "inav":
pytest.skip(
"AC-1: NFT-SEC-03 is AP-only; iNav (MSP) has no signing layer."
)
if not sitl_replay_ready:
pytest.skip(
"NFT-SEC-03 requires `E2E_SITL_REPLAY_DIR` to point at a "
"prepared SITL replay fixture (AZ-595) carrying the three "
"injection timestamps + AP STATUSTEXT + GLOBAL_POSITION_INT "
"captures. Pure rejection-logic covered by "
"e2e/_unit_tests/helpers/test_mavlink_signing_evaluator.py."
)
fixture_path = _resolve_fixture_path()
if not fixture_path.is_file():
pytest.fail(
f"NFT-SEC-03: fixture not found at {fixture_path}. "
f"`{NFT_SEC_03_FIXTURE_ENV_VAR}` env var must point at a JSON "
"file with the schema documented in the scenario docstring. "
"Production dependency: AZ-595 + FT-P-09-AP signing handshake "
"(AZ-416)."
)
payload = json.loads(fixture_path.read_text())
injections, statustexts, positions = _parse_payload(payload, fixture_path)
if len(injections) != 3:
pytest.fail(
f"NFT-SEC-03 AC-2..AC-4: fixture must contain exactly 3 "
f"injections (unsigned + wrong_key + replayed); got "
f"{len(injections)} in {fixture_path}."
)
sub_cases_seen = {inj.sub_case for inj in injections}
expected = {mse.SubCase.UNSIGNED, mse.SubCase.WRONG_KEY, mse.SubCase.REPLAYED}
if sub_cases_seen != expected:
pytest.fail(
f"NFT-SEC-03: fixture missing sub-cases {sorted(s.value for s in expected - sub_cases_seen)} "
f"in {fixture_path}."
)
report = mse.evaluate(
injections, statustexts=statustexts, positions=positions
)
out_csv = (
evidence_dir
/ "nft-sec-03"
/ f"{fc_adapter}-{vio_strategy}.csv"
)
mse.write_csv_evidence(out_csv, report)
for sub in report.sub_cases:
if sub.rejection_latency_ms is not None:
nfr_recorder.record_metric(
f"nft_sec_03.{sub.sub_case.value}.rejection_latency_ms",
float(sub.rejection_latency_ms),
ac_id=_ac_for(sub.sub_case),
)
nfr_recorder.record_metric(
f"nft_sec_03.{sub.sub_case.value}.position_drift_m",
sub.position_drift_m,
ac_id=_ac_for(sub.sub_case),
)
for sub in report.sub_cases:
ac = _ac_for(sub.sub_case)
assert sub.passes_rejection, (
f"{ac}: AP did not reject {sub.sub_case.value} GPS_INPUT within "
f"{sub.budget_ms} ms — rejection_at_ms={sub.rejection_at_ms}, "
f"rejection_text={sub.rejection_text!r}, "
f"latency_ms={sub.rejection_latency_ms}."
)
assert sub.passes_no_position_update, (
f"{ac}: AP GLOBAL_POSITION_INT drifted "
f"{sub.position_drift_m:.2f} m around injection (tolerance "
f"{mse.POSITION_DRIFT_TOLERANCE_M} m) — the rejection STATUSTEXT "
f"fired but the position update was accepted. This is a "
f"defense-bypass bug (signaling-only rejection without state "
f"enforcement)."
)
def _ac_for(sub_case: mse.SubCase) -> str:
return {
mse.SubCase.UNSIGNED: "AC-2",
mse.SubCase.WRONG_KEY: "AC-3",
mse.SubCase.REPLAYED: "AC-4",
}[sub_case]
def _resolve_fixture_path() -> Path:
raw = os.environ.get(NFT_SEC_03_FIXTURE_ENV_VAR, "").strip()
from runner.helpers import sitl_observer
root = sitl_observer.replay_dir()
if not raw:
if root is None:
return Path(f"<{NFT_SEC_03_FIXTURE_ENV_VAR}-unset>")
return root / NFT_SEC_03_DEFAULT_FIXTURE_NAME
path = Path(raw)
if not path.is_absolute() and root is not None:
path = root / path
return path
def _parse_payload(
payload: object, fixture_path: Path
) -> tuple[
list[mse.InjectionEvent],
list[mse.StatustextSample],
list[mse.PositionSample],
]:
if not isinstance(payload, dict):
pytest.fail(
f"NFT-SEC-03: fixture {fixture_path} must be a JSON object; "
f"got top-level type={type(payload).__name__}"
)
raw_inj = payload.get("injections")
if not isinstance(raw_inj, list):
pytest.fail(
f"NFT-SEC-03: fixture {fixture_path} 'injections' must be a list"
)
injections: list[mse.InjectionEvent] = []
for idx, entry in enumerate(raw_inj):
if not isinstance(entry, dict):
pytest.fail(
f"NFT-SEC-03: injections[{idx}] in {fixture_path} must be an object"
)
try:
sub_case = mse.SubCase(str(entry["sub_case"]))
except (KeyError, ValueError) as exc:
pytest.fail(
f"NFT-SEC-03: injections[{idx}] in {fixture_path} 'sub_case' "
f"must be one of {sorted(s.value for s in mse.SubCase)}; got {exc}"
)
try:
injections.append(
mse.InjectionEvent(
sub_case=sub_case,
injected_at_ms=int(entry["injected_at_ms"]),
)
)
except (KeyError, TypeError, ValueError) as exc:
pytest.fail(
f"NFT-SEC-03: injections[{idx}] in {fixture_path} shape invalid: {exc}"
)
raw_st = payload.get("statustexts", [])
if not isinstance(raw_st, list):
pytest.fail(
f"NFT-SEC-03: fixture {fixture_path} 'statustexts' must be a list"
)
statustexts: list[mse.StatustextSample] = []
for idx, entry in enumerate(raw_st):
if not isinstance(entry, dict):
pytest.fail(
f"NFT-SEC-03: statustexts[{idx}] in {fixture_path} must be an object"
)
try:
statustexts.append(
mse.StatustextSample(
monotonic_ms=int(entry["monotonic_ms"]),
text=str(entry["text"]),
)
)
except (KeyError, TypeError, ValueError) as exc:
pytest.fail(
f"NFT-SEC-03: statustexts[{idx}] in {fixture_path} shape invalid: {exc}"
)
raw_pos = payload.get("positions", [])
if not isinstance(raw_pos, list):
pytest.fail(
f"NFT-SEC-03: fixture {fixture_path} 'positions' must be a list"
)
positions: list[mse.PositionSample] = []
for idx, entry in enumerate(raw_pos):
if not isinstance(entry, dict):
pytest.fail(
f"NFT-SEC-03: positions[{idx}] in {fixture_path} must be an object"
)
try:
positions.append(
mse.PositionSample(
monotonic_ms=int(entry["monotonic_ms"]),
lat_e7=int(entry["lat_e7"]),
lon_e7=int(entry["lon_e7"]),
)
)
except (KeyError, TypeError, ValueError) as exc:
pytest.fail(
f"NFT-SEC-03: positions[{idx}] in {fixture_path} shape invalid: {exc}"
)
return injections, statustexts, positions
e2e/tests/security/test_nft_sec_04_asan_fuzz.py
+183
View File
@@ -0,0 +1,183 @@
"""NFT-SEC-04 ≥4 h ASan fuzz — release-gated (AZ-439 / RESTRICT-CVE-1 AC-2 + AC-3).
Companion to ``test_nft_sec_04_opencv_cve.py`` (the always-run probe).
This scenario consumes the captured fuzz-run summary (ASan stderr log
+ duration + corpus size) and asserts:
* AC-2: 0 ASan findings of any category;
* AC-3: ≥1000 unique JPEG corpus inputs (informational only — does NOT
contribute to ``passes`` so a fuzz with high finding count + low
corpus fails for the finding count, not the coverage proxy).
Release-gated by ``E2E_NFT_SEC_04_RELEASE_GATE=1`` because the fuzz
run takes ≥4 h. fc_adapter parameterization is irrelevant for image
decode (AC-4): only the ``ardupilot`` parameterization actually executes;
the rest skip cleanly to avoid duplicating a 4 h run.
Production dependencies surfaced:
* **AZ-444 (Tier-2 harness)**: optional. The Tier-1 path can run a
shorter fuzz against the ASan SUT image on x86; Tier-2 runs the same
fuzz on Jetson with the same SUT image.
* **AZ-595**: emit ``nft_sec_04_asan_fuzz.json`` carrying the captured
ASan stderr log lines + duration + corpus size.
Fixture JSON shape::
{
"duration_seconds": <float>,
"corpus_size": <int>,
"asan_log_lines": [<str>, <str>, ...]
}
"""
from __future__ import annotations
import json
import os
from pathlib import Path
import pytest
from runner.helpers import asan_fuzz_evaluator as afe
NFT_SEC_04_ASAN_FIXTURE_ENV_VAR = "E2E_NFT_SEC_04_ASAN_FIXTURE"
NFT_SEC_04_ASAN_DEFAULT_FIXTURE_NAME = "nft_sec_04_asan_fuzz.json"
NFT_SEC_04_RELEASE_GATE_ENV_VAR = "E2E_NFT_SEC_04_RELEASE_GATE"
@pytest.mark.scenario_id("nft-sec-04-asan-fuzz")
@pytest.mark.traces_to("RESTRICT-CVE-1,AC-2,AC-3,AC-4")
def test_nft_sec_04_asan_fuzz(
fc_adapter: str,
vio_strategy: str,
evidence_dir, # type: ignore[no-untyped-def]
run_id: str,
nfr_recorder, # type: ignore[no-untyped-def]
sitl_replay_ready: bool,
) -> None:
"""0 ASan findings across ≥4 h JPEG-fuzz; corpus ≥1000 (informational)."""
if not _release_gate_enabled():
pytest.skip(
"NFT-SEC-04 ASan-fuzz is release-gated (≥4 h run). Set "
f"`{NFT_SEC_04_RELEASE_GATE_ENV_VAR}=1` to execute. The "
"probe scenario (test_nft_sec_04_opencv_cve.py) covers "
"RESTRICT-CVE-1 AC-1 on every CI run."
)
if fc_adapter != "ardupilot":
pytest.skip(
"AC-4: NFT-SEC-04 ASan-fuzz is fc_adapter-agnostic (image "
"decode is upstream of FC); only run once per vio_strategy "
"under fc_adapter=ardupilot to avoid duplicating a 4 h run."
)
if not sitl_replay_ready:
pytest.skip(
"NFT-SEC-04 ASan-fuzz requires `E2E_SITL_REPLAY_DIR` to point "
"at a prepared SITL replay fixture (AZ-595) carrying the "
"captured fuzz-run summary. Pure ASan log classification + "
"verdict logic covered by "
"e2e/_unit_tests/helpers/test_asan_fuzz_evaluator.py."
)
fixture_path = _resolve_fixture_path()
if not fixture_path.is_file():
pytest.fail(
f"NFT-SEC-04 ASan-fuzz: fixture not found at {fixture_path}. "
f"`{NFT_SEC_04_ASAN_FIXTURE_ENV_VAR}` env var must point at a "
"JSON file with the schema documented in the scenario "
"docstring. Production dependency: AZ-595 + (optional) AZ-444."
)
payload = json.loads(fixture_path.read_text())
duration_s, corpus_size, log_lines = _parse_payload(payload, fixture_path)
report = afe.evaluate(
log_lines,
duration_seconds=duration_s,
corpus_size=corpus_size,
)
out_csv = (
evidence_dir
/ "nft-sec-04"
/ f"{fc_adapter}-{vio_strategy}-asan-fuzz.csv"
)
afe.write_csv_evidence(out_csv, report)
nfr_recorder.record_metric(
"nft_sec_04.asan_finding_count",
float(len(report.findings)),
ac_id="AC-2",
)
nfr_recorder.record_metric(
"nft_sec_04.fuzz_duration_seconds",
report.duration_seconds,
ac_id="AC-2",
)
nfr_recorder.record_metric(
"nft_sec_04.fuzz_corpus_size",
float(report.corpus_size),
ac_id="AC-3",
)
assert report.passes_duration, (
f"AC-2 pre-condition: fuzz duration {report.duration_seconds:.0f} s "
f"is below the required ≥{afe.MIN_FUZZ_DURATION_SECONDS} s — "
"the 0-finding result is not statistically meaningful for the "
"RESTRICT-CVE-1 budget without the full window."
)
assert report.passes_findings, (
f"AC-2: {len(report.findings)} ASan finding(s) recorded — "
f"see `nft-sec-04/{fc_adapter}-{vio_strategy}-asan-fuzz.csv` "
f"for per-finding categories. Any finding is a release-blocker."
)
# AC-3 is informational: emit a warning-style fail-fast message via
# the evidence CSV (already written above) but do NOT fail the test.
# The user is expected to inspect the corpus floor manually.
def _release_gate_enabled() -> bool:
return os.environ.get(NFT_SEC_04_RELEASE_GATE_ENV_VAR, "").strip().lower() in (
"1",
"true",
"yes",
)
def _resolve_fixture_path() -> Path:
raw = os.environ.get(NFT_SEC_04_ASAN_FIXTURE_ENV_VAR, "").strip()
from runner.helpers import sitl_observer
root = sitl_observer.replay_dir()
if not raw:
if root is None:
return Path(f"<{NFT_SEC_04_ASAN_FIXTURE_ENV_VAR}-unset>")
return root / NFT_SEC_04_ASAN_DEFAULT_FIXTURE_NAME
path = Path(raw)
if not path.is_absolute() and root is not None:
path = root / path
return path
def _parse_payload(
payload: object, fixture_path: Path
) -> tuple[float, int, list[str]]:
if not isinstance(payload, dict):
pytest.fail(
f"NFT-SEC-04 ASan-fuzz: fixture {fixture_path} must be a JSON "
f"object; got top-level type={type(payload).__name__}"
)
try:
duration_s = float(payload["duration_seconds"])
corpus_size = int(payload["corpus_size"])
except (KeyError, TypeError, ValueError) as exc:
pytest.fail(
f"NFT-SEC-04 ASan-fuzz: fixture {fixture_path} missing/invalid "
f"duration_seconds or corpus_size: {exc}"
)
raw_lines = payload.get("asan_log_lines", [])
if not isinstance(raw_lines, list):
pytest.fail(
f"NFT-SEC-04 ASan-fuzz: fixture {fixture_path} "
f"'asan_log_lines' must be a list (may be empty)"
)
log_lines = [str(line) for line in raw_lines]
return duration_s, corpus_size, log_lines
e2e/tests/security/test_nft_sec_04_opencv_cve.py
+173
View File
@@ -0,0 +1,173 @@
"""NFT-SEC-04 probe — OpenCV CVE-2025-53644 no-crash (AZ-439 / RESTRICT-CVE-1).
Always-runs (Tier-1 OR Tier-2). The crafted ``cve-2025-53644.jpg`` is
fed to the SUT's nav-camera as a single frame and the FDR archive is
inspected:
* AC-1a: at least one FDR record exists strictly after the probe
injection (proves the SUT process did not crash);
* AC-1b: the FDR record matched within ``±tolerance_ms`` of the probe
is one of ``decode-success`` or ``frame-decode-error`` (proves the
SUT either decoded the patched JPEG or gracefully rejected it).
The companion ≥4 h ASan fuzz lives in
``test_nft_sec_04_asan_fuzz.py`` and is release-gated.
Production dependencies surfaced:
* **AZ-595**: emit ``nft_sec_04_cve_probe.json`` carrying
``probe_injected_at_ms`` + the per-frame FDR record sequence the
runner captured;
* **SUT**: the SUT MUST honor its FDR per-frame outcome contract — a
silent drop is treated as a defense-bypass failure even when the
process does not crash.
Fixture JSON shape::
{
"probe_injected_at_ms": <int>,
"tolerance_ms": <int, optional, default 50>,
"fdr_records": [
{"monotonic_ms": <int>, "kind": <str>}, ...
]
}
"""
from __future__ import annotations
import json
import os
from pathlib import Path
import pytest
from runner.helpers import cve_probe_evaluator as cpe
NFT_SEC_04_FIXTURE_ENV_VAR = "E2E_NFT_SEC_04_FIXTURE"
NFT_SEC_04_DEFAULT_FIXTURE_NAME = "nft_sec_04_cve_probe.json"
@pytest.mark.scenario_id("nft-sec-04")
@pytest.mark.traces_to("RESTRICT-CVE-1,AC-1,AC-4")
def test_nft_sec_04_opencv_cve_probe(
fc_adapter: str,
vio_strategy: str,
evidence_dir, # type: ignore[no-untyped-def]
run_id: str,
nfr_recorder, # type: ignore[no-untyped-def]
sitl_replay_ready: bool,
) -> None:
"""SUT survives the crafted JPEG and records a deterministic outcome."""
if not sitl_replay_ready:
pytest.skip(
"NFT-SEC-04 probe requires `E2E_SITL_REPLAY_DIR` to point at a "
"prepared SITL replay fixture (AZ-595) carrying the post-probe "
"FDR record sequence. Pure no-crash / outcome-classification "
"logic covered by "
"e2e/_unit_tests/helpers/test_cve_probe_evaluator.py."
)
fixture_path = _resolve_fixture_path()
if not fixture_path.is_file():
pytest.fail(
f"NFT-SEC-04 probe: fixture not found at {fixture_path}. "
f"`{NFT_SEC_04_FIXTURE_ENV_VAR}` env var must point at a JSON "
"file with the schema documented in the scenario docstring. "
"Production dependency: AZ-595."
)
payload = json.loads(fixture_path.read_text())
probe_at_ms, tolerance_ms, fdr_records = _parse_payload(payload, fixture_path)
report = cpe.evaluate(
fdr_records,
probe_injected_at_ms=probe_at_ms,
tolerance_ms=tolerance_ms,
)
out_csv = (
evidence_dir
/ "nft-sec-04"
/ f"{fc_adapter}-{vio_strategy}-probe.csv"
)
cpe.write_csv_evidence(out_csv, report)
nfr_recorder.record_metric(
"nft_sec_04.probe_outcome_is_decode_success",
1.0 if report.probe_outcome is cpe.ProbeFrameOutcome.DECODE_SUCCESS else 0.0,
ac_id="AC-1",
)
nfr_recorder.record_metric(
"nft_sec_04.probe_outcome_is_graceful_error",
1.0 if report.probe_outcome is cpe.ProbeFrameOutcome.FRAME_DECODE_ERROR else 0.0,
ac_id="AC-1",
)
assert report.passes_no_crash, (
f"AC-1a: SUT did not produce any FDR record after probe injection "
f"at {report.probe_injected_at_ms} ms — process likely crashed. "
f"last_fdr_record_at_ms={report.last_fdr_record_at_ms}."
)
assert report.passes_graceful_outcome, (
f"AC-1b: SUT silently dropped the probe frame (no decode-success "
f"or frame-decode-error in FDR within ±{tolerance_ms} ms of "
f"probe injection at {report.probe_injected_at_ms} ms). Silent "
f"drops are a defense-bypass failure even if the process did not "
f"crash."
)
def _resolve_fixture_path() -> Path:
raw = os.environ.get(NFT_SEC_04_FIXTURE_ENV_VAR, "").strip()
from runner.helpers import sitl_observer
root = sitl_observer.replay_dir()
if not raw:
if root is None:
return Path(f"<{NFT_SEC_04_FIXTURE_ENV_VAR}-unset>")
return root / NFT_SEC_04_DEFAULT_FIXTURE_NAME
path = Path(raw)
if not path.is_absolute() and root is not None:
path = root / path
return path
def _parse_payload(
payload: object, fixture_path: Path
) -> tuple[int, int, list[cpe.FdrSurvivalRecord]]:
if not isinstance(payload, dict):
pytest.fail(
f"NFT-SEC-04 probe: fixture {fixture_path} must be a JSON object; "
f"got top-level type={type(payload).__name__}"
)
try:
probe_at = int(payload["probe_injected_at_ms"])
tolerance = int(payload.get("tolerance_ms", 50))
except (KeyError, TypeError, ValueError) as exc:
pytest.fail(
f"NFT-SEC-04 probe: fixture {fixture_path} missing/invalid "
f"probe_injected_at_ms or tolerance_ms: {exc}"
)
raw_records = payload.get("fdr_records")
if not isinstance(raw_records, list):
pytest.fail(
f"NFT-SEC-04 probe: fixture {fixture_path} 'fdr_records' must be a list"
)
records: list[cpe.FdrSurvivalRecord] = []
for idx, entry in enumerate(raw_records):
if not isinstance(entry, dict):
pytest.fail(
f"NFT-SEC-04 probe: fdr_records[{idx}] in {fixture_path} "
f"must be an object"
)
try:
records.append(
cpe.FdrSurvivalRecord(
monotonic_ms=int(entry["monotonic_ms"]),
kind=str(entry["kind"]),
)
)
except (KeyError, TypeError, ValueError) as exc:
pytest.fail(
f"NFT-SEC-04 probe: fdr_records[{idx}] in {fixture_path} "
f"shape invalid: {exc}"
)
return probe_at, tolerance, records
e2e/tests/security/test_nft_sec_05_dns_blackhole.py
+170
View File
@@ -0,0 +1,170 @@
"""NFT-SEC-05 — DNS-blackhole defense-in-depth (AZ-437 / AC-NEW-10, residual-risk #1).
Tier-1 OR Tier-2. Even if ``e2e-net.internal: true`` is misconfigured,
the DNS-blackhole sidecar MUST prevent DNS-based exfiltration. The
runner executes a ``nslookup`` inside the SUT container's network
namespace and asserts:
* AC-2: the sidecar's health endpoint returns healthy;
* AC-3a: the lookup *fails* (NXDOMAIN, timeout, "no servers can be
reached", or any other documented failure outcome);
* AC-3b: no UDP-53 packets cross the host's outbound interface during
the probe.
The combined verdict object is sourced from the SITL replay fixture
(AZ-595) for the same reason NFT-SEC-02 is fixture-sourced: the live
``docker exec`` + host-interface-counter pipeline only exists inside the
harness.
Production dependency surfaced to AZ-595: fixture JSON shape
{
"sidecar_healthy": <bool>,
"lookup_outcome": "nxdomain" | "timeout" | "no_servers_can_be_reached"
| "other_failure" | "success",
"before": {... egress snapshot ...},
"after": {... egress snapshot ...}
}
"""
from __future__ import annotations
import json
import os
from pathlib import Path
import pytest
from runner.helpers import egress_observer as eo
NFT_SEC_05_FIXTURE_ENV_VAR = "E2E_NFT_SEC_05_FIXTURE"
NFT_SEC_05_DEFAULT_FIXTURE_NAME = "nft_sec_05_dns_blackhole.json"
@pytest.mark.scenario_id("nft-sec-05")
@pytest.mark.traces_to("AC-NEW-10,AC-2,AC-3,AC-4")
def test_nft_sec_05_dns_blackhole(
fc_adapter: str,
vio_strategy: str,
evidence_dir, # type: ignore[no-untyped-def]
run_id: str,
nfr_recorder, # type: ignore[no-untyped-def]
sitl_replay_ready: bool,
) -> None:
"""Sidecar healthy + lookup fails + UDP-53 silent."""
if not sitl_replay_ready:
pytest.skip(
"NFT-SEC-05 requires `E2E_SITL_REPLAY_DIR` to point at a "
"prepared SITL replay fixture (AZ-595) carrying the DNS-probe "
"outcome + UDP-53 counter snapshots. Pure verdict logic "
"covered by e2e/_unit_tests/helpers/test_egress_observer.py."
)
fixture_path = _resolve_fixture_path()
if not fixture_path.is_file():
pytest.fail(
f"NFT-SEC-05: fixture not found at {fixture_path}. "
f"`{NFT_SEC_05_FIXTURE_ENV_VAR}` env var must point at a JSON "
"file with the schema documented in the scenario docstring. "
"Production dependency: AZ-595."
)
payload = json.loads(fixture_path.read_text())
before, after, lookup_outcome, sidecar_healthy = _parse_payload(
payload, fixture_path
)
report = eo.evaluate_dns_blackhole(
before,
after,
lookup_outcome=lookup_outcome,
sidecar_healthy=sidecar_healthy,
)
out_csv = (
evidence_dir
/ "nft-sec-05"
/ f"{fc_adapter}-{vio_strategy}.csv"
)
eo.write_dns_blackhole_csv_evidence(out_csv, report)
nfr_recorder.record_metric(
"nft_sec_05.udp53_egress_delta",
float(report.delta_udp53),
ac_id="AC-3",
)
assert report.sidecar_healthy, (
"AC-2: DNS blackhole sidecar reported unhealthy — defense-in-depth "
"is unavailable; SUT egress isolation is the only layer protecting "
"data residency."
)
assert report.passes_lookup, (
f"AC-3a: nslookup outcome = {report.lookup_outcome.value} — DNS "
"resolution must FAIL inside the SUT container (NXDOMAIN, timeout, "
"no-servers, or other-failure). SUCCESS means an exfiltration "
"path exists."
)
assert report.passes_udp_silence, (
f"AC-3b: UDP-53 egress delta = {report.delta_udp53} packets "
"during probe (budget = 0). Even a single packet leaving the "
"host means the DNS-blackhole sidecar failed to absorb the probe."
)
def _resolve_fixture_path() -> Path:
raw = os.environ.get(NFT_SEC_05_FIXTURE_ENV_VAR, "").strip()
from runner.helpers import sitl_observer
root = sitl_observer.replay_dir()
if not raw:
if root is None:
return Path(f"<{NFT_SEC_05_FIXTURE_ENV_VAR}-unset>")
return root / NFT_SEC_05_DEFAULT_FIXTURE_NAME
path = Path(raw)
if not path.is_absolute() and root is not None:
path = root / path
return path
def _parse_payload(
payload: object, fixture_path: Path
) -> tuple[eo.EgressCounterSnapshot, eo.EgressCounterSnapshot, eo.DnsLookupOutcome, bool]:
if not isinstance(payload, dict):
pytest.fail(
f"NFT-SEC-05: fixture {fixture_path} must be a JSON object; "
f"got top-level type={type(payload).__name__}"
)
try:
sidecar_healthy = bool(payload["sidecar_healthy"])
outcome_raw = str(payload["lookup_outcome"])
try:
lookup_outcome = eo.DnsLookupOutcome(outcome_raw)
except ValueError as exc:
pytest.fail(
f"NFT-SEC-05: fixture {fixture_path} 'lookup_outcome' must "
f"be one of "
f"{sorted(o.value for o in eo.DnsLookupOutcome)}; got "
f"{outcome_raw!r} ({exc})"
)
before = _parse_snapshot(payload["before"], fixture_path, "before")
after = _parse_snapshot(payload["after"], fixture_path, "after")
except (KeyError, TypeError, ValueError) as exc:
pytest.fail(
f"NFT-SEC-05: fixture {fixture_path} shape invalid: {exc}"
)
return before, after, lookup_outcome, sidecar_healthy
def _parse_snapshot(
raw: object, fixture_path: Path, label: str
) -> eo.EgressCounterSnapshot:
if not isinstance(raw, dict):
pytest.fail(
f"NFT-SEC-05: fixture {fixture_path} '{label}' must be an object"
)
return eo.EgressCounterSnapshot(
egress_packets_to_internal_net=int(raw["egress_packets_to_internal_net"]),
egress_packets_to_other_destinations=int(
raw["egress_packets_to_other_destinations"]
),
udp53_egress_packets=int(raw["udp53_egress_packets"]),
)