azaion/gps-denied-onboard
1
0
Fork 0
mirror of https://github.com/azaion/gps-denied-onboard.git synced 2026-06-22 19:11:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

[AZ-595] Batch 75: sitl_observer FDR-replay + scenario probe cleanup

Browse Source 
Implement all 11 `sitl_observer` public surfaces as an offline
FDR-replay strategy (reads JSON fixtures under `${E2E_SITL_REPLAY_DIR}`
instead of live pymavlink/yamspy). Replace 12 per-scenario
`_harness_helpers_implemented` probes with one shared session-scoped
`sitl_replay_ready` fixture in `e2e/tests/conftest.py`.

Net: -636 LoC of duplicated scenario gating, +17 LoC shared fixture,
+38 new unit tests (596 total, up from 558). Includes K=3 cumulative
review for batches 73-75 (PASS).

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Oleksandr Bezdieniezhnykh
2026-05-17 09:00:55 +03:00
parent 1d260f7e41
commit 43fdef1aac
23 changed files with 1485 additions and 639 deletions
Download Patch File Download Diff File Expand all files Collapse all files
e2e/_unit_tests/helpers/test_sitl_observer.py
+431
View File
@@ -0,0 +1,431 @@
"""Unit tests for `e2e/runner/helpers/sitl_observer.py` (AZ-595)."""
from __future__ import annotations
import json
import os
from pathlib import Path
import pytest
from e2e.runner.helpers import sitl_observer as so
@pytest.fixture
def replay_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
"""Sets `${E2E_SITL_REPLAY_DIR}` to a tmp dir for the duration of the test."""
monkeypatch.setenv("E2E_SITL_REPLAY_DIR", str(tmp_path))
return tmp_path
@pytest.fixture
def unset_replay_dir(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("E2E_SITL_REPLAY_DIR", raising=False)
def _write_json(path: Path, content) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(content))
# replay_dir / replay_dir_available
def test_replay_dir_unset_returns_none(unset_replay_dir):
# Assert
assert so.replay_dir() is None
assert so.replay_dir_available() is False
def test_replay_dir_set_but_missing_returns_false(monkeypatch: pytest.MonkeyPatch, tmp_path: Path):
# Arrange
monkeypatch.setenv("E2E_SITL_REPLAY_DIR", str(tmp_path / "nope"))
# Assert
assert so.replay_dir_available() is False
def test_replay_dir_set_and_exists_returns_true(replay_dir: Path):
# Assert
assert so.replay_dir_available() is True
def test_replay_dir_whitespace_env_treated_as_unset(monkeypatch: pytest.MonkeyPatch):
# Arrange
monkeypatch.setenv("E2E_SITL_REPLAY_DIR", " ")
# Assert
assert so.replay_dir() is None
# read_ekf_divergence_events
def test_read_ekf_events_empty_without_env(unset_replay_dir):
# Assert
assert so.read_ekf_divergence_events() == []
def test_read_ekf_events_empty_when_file_missing(replay_dir: Path):
# Assert
assert so.read_ekf_divergence_events() == []
def test_read_ekf_events_parses_records(replay_dir: Path):
# Arrange
_write_json(
replay_dir / "ekf_divergence_events.json",
[
{"monotonic_ms": 1000, "severity": "WARNING", "message": "EKF_X drift"},
{"monotonic_ms": 2000, "severity": "CRITICAL", "message": "EKF_X reset"},
],
)
# Act
events = so.read_ekf_divergence_events()
# Assert
assert len(events) == 2
assert events[0] == so.EkfDivergenceEvent(
monotonic_ms=1000, severity="WARNING", message="EKF_X drift"
)
def test_read_ekf_events_malformed_raises(replay_dir: Path):
# Arrange
_write_json(replay_dir / "ekf_divergence_events.json", [{"monotonic_ms": "bad"}])
# Act / Assert
with pytest.raises(RuntimeError, match="EKF divergence fixture malformed"):
so.read_ekf_divergence_events()
def test_read_ekf_events_wrong_top_level_type_raises(replay_dir: Path):
# Arrange
_write_json(replay_dir / "ekf_divergence_events.json", {"not": "a list"})
# Act / Assert
with pytest.raises(RuntimeError, match="must be a JSON list"):
so.read_ekf_divergence_events()
# read_gps_health_samples
def test_read_gps_health_parses(replay_dir: Path):
# Arrange
_write_json(
replay_dir / "gps_health_samples.json",
[
{"monotonic_ms": 0, "healthy": True, "spoofed": False},
{"monotonic_ms": 1000, "healthy": False, "spoofed": True},
],
)
# Act
samples = so.read_gps_health_samples()
# Assert
assert len(samples) == 2
assert samples[1].spoofed is True
def test_read_gps_health_empty_without_env(unset_replay_dir):
# Assert
assert so.read_gps_health_samples() == []
# read_consistency_check_events
def test_read_consistency_check_parses(replay_dir: Path):
# Arrange
_write_json(
replay_dir / "consistency_check_events.json",
[{"monotonic_ms": 5000, "passed": True}],
)
# Act
events = so.read_consistency_check_events()
# Assert
assert events == [so.ConsistencyCheckEvent(monotonic_ms=5000, passed=True)]
def test_read_consistency_check_empty_without_env(unset_replay_dir):
# Assert
assert so.read_consistency_check_events() == []
# get_observer
def test_get_observer_missing_env_raises(unset_replay_dir):
# Assert
with pytest.raises(RuntimeError, match="env var not set"):
so.get_observer("ardupilot", "sitl-host")
def test_get_observer_missing_fixture_raises(replay_dir: Path):
# Assert
with pytest.raises(RuntimeError, match="required fixture not found"):
so.get_observer("ardupilot", "sitl-host")
def test_get_observer_read_gps_state(replay_dir: Path):
# Arrange
_write_json(
replay_dir / "observer_ardupilot_sitl-host.json",
{
"gps_state": {
"primary_source": "MAV",
"last_position_lat_deg": 50.0,
"last_position_lon_deg": 30.0,
"last_position_alt_m": 250.0,
"fix_quality": 3,
"horizontal_accuracy_m": 1.5,
"last_update_age_ms": 100,
},
"parameters": {"EK3_SRC1_POSXY": 3},
},
)
# Act
obs = so.get_observer("ardupilot", "sitl-host")
gps = obs.read_gps_state()
# Assert
assert gps.primary_source == "MAV"
assert gps.fix_quality == 3
assert obs.read_parameter("EK3_SRC1_POSXY") == 3
assert obs.read_parameter("MISSING") is None
def test_get_observer_missing_gps_state_raises(replay_dir: Path):
# Arrange
_write_json(replay_dir / "observer_inav_h.json", {"parameters": {}})
# Act / Assert
obs = so.get_observer("inav", "h")
with pytest.raises(RuntimeError, match="fixture missing `gps_state`"):
obs.read_gps_state()
# prepare_sitl_*
def test_prepare_sitl_cold_boot_no_op(tmp_path: Path):
# Act — no env var set is fine for the no-op.
so.prepare_sitl_cold_boot(host="ardupilot-sitl", fixture_path=tmp_path / "cb.json")
def test_prepare_sitl_cold_boot_empty_host_raises(tmp_path: Path):
# Assert
with pytest.raises(RuntimeError, match="host must be non-empty"):
so.prepare_sitl_cold_boot(host="", fixture_path=tmp_path / "cb.json")
def test_prepare_sitl_cold_boot_none_fixture_path_raises():
# Assert
with pytest.raises(RuntimeError, match="fixture_path is required"):
so.prepare_sitl_cold_boot(host="ardupilot-sitl", fixture_path=None) # type: ignore[arg-type]
def test_prepare_sitl_no_gps_no_op():
# Act
so.prepare_sitl_no_gps(host="ardupilot-sitl")
def test_prepare_sitl_no_gps_empty_host_raises():
# Assert
with pytest.raises(RuntimeError, match="host must be non-empty"):
so.prepare_sitl_no_gps(host="")
# capture_ap_tlog
def test_capture_ap_tlog_missing_env_raises(unset_replay_dir):
# Assert
with pytest.raises(RuntimeError, match="env var not set"):
so.capture_ap_tlog(host="ardupilot-sitl", duration_s=1.0)
def test_capture_ap_tlog_missing_file_raises(replay_dir: Path):
# Assert
with pytest.raises(RuntimeError, match="fixture not found"):
so.capture_ap_tlog(host="ardupilot-sitl", duration_s=1.0)
def test_capture_ap_tlog_returns_path(replay_dir: Path):
# Arrange
tlog = replay_dir / "ap_tlog_ardupilot-sitl.tlog"
tlog.write_bytes(b"\x00\x01\x02")
# Act
out = so.capture_ap_tlog(host="ardupilot-sitl", duration_s=1.0)
# Assert
assert out == tlog
def test_capture_ap_tlog_zero_duration_raises():
# Assert
with pytest.raises(RuntimeError, match="duration_s must be positive"):
so.capture_ap_tlog(host="x", duration_s=0)
# read_ap_parameter
def test_read_ap_parameter_returns_value(replay_dir: Path):
# Arrange
_write_json(
replay_dir / "ap_parameters_ardupilot-sitl.json",
{"EK3_SRC1_POSXY": 3, "GPS_TYPE": 14},
)
# Act + Assert
assert so.read_ap_parameter(host="ardupilot-sitl", name="EK3_SRC1_POSXY") == 3
assert so.read_ap_parameter(host="ardupilot-sitl", name="UNKNOWN") is None
def test_read_ap_parameter_missing_file_raises(replay_dir: Path):
# Assert
with pytest.raises(RuntimeError, match="required fixture not found"):
so.read_ap_parameter(host="ardupilot-sitl", name="ANY")
# observe_inav_tcp_handshake
def test_observe_inav_tcp_handshake_returns_record(replay_dir: Path):
# Arrange
_write_json(
replay_dir / "inav_handshake_inav-sitl_5760.json",
{"established_within_s": 2.1},
)
# Act
report = so.observe_inav_tcp_handshake(host="inav-sitl", port=5760, timeout_s=5.0)
# Assert
assert report.established_within_s == pytest.approx(2.1)
def test_observe_inav_tcp_handshake_null_established(replay_dir: Path):
# Arrange — handshake did NOT establish within window.
_write_json(
replay_dir / "inav_handshake_inav-sitl_5760.json",
{"established_within_s": None},
)
# Act
report = so.observe_inav_tcp_handshake(host="inav-sitl", port=5760, timeout_s=5.0)
# Assert
assert report.established_within_s is None
def test_observe_inav_tcp_handshake_zero_timeout_raises():
# Assert
with pytest.raises(RuntimeError, match="timeout_s must be positive"):
so.observe_inav_tcp_handshake(host="x", port=1, timeout_s=0)
def test_observe_inav_tcp_handshake_bad_value_type_raises(replay_dir: Path):
# Arrange
_write_json(
replay_dir / "inav_handshake_inav-sitl_5760.json",
{"established_within_s": "not-a-number"},
)
# Act / Assert
with pytest.raises(RuntimeError, match="must be a number or null"):
so.observe_inav_tcp_handshake(host="inav-sitl", port=5760, timeout_s=5.0)
# collect_inav_msp_frames
def test_collect_inav_msp_frames_round_trip(replay_dir: Path):
# Arrange
_write_json(
replay_dir / "inav_msp_frames_inav-sitl_5760.json",
{
"frames": [
{"monotonic_ms": 0, "function_id": 0x1F03},
{"monotonic_ms": 200, "function_id": 0x1F03},
],
"expected_num_sat": 12,
},
)
# Act
capture = so.collect_inav_msp_frames(host="inav-sitl", port=5760, window_s=60.0)
# Assert
assert capture.expected_num_sat == 12
assert len(capture.frames) == 2
assert capture.frames[1].function_id == 0x1F03
def test_collect_inav_msp_frames_missing_expected_num_sat_raises(replay_dir: Path):
# Arrange
_write_json(
replay_dir / "inav_msp_frames_inav-sitl_5760.json",
{"frames": []},
)
# Act / Assert
with pytest.raises(RuntimeError, match="`expected_num_sat` must be an int"):
so.collect_inav_msp_frames(host="inav-sitl", port=5760, window_s=60.0)
def test_collect_inav_msp_frames_malformed_frame_raises(replay_dir: Path):
# Arrange
_write_json(
replay_dir / "inav_msp_frames_inav-sitl_5760.json",
{"frames": [{"monotonic_ms": "bad"}], "expected_num_sat": 12},
)
# Act / Assert
with pytest.raises(RuntimeError, match="malformed frame"):
so.collect_inav_msp_frames(host="inav-sitl", port=5760, window_s=60.0)
# query_inav_gps_state
def test_query_inav_gps_state_round_trip(replay_dir: Path):
# Arrange
_write_json(
replay_dir / "inav_gps_state_inav-sitl.json",
{"fix_type": 3, "num_sat": 14, "provider": "MSP"},
)
# Act
state = so.query_inav_gps_state(host="inav-sitl")
# Assert
assert state.fix_type == 3
assert state.num_sat == 14
assert state.provider == "MSP"
def test_query_inav_gps_state_missing_field_raises(replay_dir: Path):
# Arrange
_write_json(
replay_dir / "inav_gps_state_inav-sitl.json",
{"fix_type": 3, "num_sat": 14},
)
# Act / Assert
with pytest.raises(RuntimeError, match="iNav GPS state fixture"):
so.query_inav_gps_state(host="inav-sitl")
def test_query_inav_gps_state_missing_env_raises(unset_replay_dir):
# Assert
with pytest.raises(RuntimeError, match="env var not set"):
so.query_inav_gps_state(host="inav-sitl")
e2e/runner/helpers/sitl_observer.py
+383 -27
View File
@@ -1,59 +1,415 @@
"""ArduPilot Plane / iNav SITL state-read observers.
"""ArduPilot Plane / iNav SITL state-read observers (AZ-595 FDR-replay strategy).
Reads what the SUT delivered to the FC over its external-positioning
interface, without ever bypassing the FC's own acceptance path. This is
the only legal way for blackbox tests to assert AC-4.3 (FC output contract):
every assertion goes through the SITL's state machine.
All 11 public surfaces are backed by JSON files under
``${E2E_SITL_REPLAY_DIR}/`` — there is no live pymavlink / yamspy / TCP
connection in this implementation. This intentionally decouples scenario
execution from live SITL infrastructure: tests can run deterministically
against runner-produced fixture files, and a future "live" strategy can
plug in behind the same surface without changing any scenario code.
Public surface only; concrete pymavlink / yamspy / msp_gps_toy subprocess
plumbing is owned by AZ-416 (FT-P-09-AP) and AZ-417 (FT-P-09-iNav).
When ``E2E_SITL_REPLAY_DIR`` is unset OR the corresponding fixture file
is missing:
* `read_*` surfaces return an **empty list** (vacuous). Scenarios use the
module-level ``replay_dir_available()`` probe to detect this and skip.
* `prepare_sitl_*` surfaces are no-ops (FDR-replay does not need to
actually configure SITL state — the fixture file IS the prepared state).
* `capture_ap_tlog` / `read_ap_parameter` / `query_inav_gps_state` /
`observe_inav_tcp_handshake` / `collect_inav_msp_frames` raise
``RuntimeError`` because they require non-empty fixture data to produce
a meaningful result.
Fixture file naming (under `${E2E_SITL_REPLAY_DIR}/`):
* `ekf_divergence_events.json` — list[{monotonic_ms, severity, message}]
* `gps_health_samples.json` — list[{monotonic_ms, healthy, spoofed}]
* `consistency_check_events.json` — list[{monotonic_ms, passed}]
* `observer_<fc_kind>_<host>.json` — {gps_state: {...}, parameters: {...}}
* `ap_parameters_<host>.json` — {<param_name>: <value>, ...}
* `ap_tlog_<host>.tlog` — raw mavproxy tlog (any binary content)
* `inav_handshake_<host>.json` — {established_within_s: float | None}
* `inav_msp_frames_<host>.json` — {frames: [...], expected_num_sat: int}
* `inav_gps_state_<host>.json` — {fix_type, num_sat, provider}
Public-boundary discipline: this module does NOT import any
``src/gps_denied_onboard`` symbol.
"""
from __future__ import annotations
import json
import os
from dataclasses import dataclass
from typing import Literal, Protocol
from pathlib import Path
from typing import Iterable, Literal
_ENV_VAR = "E2E_SITL_REPLAY_DIR"
FcKind = Literal["ardupilot", "inav"]
# Dataclasses
@dataclass(frozen=True)
class FcGpsState:
"""The subset of FC state the e2e tests assert against.
"""The subset of FC state the e2e tests assert against."""
AP: assembled from EKF source-set + GLOBAL_POSITION_INT replay-back.
iNav: assembled from MSP2 GPS-provider state + getRawGPS query.
"""
primary_source: str # "MAV" (AP gps_type=14) or "MSP" (iNav)
primary_source: str
last_position_lat_deg: float
last_position_lon_deg: float
last_position_alt_m: float
fix_quality: int # 0..6 per NMEA convention
fix_quality: int
horizontal_accuracy_m: float
last_update_age_ms: int
class FcSitlObserver(Protocol):
"""Common observer protocol — implemented by `ArduPilotObserver` + `InavObserver`."""
@dataclass(frozen=True)
class EkfDivergenceEvent:
monotonic_ms: int
severity: str
message: str
@dataclass(frozen=True)
class GpsHealthSample:
monotonic_ms: int
healthy: bool
spoofed: bool
@dataclass(frozen=True)
class ConsistencyCheckEvent:
monotonic_ms: int
passed: bool
@dataclass(frozen=True)
class TcpHandshakeReport:
"""Result of an iNav SITL TCP handshake observation."""
established_within_s: float | None
@dataclass(frozen=True)
class MspFrameSample:
monotonic_ms: int
function_id: int
@dataclass(frozen=True)
class MspFrameCapture:
"""One window of MSP frame samples from the iNav SITL."""
frames: list[MspFrameSample]
expected_num_sat: int
@dataclass(frozen=True)
class InavGpsState:
fix_type: int
num_sat: int
provider: str
# Observer interface (returned by ``get_observer``)
@dataclass(frozen=True)
class _FdrReplayObserver:
"""FDR-replay observer — reads gps_state + parameters from one JSON file."""
fc_kind: FcKind
host: str
_payload: dict
def read_gps_state(self) -> FcGpsState:
...
gps = self._payload.get("gps_state")
if not isinstance(gps, dict):
raise RuntimeError(
f"sitl_observer ({self.fc_kind}/{self.host}): fixture missing `gps_state` object"
)
return FcGpsState(
primary_source=str(gps["primary_source"]),
last_position_lat_deg=float(gps["last_position_lat_deg"]),
last_position_lon_deg=float(gps["last_position_lon_deg"]),
last_position_alt_m=float(gps["last_position_alt_m"]),
fix_quality=int(gps["fix_quality"]),
horizontal_accuracy_m=float(gps["horizontal_accuracy_m"]),
last_update_age_ms=int(gps["last_update_age_ms"]),
)
def read_parameter(self, name: str) -> float | int | str | None:
...
params = self._payload.get("parameters", {})
if not isinstance(params, dict):
raise RuntimeError(
f"sitl_observer ({self.fc_kind}/{self.host}): fixture `parameters` must be an object"
)
return params.get(name)
def get_observer(fc_kind: FcKind, host: str) -> FcSitlObserver:
"""Factory — returns the matching observer for the requested FC.
# Module-level helpers
AZ-416/417 own the concrete return types. AZ-406 raises until those
tasks land so test authors can plumb the observer through their
fixtures without yet running them.
def replay_dir() -> Path | None:
"""Resolve the FDR-replay fixture root from the env var, or None if unset."""
raw = os.environ.get(_ENV_VAR, "").strip()
if not raw:
return None
return Path(raw)
def replay_dir_available() -> bool:
"""True iff ``E2E_SITL_REPLAY_DIR`` is set AND points to an existing directory."""
root = replay_dir()
return root is not None and root.is_dir()
def _load_optional_json_list(filename: str, parser) -> list:
"""Load `${E2E_SITL_REPLAY_DIR}/<filename>`; return [] when absent."""
root = replay_dir()
if root is None:
return []
path = root / filename
if not path.exists():
return []
decoded = json.loads(path.read_text())
if not isinstance(decoded, list):
raise RuntimeError(
f"sitl_observer fixture {path} must be a JSON list; got {type(decoded).__name__}"
)
return [parser(item, path) for item in decoded]
def _load_required_json(filename: str) -> tuple[dict, Path]:
"""Load `${E2E_SITL_REPLAY_DIR}/<filename>`; raise RuntimeError when absent."""
root = replay_dir()
if root is None:
raise RuntimeError(
f"sitl_observer: {_ENV_VAR} env var not set; cannot read fixture {filename}"
)
path = root / filename
if not path.exists():
raise RuntimeError(
f"sitl_observer: required fixture not found: {path}"
)
decoded = json.loads(path.read_text())
if not isinstance(decoded, dict):
raise RuntimeError(
f"sitl_observer fixture {path} must be a JSON object; got {type(decoded).__name__}"
)
return decoded, path
# get_observer factory
def get_observer(fc_kind: FcKind, host: str) -> _FdrReplayObserver:
"""Return an FDR-replay observer bound to a fixture file.
Fixture path: ``${E2E_SITL_REPLAY_DIR}/observer_<fc_kind>_<host>.json``.
Raises ``RuntimeError`` if the env var is unset or the fixture is missing.
"""
raise NotImplementedError(
f"sitl_observer.get_observer({fc_kind=}, {host=}) is owned by "
"AZ-416 (AP) / AZ-417 (iNav) — AZ-406 supplies only the contract."
payload, _ = _load_required_json(f"observer_{fc_kind}_{host}.json")
return _FdrReplayObserver(fc_kind=fc_kind, host=host, _payload=payload)
# read_* surfaces (return [] when fixtures absent)
def _parse_ekf_event(item: dict, source: Path) -> EkfDivergenceEvent:
try:
return EkfDivergenceEvent(
monotonic_ms=int(item["monotonic_ms"]),
severity=str(item["severity"]),
message=str(item["message"]),
)
except (KeyError, TypeError, ValueError) as exc:
raise RuntimeError(
f"sitl_observer EKF divergence fixture malformed at {source}: {exc}"
) from exc
def read_ekf_divergence_events() -> list[EkfDivergenceEvent]:
"""Return EKF divergence events. Empty list when fixture absent."""
return _load_optional_json_list("ekf_divergence_events.json", _parse_ekf_event)
def _parse_gps_health(item: dict, source: Path) -> GpsHealthSample:
try:
return GpsHealthSample(
monotonic_ms=int(item["monotonic_ms"]),
healthy=bool(item["healthy"]),
spoofed=bool(item["spoofed"]),
)
except (KeyError, TypeError, ValueError) as exc:
raise RuntimeError(
f"sitl_observer GPS health fixture malformed at {source}: {exc}"
) from exc
def read_gps_health_samples() -> list[GpsHealthSample]:
"""Return FC-side GPS health samples. Empty list when fixture absent."""
return _load_optional_json_list("gps_health_samples.json", _parse_gps_health)
def _parse_consistency_event(item: dict, source: Path) -> ConsistencyCheckEvent:
try:
return ConsistencyCheckEvent(
monotonic_ms=int(item["monotonic_ms"]),
passed=bool(item["passed"]),
)
except (KeyError, TypeError, ValueError) as exc:
raise RuntimeError(
f"sitl_observer consistency-check fixture malformed at {source}: {exc}"
) from exc
def read_consistency_check_events() -> list[ConsistencyCheckEvent]:
"""Return visual/satellite consistency-check events. Empty list when fixture absent."""
return _load_optional_json_list(
"consistency_check_events.json", _parse_consistency_event
)
# prepare_sitl_* — no-ops under FDR-replay
def prepare_sitl_cold_boot(host: str, fixture_path: Path) -> None:
"""No-op under FDR-replay: the cold-boot state IS the fixture file.
Raises ``RuntimeError`` if either ``host`` or ``fixture_path`` is empty —
these are required for the future live-SITL implementation and surfacing
the missing input early avoids confusing downstream errors.
"""
if not host:
raise RuntimeError("prepare_sitl_cold_boot: host must be non-empty")
if fixture_path is None:
raise RuntimeError("prepare_sitl_cold_boot: fixture_path is required")
def prepare_sitl_no_gps(host: str) -> None:
"""No-op under FDR-replay (the "no GPS" condition is encoded in the fixture)."""
if not host:
raise RuntimeError("prepare_sitl_no_gps: host must be non-empty")
# capture_ap_tlog — returns synthetic tlog path
def capture_ap_tlog(host: str, duration_s: float) -> Path:
"""Return the path to the AP mavproxy tlog fixture for ``host``.
Fixture: ``${E2E_SITL_REPLAY_DIR}/ap_tlog_<host>.tlog``.
Raises ``RuntimeError`` if env var unset or fixture missing.
``duration_s`` is recorded for future live-mode use but ignored here.
"""
if duration_s <= 0:
raise RuntimeError(f"capture_ap_tlog: duration_s must be positive; got {duration_s}")
root = replay_dir()
if root is None:
raise RuntimeError(
f"capture_ap_tlog: {_ENV_VAR} env var not set"
)
path = root / f"ap_tlog_{host}.tlog"
if not path.exists():
raise RuntimeError(
f"capture_ap_tlog: fixture not found at {path}"
)
return path
# read_ap_parameter — reads from param-dump JSON
def read_ap_parameter(host: str, name: str) -> float | int | str | None:
"""Read AP parameter ``name`` from the per-host param dump.
Fixture: ``${E2E_SITL_REPLAY_DIR}/ap_parameters_<host>.json`` ({name: value}).
Raises ``RuntimeError`` if env var unset or fixture missing.
Returns ``None`` if the parameter is not in the dump.
"""
payload, _ = _load_required_json(f"ap_parameters_{host}.json")
return payload.get(name)
# iNav surfaces
def observe_inav_tcp_handshake(host: str, port: int, timeout_s: float) -> TcpHandshakeReport:
"""Return the recorded TCP handshake outcome for ``(host, port)``.
Fixture: ``${E2E_SITL_REPLAY_DIR}/inav_handshake_<host>_<port>.json``.
Raises ``RuntimeError`` on missing fixture. ``timeout_s`` is recorded
for future live-mode use but ignored here.
"""
if timeout_s <= 0:
raise RuntimeError(
f"observe_inav_tcp_handshake: timeout_s must be positive; got {timeout_s}"
)
payload, path = _load_required_json(f"inav_handshake_{host}_{port}.json")
raw = payload.get("established_within_s")
if raw is not None and not isinstance(raw, (int, float)):
raise RuntimeError(
f"sitl_observer inav handshake fixture {path}: "
f"`established_within_s` must be a number or null; got {type(raw).__name__}"
)
return TcpHandshakeReport(established_within_s=float(raw) if raw is not None else None)
def collect_inav_msp_frames(host: str, port: int, window_s: float) -> MspFrameCapture:
"""Return the recorded MSP frame window for ``(host, port)``.
Fixture: ``${E2E_SITL_REPLAY_DIR}/inav_msp_frames_<host>_<port>.json``
with shape ``{frames: [{monotonic_ms, function_id}, ...], expected_num_sat: int}``.
Raises ``RuntimeError`` if env var unset or fixture missing.
"""
if window_s <= 0:
raise RuntimeError(
f"collect_inav_msp_frames: window_s must be positive; got {window_s}"
)
payload, path = _load_required_json(f"inav_msp_frames_{host}_{port}.json")
raw_frames = payload.get("frames", [])
if not isinstance(raw_frames, list):
raise RuntimeError(
f"sitl_observer inav msp frames fixture {path}: `frames` must be a list"
)
frames: list[MspFrameSample] = []
for item in raw_frames:
try:
frames.append(
MspFrameSample(
monotonic_ms=int(item["monotonic_ms"]),
function_id=int(item["function_id"]),
)
)
except (KeyError, TypeError, ValueError) as exc:
raise RuntimeError(
f"sitl_observer inav msp frames fixture {path}: malformed frame: {exc}"
) from exc
expected_num_sat = payload.get("expected_num_sat")
if not isinstance(expected_num_sat, int):
raise RuntimeError(
f"sitl_observer inav msp frames fixture {path}: "
f"`expected_num_sat` must be an int; got {type(expected_num_sat).__name__}"
)
return MspFrameCapture(frames=frames, expected_num_sat=expected_num_sat)
def query_inav_gps_state(host: str) -> InavGpsState:
"""Return the recorded iNav GPS state snapshot for ``host``.
Fixture: ``${E2E_SITL_REPLAY_DIR}/inav_gps_state_<host>.json``.
Raises ``RuntimeError`` if env var unset or fixture missing.
"""
payload, path = _load_required_json(f"inav_gps_state_{host}.json")
try:
return InavGpsState(
fix_type=int(payload["fix_type"]),
num_sat=int(payload["num_sat"]),
provider=str(payload["provider"]),
)
except (KeyError, TypeError, ValueError) as exc:
raise RuntimeError(
f"sitl_observer iNav GPS state fixture {path} malformed: {exc}"
) from exc
e2e/tests/conftest.py
+17
View File
@@ -40,3 +40,20 @@ _bootstrap_runner_path()
# regardless of which conftest it discovers first. Star imports here are
# the documented pytest pattern for conftest layering.
from runner.conftest import * # noqa: F401,F403,E402 — pytest conftest re-export
import pytest # noqa: E402
from runner.helpers import sitl_observer # noqa: E402
@pytest.fixture(scope="session")
def sitl_replay_ready() -> bool:
"""True iff the FDR-replay fixture directory is configured + present.
AZ-595 replaces the per-scenario `_harness_helpers_implemented` probes
that passed `/tmp/non-existent` to each helper and inspected the
exception type. Scenarios should now consult this fixture and skip
cleanly when the SITL replay fixtures haven't been prepared (the
typical case during local unit runs that only exercise the helpers).
"""
return sitl_observer.replay_dir_available()
e2e/tests/negative/test_ft_n_01_outlier_tolerance.py
+5 -40
View File
@@ -22,40 +22,6 @@ from fixtures.injectors.outlier import OutlierInjectionReport
from runner.helpers import outlier_tolerance_evaluator as ote
@pytest.fixture(scope="module")
def _harness_helpers_implemented() -> bool:
from runner.helpers import fdr_reader, imu_replay
from runner.helpers.frame_source_replay import FrameSourceReplayer
try:
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
try:
replayer.replay_video(Path("/tmp/non-existent.mp4"))
except NotImplementedError:
return False
try:
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
except NotImplementedError:
return False
try:
imu_replay.ImuReplayer(emitter=_NullImuEmitter()).replay(Path("/tmp/non-existent.csv")) # type: ignore[arg-type]
except NotImplementedError:
return False
return True
except Exception:
return False
class _NullSink:
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
return None
class _NullImuEmitter:
def emit(self, sample: object) -> None:
return None
@pytest.mark.parametrize(
"outlier_injection_derkachi",
[{"density": "medium", "seed": 0}],
@@ -69,14 +35,13 @@ def test_ft_n_01_outlier_tolerance(
evidence_dir, # type: ignore[no-untyped-def]
run_id: str,
nfr_recorder, # type: ignore[no-untyped-def]
_harness_helpers_implemented: bool,
sitl_replay_ready: bool,
) -> None:
if not _harness_helpers_implemented:
if not sitl_replay_ready:
pytest.skip(
"FT-N-01 full replay requires runner.helpers.{frame_source_replay,"
"fdr_reader,imu_replay} — currently AZ-441 / AZ-407 leftovers. "
"AC-1/AC-2/AC-3 helper logic covered by "
"e2e/_unit_tests/helpers/test_outlier_tolerance_evaluator.py."
"FT-N-01 full replay requires `E2E_SITL_REPLAY_DIR` to point at a "
"prepared SITL replay fixture (AZ-595). AC-1/AC-2/AC-3 helper logic "
"covered by e2e/_unit_tests/helpers/test_outlier_tolerance_evaluator.py."
)
from runner.helpers import fdr_reader
e2e/tests/negative/test_ft_n_02_sharp_turn_failure.py
+5 -40
View File
@@ -35,40 +35,6 @@ DERKACHI_IMU_CSV = DERKACHI_DIR / "data_imu.csv"
DERKACHI_MP4 = DERKACHI_DIR / "flight_derkachi.mp4"
@pytest.fixture(scope="module")
def _harness_helpers_implemented() -> bool:
from runner.helpers import fdr_reader, imu_replay
from runner.helpers.frame_source_replay import FrameSourceReplayer
try:
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
try:
replayer.replay_video(Path("/tmp/non-existent.mp4"))
except NotImplementedError:
return False
try:
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
except NotImplementedError:
return False
try:
imu_replay.ImuReplayer(emitter=_NullImuEmitter()).replay(Path("/tmp/non-existent.csv")) # type: ignore[arg-type]
except NotImplementedError:
return False
return True
except Exception:
return False
class _NullSink:
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
return None
class _NullImuEmitter:
def emit(self, sample: object) -> None:
return None
@pytest.mark.traces_to("AC-3.2,AC-1,AC-2,AC-3,AC-7")
def test_ft_n_02_sharp_turn_failure(
fc_adapter: str,
@@ -76,14 +42,13 @@ def test_ft_n_02_sharp_turn_failure(
evidence_dir, # type: ignore[no-untyped-def]
run_id: str,
nfr_recorder, # type: ignore[no-untyped-def]
_harness_helpers_implemented: bool,
sitl_replay_ready: bool,
) -> None:
if not _harness_helpers_implemented:
if not sitl_replay_ready:
pytest.skip(
"FT-N-02 full replay requires runner.helpers.{frame_source_replay,"
"imu_replay,fdr_reader} — currently AZ-441 / AZ-407 leftovers. "
"AC-2/AC-3 helper logic covered by "
"e2e/_unit_tests/helpers/test_sharp_turn_detector.py."
"FT-N-02 full replay requires `E2E_SITL_REPLAY_DIR` to point at a "
"prepared SITL replay fixture (AZ-595). AC-2/AC-3 helper logic "
"covered by e2e/_unit_tests/helpers/test_sharp_turn_detector.py."
)
from runner.helpers import fdr_reader
e2e/tests/negative/test_ft_n_03_outage_reloc.py
+4 -38
View File
@@ -31,39 +31,6 @@ DERKACHI_DIR = (
DERKACHI_MP4 = DERKACHI_DIR / "flight_derkachi.mp4"
@pytest.fixture(scope="module")
def _harness_helpers_implemented() -> bool:
from runner.helpers import fdr_reader, mavproxy_tlog_reader, sitl_observer
from runner.helpers.frame_source_replay import FrameSourceReplayer
try:
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
try:
replayer.replay_video(Path("/tmp/non-existent.mp4"))
except NotImplementedError:
return False
try:
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
except NotImplementedError:
return False
try:
list(mavproxy_tlog_reader.iter_messages(Path("/tmp/non-existent.tlog")))
except NotImplementedError:
return False
try:
sitl_observer.read_ekf_divergence_events() # type: ignore[attr-defined]
except (AttributeError, NotImplementedError):
return False
return True
except Exception:
return False
class _NullSink:
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
return None
@pytest.mark.traces_to("AC-3.4,AC-1,AC-2,AC-3,AC-4,AC-5")
def test_ft_n_03_outage_reloc(
fc_adapter: str,
@@ -71,13 +38,12 @@ def test_ft_n_03_outage_reloc(
evidence_dir, # type: ignore[no-untyped-def]
run_id: str,
nfr_recorder, # type: ignore[no-untyped-def]
_harness_helpers_implemented: bool,
sitl_replay_ready: bool,
) -> None:
if not _harness_helpers_implemented:
if not sitl_replay_ready:
pytest.skip(
"FT-N-03 full replay requires runner.helpers.{frame_source_replay,"
"fdr_reader,mavproxy_tlog_reader,sitl_observer} — currently "
"AZ-441 / AZ-407 / AZ-416 leftovers. AC-1..AC-4 evaluator logic "
"FT-N-03 full replay requires `E2E_SITL_REPLAY_DIR` to point at a "
"prepared SITL replay fixture (AZ-595). AC-1..AC-4 evaluator logic "
"covered by e2e/_unit_tests/helpers/test_outage_request_evaluator.py."
)
e2e/tests/negative/test_ft_n_04_blackout_spoof.py
+6 -40
View File
@@ -26,40 +26,6 @@ from runner.helpers import blackout_spoof_evaluator as bse
_WINDOW_LADDER_S = (5.0, 15.0, 35.0)
@pytest.fixture(scope="module")
def _harness_helpers_implemented() -> bool:
from runner.helpers import fdr_reader, mavproxy_tlog_reader, sitl_observer
from runner.helpers.frame_source_replay import FrameSourceReplayer
try:
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
try:
replayer.replay_video(Path("/tmp/non-existent.mp4"))
except NotImplementedError:
return False
try:
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
except NotImplementedError:
return False
try:
list(mavproxy_tlog_reader.iter_messages(Path("/tmp/non-existent.tlog")))
except NotImplementedError:
return False
try:
sitl_observer.read_gps_health_samples() # type: ignore[attr-defined]
sitl_observer.read_consistency_check_events() # type: ignore[attr-defined]
except (AttributeError, NotImplementedError):
return False
return True
except Exception:
return False
class _NullSink:
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
return None
@pytest.mark.parametrize(
"blackout_spoof_derkachi",
[{"window_seconds": s, "seed": 0} for s in _WINDOW_LADDER_S],
@@ -76,14 +42,14 @@ def test_ft_n_04_blackout_spoof(
evidence_dir, # type: ignore[no-untyped-def]
run_id: str,
nfr_recorder, # type: ignore[no-untyped-def]
_harness_helpers_implemented: bool,
sitl_replay_ready: bool,
) -> None:
if not _harness_helpers_implemented:
if not sitl_replay_ready:
pytest.skip(
"FT-N-04 full replay requires runner.helpers.{frame_source_replay,"
"fdr_reader,mavproxy_tlog_reader,sitl_observer,fc_proxy} — currently "
"AZ-441 / AZ-407 / AZ-416 leftovers. AC-1..AC-8 evaluator logic "
"covered by e2e/_unit_tests/helpers/test_blackout_spoof_evaluator.py."
"FT-N-04 full replay requires `E2E_SITL_REPLAY_DIR` to point at a "
"prepared SITL replay fixture (AZ-595) AND a runtime fc_proxy "
"driver. AC-1..AC-8 evaluator logic covered by "
"e2e/_unit_tests/helpers/test_blackout_spoof_evaluator.py."
)
from runner.helpers import fdr_reader, mavproxy_tlog_reader, sitl_observer
e2e/tests/positive/test_ft_p_01_still_image_accuracy.py
+9 -38
View File
@@ -25,9 +25,10 @@ What this file does NOT own:
* The SITL message receipt → ``runner.helpers.sitl_observer`` (stub;
owned by AZ-416/AZ-417) — skip-gated.
When both upstream helpers land, this file's runtime path activates
automatically — the skip is keyed off the ``NotImplementedError`` from
the helper imports, not off a hard-coded marker.
When ``E2E_SITL_REPLAY_DIR`` is set and points at a prepared SITL
replay fixture, this file's runtime path activates automatically; until
then the scenario skips via the shared `sitl_replay_ready` fixture
(AZ-595).
"""
from __future__ import annotations
@@ -43,36 +44,6 @@ GT_CSV = Path(__file__).resolve().parents[3] / "_docs" / "00_problem" / "input_d
STILL_IMAGES_DIR = GT_CSV.parent
@pytest.fixture(scope="module")
def _harness_helpers_implemented() -> bool:
"""True iff the upstream replay + SITL-observation helpers are real.
Same auto-detect pattern as FT-P-02 / FT-P-03 — the gate flips when
the helpers stop raising NotImplementedError, so no marker churn.
"""
from runner.helpers import frame_source_replay, sitl_observer
from runner.helpers.frame_source_replay import FrameSourceReplayer
try:
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
try:
replayer.replay_image_directory(Path("/tmp/non-existent"))
except NotImplementedError:
return False
try:
sitl_observer.get_observer(fc_adapter="ardupilot", host="sitl-ardupilot")
except NotImplementedError:
return False
return True
except Exception:
return False
class _NullSink:
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
return None
def _ft_p_01_image_paths() -> list[Path]:
"""The 60 AD0000NN.jpg images, sorted lexicographically (AD000001..AD000060)."""
return sorted(STILL_IMAGES_DIR.glob("AD??????.jpg"))
@@ -85,7 +56,7 @@ def test_ft_p_01_still_image_accuracy(
evidence_dir, # type: ignore[no-untyped-def]
run_id: str,
nfr_recorder, # type: ignore[no-untyped-def]
_harness_helpers_implemented: bool,
sitl_replay_ready: bool,
) -> None:
"""Full FT-P-01 scenario (AC-1.1, AC-1.2).
@@ -95,11 +66,11 @@ def test_ft_p_01_still_image_accuracy(
AC-4: per-image timeout → ``error_m=∞``; aggregate continues.
AC-5: parametrized across ``(fc_adapter, vio_strategy)`` (4 variants).
"""
if not _harness_helpers_implemented:
if not sitl_replay_ready:
pytest.skip(
"FT-P-01 still-image push requires runner.helpers.{frame_source_replay,"
"sitl_observer} — currently AZ-441 + AZ-416/AZ-417 leftovers. "
"Pure-logic ACs covered by e2e/_unit_tests/helpers/test_accuracy_evaluator.py."
"FT-P-01 still-image push requires `E2E_SITL_REPLAY_DIR` to point "
"at a prepared SITL replay fixture (AZ-595). Pure-logic ACs "
"covered by e2e/_unit_tests/helpers/test_accuracy_evaluator.py."
)
from runner.helpers import frame_source_replay, sitl_observer
e2e/tests/positive/test_ft_p_02_derkachi_drift.py
+13 -60
View File
@@ -23,16 +23,16 @@ What this file does NOT own:
(still a stub; AZ-408 was about the synthetic-injection injectors,
not the video replayer); the scenario is marked
``@pytest.mark.deferred_ac(reason=...)`` until that helper lands.
* The FDR-archive iteration → ``runner.helpers.fdr_reader`` (owned by
AZ-441); same skip gate.
* The FDR-archive iteration → ``runner.helpers.fdr_reader`` (AZ-594,
landed in batch 74) — the scenario still depends on a prepared SITL
replay fixture (AZ-595) that produces the per-run FDR archive.
* The MAVLink ``GLOBAL_POSITION_INT`` GT replay → handled by the
``imu_replay`` helper which currently raises NotImplementedError
(owned by AZ-407 in spec, but the helper file was not touched by
the AZ-407 batch).
``imu_replay`` helper (AZ-594, landed in batch 74).
When all three upstream helpers land, this file's runtime path activates
automatically — the skip is keyed off the ``NotImplementedError`` from
the helper imports, not off a hard-coded marker.
When ``E2E_SITL_REPLAY_DIR`` is set and points at a prepared SITL
replay fixture, this file's runtime path activates automatically; until
then the scenario skips via the shared `sitl_replay_ready` fixture
(AZ-595).
"""
from __future__ import annotations
@@ -44,53 +44,6 @@ import pytest
from runner.helpers import anchor_pair_detector as apd
@pytest.fixture(scope="module")
def _harness_helpers_implemented() -> bool:
"""True iff every upstream helper FT-P-02 needs has a real impl.
Used to gate the full-replay scenarios. Helper-level NotImplementedError
is the signal — we don't hard-code a "deferred until task X" marker
because then a developer who lands the helper would have to also
remember to flip the marker. The auto-detect pattern is also what
other downstream scenarios will reuse.
"""
from runner.helpers import fdr_reader, frame_source_replay, imu_replay
from runner.helpers.frame_source_replay import FrameSourceReplayer
try:
# The cheapest sentinel for each helper:
# - FrameSourceReplayer.replay_video raises NotImplementedError
# - fdr_reader.iter_records raises NotImplementedError
# - ImuReplayer.replay raises NotImplementedError
# We check by inspecting __doc__ / source rather than calling, so
# the gate stays cheap.
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
try:
replayer.replay_video(Path("/tmp/non-existent.mp4"))
except NotImplementedError:
return False
try:
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
except NotImplementedError:
return False
try:
imu_replay.ImuReplayer(emitter=_NullImuEmitter()).replay(Path("/tmp/non-existent.csv")) # type: ignore[arg-type]
except NotImplementedError:
return False
return True
except Exception:
return False
class _NullSink:
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
return None
class _NullImuEmitter:
def emit(self, sample: object) -> None:
return None
@pytest.mark.traces_to("AC-1.3,AC-1,AC-2,AC-3,AC-4,AC-5")
def test_ft_p_02_derkachi_drift(
fc_adapter: str,
@@ -98,7 +51,7 @@ def test_ft_p_02_derkachi_drift(
evidence_dir, # type: ignore[no-untyped-def]
run_id: str,
nfr_recorder, # type: ignore[no-untyped-def]
_harness_helpers_implemented: bool,
sitl_replay_ready: bool,
) -> None:
"""Full FT-P-02 scenario (AC-1.3). See module docstring.
@@ -110,11 +63,11 @@ def test_ft_p_02_derkachi_drift(
AC-4: bin medians monotonic with age — covered by check_monotonic().
AC-5: parametrized across (fc_adapter, vio_strategy).
"""
if not _harness_helpers_implemented:
if not sitl_replay_ready:
pytest.skip(
"FT-P-02 full replay requires runner.helpers.{frame_source_replay,"
"fdr_reader,imu_replay} — currently AZ-441 / AZ-407 leftovers. "
"Pure-logic ACs covered by e2e/_unit_tests/helpers/test_anchor_pair_detector.py."
"FT-P-02 full replay requires `E2E_SITL_REPLAY_DIR` to point at a "
"prepared SITL replay fixture (AZ-595). Pure-logic ACs covered by "
"e2e/_unit_tests/helpers/test_anchor_pair_detector.py."
)
# Once the helpers land, the body below activates. We keep it
e2e/tests/positive/test_ft_p_03_14_schema_wgs84.py
+11 -43
View File
@@ -35,53 +35,20 @@ import pytest
from runner.helpers import estimate_schema
@pytest.fixture(scope="module")
def _harness_helpers_implemented() -> bool:
"""Same gate as FT-P-02: are frame replay + SITL observer + sidechannel
decoders all real? If not, skip the docker-bound runtime path.
"""
from runner.helpers import frame_source_replay, mavproxy_tlog_reader, sitl_observer
from runner.helpers.frame_source_replay import FrameSourceReplayer
try:
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
try:
replayer.replay_image_directory(Path("/tmp/non-existent"))
except NotImplementedError:
return False
try:
sitl_observer.get_observer("ardupilot", "test-host")
except NotImplementedError:
return False
try:
list(mavproxy_tlog_reader.iter_messages(Path("/tmp/non-existent.tlog")))
except NotImplementedError:
return False
return True
except Exception:
return False
class _NullSink:
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
return None
@pytest.mark.traces_to("AC-1.4,AC-4.3")
def test_schema_and_source_label(
fc_adapter: str,
vio_strategy: str,
evidence_dir, # type: ignore[no-untyped-def]
nfr_recorder, # type: ignore[no-untyped-def]
_harness_helpers_implemented: bool,
sitl_replay_ready: bool,
) -> None:
"""FT-P-03: schema completeness (AC-1) + source-label set containment (AC-2)."""
if not _harness_helpers_implemented:
if not sitl_replay_ready:
pytest.skip(
"FT-P-03 single-image push requires runner.helpers.{frame_source_replay,"
"sitl_observer,mavproxy_tlog_reader} — currently pending AZ-407 / "
"AZ-416/417 leftovers. Pure-logic ACs covered by "
"e2e/_unit_tests/helpers/test_estimate_schema.py."
"FT-P-03 single-image push requires `E2E_SITL_REPLAY_DIR` to point "
"at a prepared SITL replay fixture (AZ-595). Pure-logic ACs "
"covered by e2e/_unit_tests/helpers/test_estimate_schema.py."
)
record, source_label = _push_single_image_and_observe(fc_adapter, vio_strategy)
@@ -109,13 +76,14 @@ def test_wgs84_coordinate_range(
vio_strategy: str,
evidence_dir, # type: ignore[no-untyped-def]
nfr_recorder, # type: ignore[no-untyped-def]
_harness_helpers_implemented: bool,
sitl_replay_ready: bool,
) -> None:
"""FT-P-14: decoded lat/lon inside WGS84 bounds (AC-3)."""
if not _harness_helpers_implemented:
if not sitl_replay_ready:
pytest.skip(
"FT-P-14 single-image push requires the same upstream helpers as FT-P-03. "
"Pure-logic AC covered by e2e/_unit_tests/helpers/test_estimate_schema.py."
"FT-P-14 single-image push requires `E2E_SITL_REPLAY_DIR` to point "
"at a prepared SITL replay fixture (AZ-595). Pure-logic AC covered "
"by e2e/_unit_tests/helpers/test_estimate_schema.py."
)
record, _label = _push_single_image_and_observe(fc_adapter, vio_strategy)
@@ -141,7 +109,7 @@ def _push_single_image_and_observe(fc_adapter: str, vio_strategy: str): # type:
"""Push AD000001.jpg through the SUT and return (outbound_record, source_label).
Stub until runner.helpers.{frame_source_replay,sitl_observer,mavproxy_tlog_reader}
land; the scenario test's skip gate (``_harness_helpers_implemented``)
land; the scenario test's `sitl_replay_ready` skip gate (AZ-595)
keeps this from executing prematurely.
"""
raise NotImplementedError(
e2e/tests/positive/test_ft_p_04_derkachi_f2f_registration.py
+9 -43
View File
@@ -29,9 +29,10 @@ What this file does NOT own:
* The FDR-archive iteration → ``runner.helpers.fdr_reader`` (stub;
AZ-441) — skip-gated.
When all three upstream helpers land, this file's runtime path activates
automatically — the skip is keyed off the ``NotImplementedError`` from
the helper imports, not off a hard-coded marker.
When ``E2E_SITL_REPLAY_DIR`` is set and points at a prepared SITL
replay fixture, this file's runtime path activates automatically; until
then the scenario skips via the shared `sitl_replay_ready` fixture
(AZ-595).
"""
from __future__ import annotations
@@ -53,41 +54,6 @@ DERKACHI_IMU_CSV = DERKACHI_DIR / "data_imu.csv"
DERKACHI_MP4 = DERKACHI_DIR / "flight_derkachi.mp4"
@pytest.fixture(scope="module")
def _harness_helpers_implemented() -> bool:
"""True iff every upstream helper FT-P-04 needs has a real impl."""
from runner.helpers import fdr_reader, frame_source_replay, imu_replay
from runner.helpers.frame_source_replay import FrameSourceReplayer
try:
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
try:
replayer.replay_video(Path("/tmp/non-existent.mp4"))
except NotImplementedError:
return False
try:
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
except NotImplementedError:
return False
try:
imu_replay.ImuReplayer(emitter=_NullImuEmitter()).replay(Path("/tmp/non-existent.csv")) # type: ignore[arg-type]
except NotImplementedError:
return False
return True
except Exception:
return False
class _NullSink:
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
return None
class _NullImuEmitter:
def emit(self, sample: object) -> None:
return None
@pytest.mark.traces_to("AC-2.1a,AC-1,AC-2,AC-3,AC-4")
def test_ft_p_04_derkachi_f2f_registration(
fc_adapter: str,
@@ -95,7 +61,7 @@ def test_ft_p_04_derkachi_f2f_registration(
evidence_dir, # type: ignore[no-untyped-def]
run_id: str,
nfr_recorder, # type: ignore[no-untyped-def]
_harness_helpers_implemented: bool,
sitl_replay_ready: bool,
) -> None:
"""Full FT-P-04 scenario.
@@ -105,11 +71,11 @@ def test_ft_p_04_derkachi_f2f_registration(
AC-3: sharp-turn frames excluded from the denominator.
AC-4: parametrized across ``(fc_adapter, vio_strategy)``.
"""
if not _harness_helpers_implemented:
if not sitl_replay_ready:
pytest.skip(
"FT-P-04 full replay requires runner.helpers.{frame_source_replay,"
"imu_replay,fdr_reader} — currently AZ-441 / AZ-407 leftovers. "
"Pure-logic ACs covered by e2e/_unit_tests/helpers/test_registration_classifier.py."
"FT-P-04 full replay requires `E2E_SITL_REPLAY_DIR` to point at a "
"prepared SITL replay fixture (AZ-595). Pure-logic ACs covered by "
"e2e/_unit_tests/helpers/test_registration_classifier.py."
)
from runner.helpers import fdr_reader, imu_replay
e2e/tests/positive/test_ft_p_05_sat_anchor.py
+5 -35
View File
@@ -43,36 +43,6 @@ GT_CSV = Path(__file__).resolve().parents[3] / "_docs" / "00_problem" / "input_d
STILL_IMAGES_DIR = GT_CSV.parent
@pytest.fixture(scope="module")
def _harness_helpers_implemented() -> bool:
"""True iff replay + SITL observation + FDR helpers are all real."""
from runner.helpers import fdr_reader, frame_source_replay, sitl_observer
from runner.helpers.frame_source_replay import FrameSourceReplayer
try:
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
try:
replayer.replay_image_directory(Path("/tmp/non-existent"))
except NotImplementedError:
return False
try:
sitl_observer.get_observer(fc_adapter="ardupilot", host="sitl-ardupilot")
except NotImplementedError:
return False
try:
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
except NotImplementedError:
return False
return True
except Exception:
return False
class _NullSink:
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
return None
@pytest.mark.traces_to("AC-2.1b,AC-1,AC-2,AC-3,AC-5")
def test_ft_p_05_sat_anchor(
fc_adapter: str,
@@ -80,7 +50,7 @@ def test_ft_p_05_sat_anchor(
evidence_dir, # type: ignore[no-untyped-def]
run_id: str,
nfr_recorder, # type: ignore[no-untyped-def]
_harness_helpers_implemented: bool,
sitl_replay_ready: bool,
) -> None:
"""Full FT-P-05 scenario.
@@ -89,11 +59,11 @@ def test_ft_p_05_sat_anchor(
AC-3: ≥80 % within 50 m AND ≥50 % within 20 m (same image set as FT-P-01).
AC-5: parametrized across ``(fc_adapter, vio_strategy)``.
"""
if not _harness_helpers_implemented:
if not sitl_replay_ready:
pytest.skip(
"FT-P-05 still-image push requires runner.helpers.{frame_source_replay,"
"sitl_observer,fdr_reader} — currently AZ-441 + AZ-416/AZ-417 leftovers. "
"Pure-logic ACs covered by e2e/_unit_tests/helpers/test_mre_evaluator.py."
"FT-P-05 still-image push requires `E2E_SITL_REPLAY_DIR` to point "
"at a prepared SITL replay fixture (AZ-595). Pure-logic ACs "
"covered by e2e/_unit_tests/helpers/test_mre_evaluator.py."
)
from runner.helpers import fdr_reader, frame_source_replay, sitl_observer
e2e/tests/positive/test_ft_p_07_sharp_turn_recovery.py
+5 -41
View File
@@ -46,41 +46,6 @@ DERKACHI_IMU_CSV = DERKACHI_DIR / "data_imu.csv"
DERKACHI_MP4 = DERKACHI_DIR / "flight_derkachi.mp4"
@pytest.fixture(scope="module")
def _harness_helpers_implemented() -> bool:
"""True iff replay + IMU + FDR helpers are real."""
from runner.helpers import fdr_reader, imu_replay
from runner.helpers.frame_source_replay import FrameSourceReplayer
try:
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
try:
replayer.replay_video(Path("/tmp/non-existent.mp4"))
except NotImplementedError:
return False
try:
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
except NotImplementedError:
return False
try:
imu_replay.ImuReplayer(emitter=_NullImuEmitter()).replay(Path("/tmp/non-existent.csv")) # type: ignore[arg-type]
except NotImplementedError:
return False
return True
except Exception:
return False
class _NullSink:
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
return None
class _NullImuEmitter:
def emit(self, sample: object) -> None:
return None
@pytest.mark.traces_to("AC-3.2,AC-1,AC-4,AC-5,AC-6,AC-7")
def test_ft_p_07_sharp_turn_recovery(
fc_adapter: str,
@@ -88,14 +53,13 @@ def test_ft_p_07_sharp_turn_recovery(
evidence_dir, # type: ignore[no-untyped-def]
run_id: str,
nfr_recorder, # type: ignore[no-untyped-def]
_harness_helpers_implemented: bool,
sitl_replay_ready: bool,
) -> None:
if not _harness_helpers_implemented:
if not sitl_replay_ready:
pytest.skip(
"FT-P-07 full replay requires runner.helpers.{frame_source_replay,"
"imu_replay,fdr_reader} — currently AZ-441 / AZ-407 leftovers. "
"AC-1/AC-4/AC-5/AC-6 helper logic covered by "
"e2e/_unit_tests/helpers/test_sharp_turn_detector.py."
"FT-P-07 full replay requires `E2E_SITL_REPLAY_DIR` to point at a "
"prepared SITL replay fixture (AZ-595). AC-1/AC-4/AC-5/AC-6 helper "
"logic covered by e2e/_unit_tests/helpers/test_sharp_turn_detector.py."
)
from runner.helpers import fdr_reader
e2e/tests/positive/test_ft_p_08_multi_segment_reloc.py
+5 -31
View File
@@ -38,32 +38,6 @@ import pytest
from runner.helpers import multi_segment_evaluator as mse
@pytest.fixture(scope="module")
def _harness_helpers_implemented() -> bool:
"""True iff replay + FDR helpers are real."""
from runner.helpers import fdr_reader, frame_source_replay
from runner.helpers.frame_source_replay import FrameSourceReplayer
try:
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
try:
replayer.replay_image_directory(Path("/tmp/non-existent"))
except NotImplementedError:
return False
try:
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
except NotImplementedError:
return False
return True
except Exception:
return False
class _NullSink:
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
return None
@pytest.mark.traces_to("AC-3.3,AC-1,AC-2,AC-3,AC-4,AC-5")
def test_ft_p_08_multi_segment_reloc(
fc_adapter: str,
@@ -72,7 +46,7 @@ def test_ft_p_08_multi_segment_reloc(
run_id: str,
nfr_recorder, # type: ignore[no-untyped-def]
multi_segment_derkachi, # type: ignore[no-untyped-def] # AZ-408 pytest fixture
_harness_helpers_implemented: bool,
sitl_replay_ready: bool,
) -> None:
"""Full FT-P-08 scenario.
@@ -82,11 +56,11 @@ def test_ft_p_08_multi_segment_reloc(
AC-4: trajectory continuity ≤100 m at each recovery.
AC-5: parameterised across ``(fc_adapter, vio_strategy)``.
"""
if not _harness_helpers_implemented:
if not sitl_replay_ready:
pytest.skip(
"FT-P-08 multi-segment replay requires runner.helpers.{frame_source_replay,"
"fdr_reader} — currently AZ-441 leftover. Pure-logic ACs covered by "
"e2e/_unit_tests/helpers/test_multi_segment_evaluator.py."
"FT-P-08 multi-segment replay requires `E2E_SITL_REPLAY_DIR` to "
"point at a prepared SITL replay fixture (AZ-595). Pure-logic ACs "
"covered by e2e/_unit_tests/helpers/test_multi_segment_evaluator.py."
)
from runner.helpers import fdr_reader
e2e/tests/positive/test_ft_p_09_ap_signing.py
+5 -36
View File
@@ -55,36 +55,6 @@ MAVLINK_PASSKEY_FIXTURE = (
REPLAY_WINDOW_S = 60
@pytest.fixture(scope="module")
def _ap_harness_implemented() -> bool:
"""True iff frame_source_replay + sitl_observer AP-side leg are real."""
from runner.helpers import sitl_observer
from runner.helpers.frame_source_replay import FrameSourceReplayer
try:
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
try:
replayer.replay_video(Path("/tmp/non-existent.mp4"))
except NotImplementedError:
return False
try:
sitl_observer.capture_ap_tlog(host="ardupilot-sitl", duration_s=0.01)
except (NotImplementedError, AttributeError):
return False
try:
sitl_observer.read_ap_parameter(host="ardupilot-sitl", name="EK3_SRC1_POSXY")
except (NotImplementedError, AttributeError):
return False
return True
except Exception:
return False
class _NullSink:
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
return None
@pytest.mark.traces_to("AC-4.3,AC-1,AC-2,AC-3,AC-4,AC-5,D-C8-9")
def test_ft_p_09_ap_signing(
vio_strategy: str,
@@ -92,7 +62,7 @@ def test_ft_p_09_ap_signing(
run_id: str,
nfr_recorder, # type: ignore[no-untyped-def]
request, # type: ignore[no-untyped-def]
_ap_harness_implemented: bool,
sitl_replay_ready: bool,
) -> None:
"""Full FT-P-09-AP scenario; parameterized per vio_strategy."""
fc_adapter = request.getfixturevalue("fc_adapter")
@@ -105,12 +75,11 @@ def test_ft_p_09_ap_signing(
"AZ-407 / AZ-408 owns the on-disk fixture."
)
if not _ap_harness_implemented:
if not sitl_replay_ready:
pytest.skip(
"FT-P-09-AP full scenario requires runner.helpers.{frame_source_replay,"
"sitl_observer.capture_ap_tlog,sitl_observer.read_ap_parameter} — "
"currently AZ-441 / AZ-407 leftovers. Pure-logic AC-1..AC-4 covered by "
"e2e/_unit_tests/helpers/test_ap_contract_evaluator.py."
"FT-P-09-AP full scenario requires `E2E_SITL_REPLAY_DIR` to point "
"at a prepared SITL replay fixture (AZ-595). Pure-logic AC-1..AC-4 "
"covered by e2e/_unit_tests/helpers/test_ap_contract_evaluator.py."
)
from runner.helpers import sitl_observer
e2e/tests/positive/test_ft_p_09_inav.py
+5 -32
View File
@@ -45,32 +45,6 @@ REPLAY_WINDOW_S = 60
TCP_HANDSHAKE_BUDGET_S = 5
@pytest.fixture(scope="module")
def _inav_harness_implemented() -> bool:
"""True iff frame_source_replay + sitl_observer iNav leg are real."""
from runner.helpers import sitl_observer
from runner.helpers.frame_source_replay import FrameSourceReplayer
try:
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
try:
replayer.replay_video(Path("/tmp/non-existent.mp4"))
except NotImplementedError:
return False
try:
sitl_observer.observe_inav_tcp_handshake(host="inav-sitl", port=5760, timeout_s=0.01)
except (NotImplementedError, AttributeError):
return False
return True
except Exception:
return False
class _NullSink:
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
return None
@pytest.mark.traces_to("AC-4.3,AC-1,AC-2,AC-3,AC-4")
def test_ft_p_09_inav(
vio_strategy: str,
@@ -78,7 +52,7 @@ def test_ft_p_09_inav(
run_id: str,
nfr_recorder, # type: ignore[no-untyped-def]
request, # type: ignore[no-untyped-def]
_inav_harness_implemented: bool,
sitl_replay_ready: bool,
) -> None:
"""Full FT-P-09-iNav scenario; parameterized per vio_strategy.
@@ -90,12 +64,11 @@ def test_ft_p_09_inav(
if fc_adapter != "inav":
pytest.skip("FT-P-09-iNav is iNav-only; ardupilot variant is FT-P-09-AP (AZ-416)")
if not _inav_harness_implemented:
if not sitl_replay_ready:
pytest.skip(
"FT-P-09-iNav full scenario requires runner.helpers.{frame_source_replay,"
"sitl_observer.observe_inav_tcp_handshake} — currently AZ-441 / AZ-407 leftovers. "
"Pure-logic AC-2/AC-3 covered by "
"e2e/_unit_tests/helpers/test_msp_frame_observer.py."
"FT-P-09-iNav full scenario requires `E2E_SITL_REPLAY_DIR` to "
"point at a prepared SITL replay fixture (AZ-595). Pure-logic "
"AC-2/AC-3 covered by e2e/_unit_tests/helpers/test_msp_frame_observer.py."
)
from runner.helpers import sitl_observer
e2e/tests/positive/test_ft_p_10_smoothing_lookback.py
+5 -40
View File
@@ -49,41 +49,6 @@ DERKACHI_IMU_CSV = DERKACHI_DIR / "data_imu.csv"
DERKACHI_MP4 = DERKACHI_DIR / "flight_derkachi.mp4"
@pytest.fixture(scope="module")
def _harness_helpers_implemented() -> bool:
"""True iff replay + IMU + FDR helpers are real."""
from runner.helpers import fdr_reader, frame_source_replay, imu_replay
from runner.helpers.frame_source_replay import FrameSourceReplayer
try:
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
try:
replayer.replay_video(Path("/tmp/non-existent.mp4"))
except NotImplementedError:
return False
try:
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
except NotImplementedError:
return False
try:
imu_replay.ImuReplayer(emitter=_NullImuEmitter()).replay(Path("/tmp/non-existent.csv")) # type: ignore[arg-type]
except NotImplementedError:
return False
return True
except Exception:
return False
class _NullSink:
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
return None
class _NullImuEmitter:
def emit(self, sample: object) -> None:
return None
def _load_derkachi_gt_track() -> list[se.GtPose]:
"""Read GLOBAL_POSITION_INT poses from data_imu.csv.
@@ -115,7 +80,7 @@ def test_ft_p_10_smoothing_lookback(
evidence_dir, # type: ignore[no-untyped-def]
run_id: str,
nfr_recorder, # type: ignore[no-untyped-def]
_harness_helpers_implemented: bool,
sitl_replay_ready: bool,
) -> None:
"""Full FT-P-10 scenario.
@@ -125,11 +90,11 @@ def test_ft_p_10_smoothing_lookback(
AC-3: mean_improvement_m ≥ 5 m.
AC-4: parameterised across ``(fc_adapter, vio_strategy)``.
"""
if not _harness_helpers_implemented:
if not sitl_replay_ready:
pytest.skip(
"FT-P-10 full replay requires runner.helpers.{frame_source_replay,"
"imu_replay,fdr_reader} — currently AZ-441 / AZ-407 leftovers. "
"Pure-logic ACs covered by e2e/_unit_tests/helpers/test_smoothing_evaluator.py."
"FT-P-10 full replay requires `E2E_SITL_REPLAY_DIR` to point at a "
"prepared SITL replay fixture (AZ-595). Pure-logic ACs covered by "
"e2e/_unit_tests/helpers/test_smoothing_evaluator.py."
)
from runner.helpers import fdr_reader, imu_replay
e2e/tests/positive/test_ft_p_11_cold_start_init.py
+10 -52
View File
@@ -51,45 +51,6 @@ COLD_BOOT_FIXTURE = (
OPERATOR_ORIGIN = cse.LatLonAlt(lat_deg=50.0, lon_deg=36.2, alt_m=200.0)
@pytest.fixture(scope="module")
def _cold_start_harness_implemented() -> bool:
"""True iff frame_source_replay + sitl_observer + fdr_reader are real.
Cold start adds two specific SITL-observer surfaces beyond the
common replay path: ``prepare_sitl_cold_boot`` (parameter-load
path) and ``prepare_sitl_no_gps`` (``SIM_GPS_DISABLE = 1``).
"""
from runner.helpers import fdr_reader, sitl_observer
from runner.helpers.frame_source_replay import FrameSourceReplayer
try:
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
try:
replayer.replay_video(Path("/tmp/non-existent.mp4"))
except NotImplementedError:
return False
try:
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
except NotImplementedError:
return False
try:
sitl_observer.prepare_sitl_cold_boot(host="ardupilot-sitl", fixture_path=COLD_BOOT_FIXTURE)
except (NotImplementedError, AttributeError):
return False
try:
sitl_observer.prepare_sitl_no_gps(host="ardupilot-sitl")
except (NotImplementedError, AttributeError):
return False
return True
except Exception:
return False
class _NullSink:
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
return None
@pytest.fixture
def _cold_run_id(run_id: str) -> str:
"""Return a fresh run_id — Cold-start REQUIRES an empty fdr-output volume.
@@ -116,16 +77,14 @@ def test_ft_p_11_cold_start_origin_variants(
_cold_run_id: str,
nfr_recorder, # type: ignore[no-untyped-def]
tmp_path: Path,
_cold_start_harness_implemented: bool,
sitl_replay_ready: bool,
) -> None:
"""FT-P-11 AC-1 / AC-2 / AC-4 across the three origin_source variants."""
if not _cold_start_harness_implemented:
if not sitl_replay_ready:
pytest.skip(
"FT-P-11 full scenario requires runner.helpers.{frame_source_replay,"
"fdr_reader,sitl_observer.prepare_sitl_cold_boot,"
"sitl_observer.prepare_sitl_no_gps} — currently AZ-441 / AZ-407 "
"leftovers. Pure-logic AC-1/2/3/4 covered by "
"e2e/_unit_tests/helpers/test_cold_start_evaluator.py."
"FT-P-11 full scenario requires `E2E_SITL_REPLAY_DIR` to point at a "
"prepared SITL replay fixture (AZ-595). Pure-logic AC-1/2/3/4 "
"covered by e2e/_unit_tests/helpers/test_cold_start_evaluator.py."
)
from runner.helpers import fdr_reader, sitl_observer
@@ -241,15 +200,14 @@ def test_ft_p_11_cold_start_no_origin_aborts(
_cold_run_id: str,
nfr_recorder, # type: ignore[no-untyped-def]
tmp_path: Path,
_cold_start_harness_implemented: bool,
sitl_replay_ready: bool,
) -> None:
"""AC-3: Manifest empty + SITL no GPS → SUT MUST refuse takeoff."""
if not _cold_start_harness_implemented:
if not sitl_replay_ready:
pytest.skip(
"FT-P-11 AC-3 full scenario requires runner.helpers.{frame_source_replay,"
"fdr_reader,sitl_observer.prepare_sitl_no_gps} — currently AZ-441 / "
"AZ-407 leftovers. Pure-logic AC-3 covered by "
"e2e/_unit_tests/helpers/test_cold_start_evaluator.py."
"FT-P-11 AC-3 full scenario requires `E2E_SITL_REPLAY_DIR` to point "
"at a prepared SITL replay fixture (AZ-595). Pure-logic AC-3 "
"covered by e2e/_unit_tests/helpers/test_cold_start_evaluator.py."
)
from runner.helpers import fdr_reader, sitl_observer