mirror of
https://github.com/azaion/gps-denied-onboard.git
synced 2026-06-22 20:41:12 +00:00
[AZ-595] Batch 75: sitl_observer FDR-replay + scenario probe cleanup
Implement all 11 `sitl_observer` public surfaces as an offline
FDR-replay strategy (reads JSON fixtures under `${E2E_SITL_REPLAY_DIR}`
instead of live pymavlink/yamspy). Replace 12 per-scenario
`_harness_helpers_implemented` probes with one shared session-scoped
`sitl_replay_ready` fixture in `e2e/tests/conftest.py`.
Net: -636 LoC of duplicated scenario gating, +17 LoC shared fixture,
+38 new unit tests (596 total, up from 558). Includes K=3 cumulative
review for batches 73-75 (PASS).
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Oleksandr Bezdieniezhnykh
@@ -25,9 +25,10 @@ What this file does NOT own:
|
||||
* The SITL message receipt → ``runner.helpers.sitl_observer`` (stub;
|
||||
owned by AZ-416/AZ-417) — skip-gated.
|
||||
|
||||
When both upstream helpers land, this file's runtime path activates
|
||||
automatically — the skip is keyed off the ``NotImplementedError`` from
|
||||
the helper imports, not off a hard-coded marker.
|
||||
When ``E2E_SITL_REPLAY_DIR`` is set and points at a prepared SITL
|
||||
replay fixture, this file's runtime path activates automatically; until
|
||||
then the scenario skips via the shared `sitl_replay_ready` fixture
|
||||
(AZ-595).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -43,36 +44,6 @@ GT_CSV = Path(__file__).resolve().parents[3] / "_docs" / "00_problem" / "input_d
|
||||
STILL_IMAGES_DIR = GT_CSV.parent
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def _harness_helpers_implemented() -> bool:
|
||||
"""True iff the upstream replay + SITL-observation helpers are real.
|
||||
|
||||
Same auto-detect pattern as FT-P-02 / FT-P-03 — the gate flips when
|
||||
the helpers stop raising NotImplementedError, so no marker churn.
|
||||
"""
|
||||
from runner.helpers import frame_source_replay, sitl_observer
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
try:
|
||||
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
|
||||
try:
|
||||
replayer.replay_image_directory(Path("/tmp/non-existent"))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
sitl_observer.get_observer(fc_adapter="ardupilot", host="sitl-ardupilot")
|
||||
except NotImplementedError:
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class _NullSink:
|
||||
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _ft_p_01_image_paths() -> list[Path]:
|
||||
"""The 60 AD0000NN.jpg images, sorted lexicographically (AD000001..AD000060)."""
|
||||
return sorted(STILL_IMAGES_DIR.glob("AD??????.jpg"))
|
||||
@@ -85,7 +56,7 @@ def test_ft_p_01_still_image_accuracy(
|
||||
evidence_dir, # type: ignore[no-untyped-def]
|
||||
run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
_harness_helpers_implemented: bool,
|
||||
sitl_replay_ready: bool,
|
||||
) -> None:
|
||||
"""Full FT-P-01 scenario (AC-1.1, AC-1.2).
|
||||
|
||||
@@ -95,11 +66,11 @@ def test_ft_p_01_still_image_accuracy(
|
||||
AC-4: per-image timeout → ``error_m=∞``; aggregate continues.
|
||||
AC-5: parametrized across ``(fc_adapter, vio_strategy)`` (4 variants).
|
||||
"""
|
||||
if not _harness_helpers_implemented:
|
||||
if not sitl_replay_ready:
|
||||
pytest.skip(
|
||||
"FT-P-01 still-image push requires runner.helpers.{frame_source_replay,"
|
||||
"sitl_observer} — currently AZ-441 + AZ-416/AZ-417 leftovers. "
|
||||
"Pure-logic ACs covered by e2e/_unit_tests/helpers/test_accuracy_evaluator.py."
|
||||
"FT-P-01 still-image push requires `E2E_SITL_REPLAY_DIR` to point "
|
||||
"at a prepared SITL replay fixture (AZ-595). Pure-logic ACs "
|
||||
"covered by e2e/_unit_tests/helpers/test_accuracy_evaluator.py."
|
||||
)
|
||||
|
||||
from runner.helpers import frame_source_replay, sitl_observer
|
||||
|
||||
@@ -23,16 +23,16 @@ What this file does NOT own:
|
||||
(still a stub; AZ-408 was about the synthetic-injection injectors,
|
||||
not the video replayer); the scenario is marked
|
||||
``@pytest.mark.deferred_ac(reason=...)`` until that helper lands.
|
||||
* The FDR-archive iteration → ``runner.helpers.fdr_reader`` (owned by
|
||||
AZ-441); same skip gate.
|
||||
* The FDR-archive iteration → ``runner.helpers.fdr_reader`` (AZ-594,
|
||||
landed in batch 74) — the scenario still depends on a prepared SITL
|
||||
replay fixture (AZ-595) that produces the per-run FDR archive.
|
||||
* The MAVLink ``GLOBAL_POSITION_INT`` GT replay → handled by the
|
||||
``imu_replay`` helper which currently raises NotImplementedError
|
||||
(owned by AZ-407 in spec, but the helper file was not touched by
|
||||
the AZ-407 batch).
|
||||
``imu_replay`` helper (AZ-594, landed in batch 74).
|
||||
|
||||
When all three upstream helpers land, this file's runtime path activates
|
||||
automatically — the skip is keyed off the ``NotImplementedError`` from
|
||||
the helper imports, not off a hard-coded marker.
|
||||
When ``E2E_SITL_REPLAY_DIR`` is set and points at a prepared SITL
|
||||
replay fixture, this file's runtime path activates automatically; until
|
||||
then the scenario skips via the shared `sitl_replay_ready` fixture
|
||||
(AZ-595).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -44,53 +44,6 @@ import pytest
|
||||
from runner.helpers import anchor_pair_detector as apd
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def _harness_helpers_implemented() -> bool:
|
||||
"""True iff every upstream helper FT-P-02 needs has a real impl.
|
||||
|
||||
Used to gate the full-replay scenarios. Helper-level NotImplementedError
|
||||
is the signal — we don't hard-code a "deferred until task X" marker
|
||||
because then a developer who lands the helper would have to also
|
||||
remember to flip the marker. The auto-detect pattern is also what
|
||||
other downstream scenarios will reuse.
|
||||
"""
|
||||
from runner.helpers import fdr_reader, frame_source_replay, imu_replay
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
try:
|
||||
# The cheapest sentinel for each helper:
|
||||
# - FrameSourceReplayer.replay_video raises NotImplementedError
|
||||
# - fdr_reader.iter_records raises NotImplementedError
|
||||
# - ImuReplayer.replay raises NotImplementedError
|
||||
# We check by inspecting __doc__ / source rather than calling, so
|
||||
# the gate stays cheap.
|
||||
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
|
||||
try:
|
||||
replayer.replay_video(Path("/tmp/non-existent.mp4"))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
imu_replay.ImuReplayer(emitter=_NullImuEmitter()).replay(Path("/tmp/non-existent.csv")) # type: ignore[arg-type]
|
||||
except NotImplementedError:
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class _NullSink:
|
||||
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class _NullImuEmitter:
|
||||
def emit(self, sample: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
@pytest.mark.traces_to("AC-1.3,AC-1,AC-2,AC-3,AC-4,AC-5")
|
||||
def test_ft_p_02_derkachi_drift(
|
||||
fc_adapter: str,
|
||||
@@ -98,7 +51,7 @@ def test_ft_p_02_derkachi_drift(
|
||||
evidence_dir, # type: ignore[no-untyped-def]
|
||||
run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
_harness_helpers_implemented: bool,
|
||||
sitl_replay_ready: bool,
|
||||
) -> None:
|
||||
"""Full FT-P-02 scenario (AC-1.3). See module docstring.
|
||||
|
||||
@@ -110,11 +63,11 @@ def test_ft_p_02_derkachi_drift(
|
||||
AC-4: bin medians monotonic with age — covered by check_monotonic().
|
||||
AC-5: parametrized across (fc_adapter, vio_strategy).
|
||||
"""
|
||||
if not _harness_helpers_implemented:
|
||||
if not sitl_replay_ready:
|
||||
pytest.skip(
|
||||
"FT-P-02 full replay requires runner.helpers.{frame_source_replay,"
|
||||
"fdr_reader,imu_replay} — currently AZ-441 / AZ-407 leftovers. "
|
||||
"Pure-logic ACs covered by e2e/_unit_tests/helpers/test_anchor_pair_detector.py."
|
||||
"FT-P-02 full replay requires `E2E_SITL_REPLAY_DIR` to point at a "
|
||||
"prepared SITL replay fixture (AZ-595). Pure-logic ACs covered by "
|
||||
"e2e/_unit_tests/helpers/test_anchor_pair_detector.py."
|
||||
)
|
||||
|
||||
# Once the helpers land, the body below activates. We keep it
|
||||
|
||||
@@ -35,53 +35,20 @@ import pytest
|
||||
from runner.helpers import estimate_schema
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def _harness_helpers_implemented() -> bool:
|
||||
"""Same gate as FT-P-02: are frame replay + SITL observer + sidechannel
|
||||
decoders all real? If not, skip the docker-bound runtime path.
|
||||
"""
|
||||
from runner.helpers import frame_source_replay, mavproxy_tlog_reader, sitl_observer
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
try:
|
||||
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
|
||||
try:
|
||||
replayer.replay_image_directory(Path("/tmp/non-existent"))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
sitl_observer.get_observer("ardupilot", "test-host")
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
list(mavproxy_tlog_reader.iter_messages(Path("/tmp/non-existent.tlog")))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class _NullSink:
|
||||
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
@pytest.mark.traces_to("AC-1.4,AC-4.3")
|
||||
def test_schema_and_source_label(
|
||||
fc_adapter: str,
|
||||
vio_strategy: str,
|
||||
evidence_dir, # type: ignore[no-untyped-def]
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
_harness_helpers_implemented: bool,
|
||||
sitl_replay_ready: bool,
|
||||
) -> None:
|
||||
"""FT-P-03: schema completeness (AC-1) + source-label set containment (AC-2)."""
|
||||
if not _harness_helpers_implemented:
|
||||
if not sitl_replay_ready:
|
||||
pytest.skip(
|
||||
"FT-P-03 single-image push requires runner.helpers.{frame_source_replay,"
|
||||
"sitl_observer,mavproxy_tlog_reader} — currently pending AZ-407 / "
|
||||
"AZ-416/417 leftovers. Pure-logic ACs covered by "
|
||||
"e2e/_unit_tests/helpers/test_estimate_schema.py."
|
||||
"FT-P-03 single-image push requires `E2E_SITL_REPLAY_DIR` to point "
|
||||
"at a prepared SITL replay fixture (AZ-595). Pure-logic ACs "
|
||||
"covered by e2e/_unit_tests/helpers/test_estimate_schema.py."
|
||||
)
|
||||
|
||||
record, source_label = _push_single_image_and_observe(fc_adapter, vio_strategy)
|
||||
@@ -109,13 +76,14 @@ def test_wgs84_coordinate_range(
|
||||
vio_strategy: str,
|
||||
evidence_dir, # type: ignore[no-untyped-def]
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
_harness_helpers_implemented: bool,
|
||||
sitl_replay_ready: bool,
|
||||
) -> None:
|
||||
"""FT-P-14: decoded lat/lon inside WGS84 bounds (AC-3)."""
|
||||
if not _harness_helpers_implemented:
|
||||
if not sitl_replay_ready:
|
||||
pytest.skip(
|
||||
"FT-P-14 single-image push requires the same upstream helpers as FT-P-03. "
|
||||
"Pure-logic AC covered by e2e/_unit_tests/helpers/test_estimate_schema.py."
|
||||
"FT-P-14 single-image push requires `E2E_SITL_REPLAY_DIR` to point "
|
||||
"at a prepared SITL replay fixture (AZ-595). Pure-logic AC covered "
|
||||
"by e2e/_unit_tests/helpers/test_estimate_schema.py."
|
||||
)
|
||||
|
||||
record, _label = _push_single_image_and_observe(fc_adapter, vio_strategy)
|
||||
@@ -141,7 +109,7 @@ def _push_single_image_and_observe(fc_adapter: str, vio_strategy: str): # type:
|
||||
"""Push AD000001.jpg through the SUT and return (outbound_record, source_label).
|
||||
|
||||
Stub until runner.helpers.{frame_source_replay,sitl_observer,mavproxy_tlog_reader}
|
||||
land; the scenario test's skip gate (``_harness_helpers_implemented``)
|
||||
land; the scenario test's `sitl_replay_ready` skip gate (AZ-595)
|
||||
keeps this from executing prematurely.
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
|
||||
@@ -29,9 +29,10 @@ What this file does NOT own:
|
||||
* The FDR-archive iteration → ``runner.helpers.fdr_reader`` (stub;
|
||||
AZ-441) — skip-gated.
|
||||
|
||||
When all three upstream helpers land, this file's runtime path activates
|
||||
automatically — the skip is keyed off the ``NotImplementedError`` from
|
||||
the helper imports, not off a hard-coded marker.
|
||||
When ``E2E_SITL_REPLAY_DIR`` is set and points at a prepared SITL
|
||||
replay fixture, this file's runtime path activates automatically; until
|
||||
then the scenario skips via the shared `sitl_replay_ready` fixture
|
||||
(AZ-595).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -53,41 +54,6 @@ DERKACHI_IMU_CSV = DERKACHI_DIR / "data_imu.csv"
|
||||
DERKACHI_MP4 = DERKACHI_DIR / "flight_derkachi.mp4"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def _harness_helpers_implemented() -> bool:
|
||||
"""True iff every upstream helper FT-P-04 needs has a real impl."""
|
||||
from runner.helpers import fdr_reader, frame_source_replay, imu_replay
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
try:
|
||||
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
|
||||
try:
|
||||
replayer.replay_video(Path("/tmp/non-existent.mp4"))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
imu_replay.ImuReplayer(emitter=_NullImuEmitter()).replay(Path("/tmp/non-existent.csv")) # type: ignore[arg-type]
|
||||
except NotImplementedError:
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class _NullSink:
|
||||
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class _NullImuEmitter:
|
||||
def emit(self, sample: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
@pytest.mark.traces_to("AC-2.1a,AC-1,AC-2,AC-3,AC-4")
|
||||
def test_ft_p_04_derkachi_f2f_registration(
|
||||
fc_adapter: str,
|
||||
@@ -95,7 +61,7 @@ def test_ft_p_04_derkachi_f2f_registration(
|
||||
evidence_dir, # type: ignore[no-untyped-def]
|
||||
run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
_harness_helpers_implemented: bool,
|
||||
sitl_replay_ready: bool,
|
||||
) -> None:
|
||||
"""Full FT-P-04 scenario.
|
||||
|
||||
@@ -105,11 +71,11 @@ def test_ft_p_04_derkachi_f2f_registration(
|
||||
AC-3: sharp-turn frames excluded from the denominator.
|
||||
AC-4: parametrized across ``(fc_adapter, vio_strategy)``.
|
||||
"""
|
||||
if not _harness_helpers_implemented:
|
||||
if not sitl_replay_ready:
|
||||
pytest.skip(
|
||||
"FT-P-04 full replay requires runner.helpers.{frame_source_replay,"
|
||||
"imu_replay,fdr_reader} — currently AZ-441 / AZ-407 leftovers. "
|
||||
"Pure-logic ACs covered by e2e/_unit_tests/helpers/test_registration_classifier.py."
|
||||
"FT-P-04 full replay requires `E2E_SITL_REPLAY_DIR` to point at a "
|
||||
"prepared SITL replay fixture (AZ-595). Pure-logic ACs covered by "
|
||||
"e2e/_unit_tests/helpers/test_registration_classifier.py."
|
||||
)
|
||||
|
||||
from runner.helpers import fdr_reader, imu_replay
|
||||
|
||||
@@ -43,36 +43,6 @@ GT_CSV = Path(__file__).resolve().parents[3] / "_docs" / "00_problem" / "input_d
|
||||
STILL_IMAGES_DIR = GT_CSV.parent
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def _harness_helpers_implemented() -> bool:
|
||||
"""True iff replay + SITL observation + FDR helpers are all real."""
|
||||
from runner.helpers import fdr_reader, frame_source_replay, sitl_observer
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
try:
|
||||
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
|
||||
try:
|
||||
replayer.replay_image_directory(Path("/tmp/non-existent"))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
sitl_observer.get_observer(fc_adapter="ardupilot", host="sitl-ardupilot")
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class _NullSink:
|
||||
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
@pytest.mark.traces_to("AC-2.1b,AC-1,AC-2,AC-3,AC-5")
|
||||
def test_ft_p_05_sat_anchor(
|
||||
fc_adapter: str,
|
||||
@@ -80,7 +50,7 @@ def test_ft_p_05_sat_anchor(
|
||||
evidence_dir, # type: ignore[no-untyped-def]
|
||||
run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
_harness_helpers_implemented: bool,
|
||||
sitl_replay_ready: bool,
|
||||
) -> None:
|
||||
"""Full FT-P-05 scenario.
|
||||
|
||||
@@ -89,11 +59,11 @@ def test_ft_p_05_sat_anchor(
|
||||
AC-3: ≥80 % within 50 m AND ≥50 % within 20 m (same image set as FT-P-01).
|
||||
AC-5: parametrized across ``(fc_adapter, vio_strategy)``.
|
||||
"""
|
||||
if not _harness_helpers_implemented:
|
||||
if not sitl_replay_ready:
|
||||
pytest.skip(
|
||||
"FT-P-05 still-image push requires runner.helpers.{frame_source_replay,"
|
||||
"sitl_observer,fdr_reader} — currently AZ-441 + AZ-416/AZ-417 leftovers. "
|
||||
"Pure-logic ACs covered by e2e/_unit_tests/helpers/test_mre_evaluator.py."
|
||||
"FT-P-05 still-image push requires `E2E_SITL_REPLAY_DIR` to point "
|
||||
"at a prepared SITL replay fixture (AZ-595). Pure-logic ACs "
|
||||
"covered by e2e/_unit_tests/helpers/test_mre_evaluator.py."
|
||||
)
|
||||
|
||||
from runner.helpers import fdr_reader, frame_source_replay, sitl_observer
|
||||
|
||||
@@ -46,41 +46,6 @@ DERKACHI_IMU_CSV = DERKACHI_DIR / "data_imu.csv"
|
||||
DERKACHI_MP4 = DERKACHI_DIR / "flight_derkachi.mp4"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def _harness_helpers_implemented() -> bool:
|
||||
"""True iff replay + IMU + FDR helpers are real."""
|
||||
from runner.helpers import fdr_reader, imu_replay
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
try:
|
||||
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
|
||||
try:
|
||||
replayer.replay_video(Path("/tmp/non-existent.mp4"))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
imu_replay.ImuReplayer(emitter=_NullImuEmitter()).replay(Path("/tmp/non-existent.csv")) # type: ignore[arg-type]
|
||||
except NotImplementedError:
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class _NullSink:
|
||||
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class _NullImuEmitter:
|
||||
def emit(self, sample: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
@pytest.mark.traces_to("AC-3.2,AC-1,AC-4,AC-5,AC-6,AC-7")
|
||||
def test_ft_p_07_sharp_turn_recovery(
|
||||
fc_adapter: str,
|
||||
@@ -88,14 +53,13 @@ def test_ft_p_07_sharp_turn_recovery(
|
||||
evidence_dir, # type: ignore[no-untyped-def]
|
||||
run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
_harness_helpers_implemented: bool,
|
||||
sitl_replay_ready: bool,
|
||||
) -> None:
|
||||
if not _harness_helpers_implemented:
|
||||
if not sitl_replay_ready:
|
||||
pytest.skip(
|
||||
"FT-P-07 full replay requires runner.helpers.{frame_source_replay,"
|
||||
"imu_replay,fdr_reader} — currently AZ-441 / AZ-407 leftovers. "
|
||||
"AC-1/AC-4/AC-5/AC-6 helper logic covered by "
|
||||
"e2e/_unit_tests/helpers/test_sharp_turn_detector.py."
|
||||
"FT-P-07 full replay requires `E2E_SITL_REPLAY_DIR` to point at a "
|
||||
"prepared SITL replay fixture (AZ-595). AC-1/AC-4/AC-5/AC-6 helper "
|
||||
"logic covered by e2e/_unit_tests/helpers/test_sharp_turn_detector.py."
|
||||
)
|
||||
|
||||
from runner.helpers import fdr_reader
|
||||
|
||||
@@ -38,32 +38,6 @@ import pytest
|
||||
from runner.helpers import multi_segment_evaluator as mse
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def _harness_helpers_implemented() -> bool:
|
||||
"""True iff replay + FDR helpers are real."""
|
||||
from runner.helpers import fdr_reader, frame_source_replay
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
try:
|
||||
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
|
||||
try:
|
||||
replayer.replay_image_directory(Path("/tmp/non-existent"))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class _NullSink:
|
||||
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
@pytest.mark.traces_to("AC-3.3,AC-1,AC-2,AC-3,AC-4,AC-5")
|
||||
def test_ft_p_08_multi_segment_reloc(
|
||||
fc_adapter: str,
|
||||
@@ -72,7 +46,7 @@ def test_ft_p_08_multi_segment_reloc(
|
||||
run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
multi_segment_derkachi, # type: ignore[no-untyped-def] # AZ-408 pytest fixture
|
||||
_harness_helpers_implemented: bool,
|
||||
sitl_replay_ready: bool,
|
||||
) -> None:
|
||||
"""Full FT-P-08 scenario.
|
||||
|
||||
@@ -82,11 +56,11 @@ def test_ft_p_08_multi_segment_reloc(
|
||||
AC-4: trajectory continuity ≤100 m at each recovery.
|
||||
AC-5: parameterised across ``(fc_adapter, vio_strategy)``.
|
||||
"""
|
||||
if not _harness_helpers_implemented:
|
||||
if not sitl_replay_ready:
|
||||
pytest.skip(
|
||||
"FT-P-08 multi-segment replay requires runner.helpers.{frame_source_replay,"
|
||||
"fdr_reader} — currently AZ-441 leftover. Pure-logic ACs covered by "
|
||||
"e2e/_unit_tests/helpers/test_multi_segment_evaluator.py."
|
||||
"FT-P-08 multi-segment replay requires `E2E_SITL_REPLAY_DIR` to "
|
||||
"point at a prepared SITL replay fixture (AZ-595). Pure-logic ACs "
|
||||
"covered by e2e/_unit_tests/helpers/test_multi_segment_evaluator.py."
|
||||
)
|
||||
|
||||
from runner.helpers import fdr_reader
|
||||
|
||||
@@ -55,36 +55,6 @@ MAVLINK_PASSKEY_FIXTURE = (
|
||||
REPLAY_WINDOW_S = 60
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def _ap_harness_implemented() -> bool:
|
||||
"""True iff frame_source_replay + sitl_observer AP-side leg are real."""
|
||||
from runner.helpers import sitl_observer
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
try:
|
||||
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
|
||||
try:
|
||||
replayer.replay_video(Path("/tmp/non-existent.mp4"))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
sitl_observer.capture_ap_tlog(host="ardupilot-sitl", duration_s=0.01)
|
||||
except (NotImplementedError, AttributeError):
|
||||
return False
|
||||
try:
|
||||
sitl_observer.read_ap_parameter(host="ardupilot-sitl", name="EK3_SRC1_POSXY")
|
||||
except (NotImplementedError, AttributeError):
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class _NullSink:
|
||||
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
@pytest.mark.traces_to("AC-4.3,AC-1,AC-2,AC-3,AC-4,AC-5,D-C8-9")
|
||||
def test_ft_p_09_ap_signing(
|
||||
vio_strategy: str,
|
||||
@@ -92,7 +62,7 @@ def test_ft_p_09_ap_signing(
|
||||
run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
request, # type: ignore[no-untyped-def]
|
||||
_ap_harness_implemented: bool,
|
||||
sitl_replay_ready: bool,
|
||||
) -> None:
|
||||
"""Full FT-P-09-AP scenario; parameterized per vio_strategy."""
|
||||
fc_adapter = request.getfixturevalue("fc_adapter")
|
||||
@@ -105,12 +75,11 @@ def test_ft_p_09_ap_signing(
|
||||
"AZ-407 / AZ-408 owns the on-disk fixture."
|
||||
)
|
||||
|
||||
if not _ap_harness_implemented:
|
||||
if not sitl_replay_ready:
|
||||
pytest.skip(
|
||||
"FT-P-09-AP full scenario requires runner.helpers.{frame_source_replay,"
|
||||
"sitl_observer.capture_ap_tlog,sitl_observer.read_ap_parameter} — "
|
||||
"currently AZ-441 / AZ-407 leftovers. Pure-logic AC-1..AC-4 covered by "
|
||||
"e2e/_unit_tests/helpers/test_ap_contract_evaluator.py."
|
||||
"FT-P-09-AP full scenario requires `E2E_SITL_REPLAY_DIR` to point "
|
||||
"at a prepared SITL replay fixture (AZ-595). Pure-logic AC-1..AC-4 "
|
||||
"covered by e2e/_unit_tests/helpers/test_ap_contract_evaluator.py."
|
||||
)
|
||||
|
||||
from runner.helpers import sitl_observer
|
||||
|
||||
@@ -45,32 +45,6 @@ REPLAY_WINDOW_S = 60
|
||||
TCP_HANDSHAKE_BUDGET_S = 5
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def _inav_harness_implemented() -> bool:
|
||||
"""True iff frame_source_replay + sitl_observer iNav leg are real."""
|
||||
from runner.helpers import sitl_observer
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
try:
|
||||
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
|
||||
try:
|
||||
replayer.replay_video(Path("/tmp/non-existent.mp4"))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
sitl_observer.observe_inav_tcp_handshake(host="inav-sitl", port=5760, timeout_s=0.01)
|
||||
except (NotImplementedError, AttributeError):
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class _NullSink:
|
||||
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
@pytest.mark.traces_to("AC-4.3,AC-1,AC-2,AC-3,AC-4")
|
||||
def test_ft_p_09_inav(
|
||||
vio_strategy: str,
|
||||
@@ -78,7 +52,7 @@ def test_ft_p_09_inav(
|
||||
run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
request, # type: ignore[no-untyped-def]
|
||||
_inav_harness_implemented: bool,
|
||||
sitl_replay_ready: bool,
|
||||
) -> None:
|
||||
"""Full FT-P-09-iNav scenario; parameterized per vio_strategy.
|
||||
|
||||
@@ -90,12 +64,11 @@ def test_ft_p_09_inav(
|
||||
if fc_adapter != "inav":
|
||||
pytest.skip("FT-P-09-iNav is iNav-only; ardupilot variant is FT-P-09-AP (AZ-416)")
|
||||
|
||||
if not _inav_harness_implemented:
|
||||
if not sitl_replay_ready:
|
||||
pytest.skip(
|
||||
"FT-P-09-iNav full scenario requires runner.helpers.{frame_source_replay,"
|
||||
"sitl_observer.observe_inav_tcp_handshake} — currently AZ-441 / AZ-407 leftovers. "
|
||||
"Pure-logic AC-2/AC-3 covered by "
|
||||
"e2e/_unit_tests/helpers/test_msp_frame_observer.py."
|
||||
"FT-P-09-iNav full scenario requires `E2E_SITL_REPLAY_DIR` to "
|
||||
"point at a prepared SITL replay fixture (AZ-595). Pure-logic "
|
||||
"AC-2/AC-3 covered by e2e/_unit_tests/helpers/test_msp_frame_observer.py."
|
||||
)
|
||||
|
||||
from runner.helpers import sitl_observer
|
||||
|
||||
@@ -49,41 +49,6 @@ DERKACHI_IMU_CSV = DERKACHI_DIR / "data_imu.csv"
|
||||
DERKACHI_MP4 = DERKACHI_DIR / "flight_derkachi.mp4"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def _harness_helpers_implemented() -> bool:
|
||||
"""True iff replay + IMU + FDR helpers are real."""
|
||||
from runner.helpers import fdr_reader, frame_source_replay, imu_replay
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
try:
|
||||
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
|
||||
try:
|
||||
replayer.replay_video(Path("/tmp/non-existent.mp4"))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
imu_replay.ImuReplayer(emitter=_NullImuEmitter()).replay(Path("/tmp/non-existent.csv")) # type: ignore[arg-type]
|
||||
except NotImplementedError:
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class _NullSink:
|
||||
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class _NullImuEmitter:
|
||||
def emit(self, sample: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _load_derkachi_gt_track() -> list[se.GtPose]:
|
||||
"""Read GLOBAL_POSITION_INT poses from data_imu.csv.
|
||||
|
||||
@@ -115,7 +80,7 @@ def test_ft_p_10_smoothing_lookback(
|
||||
evidence_dir, # type: ignore[no-untyped-def]
|
||||
run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
_harness_helpers_implemented: bool,
|
||||
sitl_replay_ready: bool,
|
||||
) -> None:
|
||||
"""Full FT-P-10 scenario.
|
||||
|
||||
@@ -125,11 +90,11 @@ def test_ft_p_10_smoothing_lookback(
|
||||
AC-3: mean_improvement_m ≥ 5 m.
|
||||
AC-4: parameterised across ``(fc_adapter, vio_strategy)``.
|
||||
"""
|
||||
if not _harness_helpers_implemented:
|
||||
if not sitl_replay_ready:
|
||||
pytest.skip(
|
||||
"FT-P-10 full replay requires runner.helpers.{frame_source_replay,"
|
||||
"imu_replay,fdr_reader} — currently AZ-441 / AZ-407 leftovers. "
|
||||
"Pure-logic ACs covered by e2e/_unit_tests/helpers/test_smoothing_evaluator.py."
|
||||
"FT-P-10 full replay requires `E2E_SITL_REPLAY_DIR` to point at a "
|
||||
"prepared SITL replay fixture (AZ-595). Pure-logic ACs covered by "
|
||||
"e2e/_unit_tests/helpers/test_smoothing_evaluator.py."
|
||||
)
|
||||
|
||||
from runner.helpers import fdr_reader, imu_replay
|
||||
|
||||
@@ -51,45 +51,6 @@ COLD_BOOT_FIXTURE = (
|
||||
OPERATOR_ORIGIN = cse.LatLonAlt(lat_deg=50.0, lon_deg=36.2, alt_m=200.0)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def _cold_start_harness_implemented() -> bool:
|
||||
"""True iff frame_source_replay + sitl_observer + fdr_reader are real.
|
||||
|
||||
Cold start adds two specific SITL-observer surfaces beyond the
|
||||
common replay path: ``prepare_sitl_cold_boot`` (parameter-load
|
||||
path) and ``prepare_sitl_no_gps`` (``SIM_GPS_DISABLE = 1``).
|
||||
"""
|
||||
from runner.helpers import fdr_reader, sitl_observer
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
try:
|
||||
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
|
||||
try:
|
||||
replayer.replay_video(Path("/tmp/non-existent.mp4"))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
sitl_observer.prepare_sitl_cold_boot(host="ardupilot-sitl", fixture_path=COLD_BOOT_FIXTURE)
|
||||
except (NotImplementedError, AttributeError):
|
||||
return False
|
||||
try:
|
||||
sitl_observer.prepare_sitl_no_gps(host="ardupilot-sitl")
|
||||
except (NotImplementedError, AttributeError):
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class _NullSink:
|
||||
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _cold_run_id(run_id: str) -> str:
|
||||
"""Return a fresh run_id — Cold-start REQUIRES an empty fdr-output volume.
|
||||
@@ -116,16 +77,14 @@ def test_ft_p_11_cold_start_origin_variants(
|
||||
_cold_run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
tmp_path: Path,
|
||||
_cold_start_harness_implemented: bool,
|
||||
sitl_replay_ready: bool,
|
||||
) -> None:
|
||||
"""FT-P-11 AC-1 / AC-2 / AC-4 across the three origin_source variants."""
|
||||
if not _cold_start_harness_implemented:
|
||||
if not sitl_replay_ready:
|
||||
pytest.skip(
|
||||
"FT-P-11 full scenario requires runner.helpers.{frame_source_replay,"
|
||||
"fdr_reader,sitl_observer.prepare_sitl_cold_boot,"
|
||||
"sitl_observer.prepare_sitl_no_gps} — currently AZ-441 / AZ-407 "
|
||||
"leftovers. Pure-logic AC-1/2/3/4 covered by "
|
||||
"e2e/_unit_tests/helpers/test_cold_start_evaluator.py."
|
||||
"FT-P-11 full scenario requires `E2E_SITL_REPLAY_DIR` to point at a "
|
||||
"prepared SITL replay fixture (AZ-595). Pure-logic AC-1/2/3/4 "
|
||||
"covered by e2e/_unit_tests/helpers/test_cold_start_evaluator.py."
|
||||
)
|
||||
|
||||
from runner.helpers import fdr_reader, sitl_observer
|
||||
@@ -241,15 +200,14 @@ def test_ft_p_11_cold_start_no_origin_aborts(
|
||||
_cold_run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
tmp_path: Path,
|
||||
_cold_start_harness_implemented: bool,
|
||||
sitl_replay_ready: bool,
|
||||
) -> None:
|
||||
"""AC-3: Manifest empty + SITL no GPS → SUT MUST refuse takeoff."""
|
||||
if not _cold_start_harness_implemented:
|
||||
if not sitl_replay_ready:
|
||||
pytest.skip(
|
||||
"FT-P-11 AC-3 full scenario requires runner.helpers.{frame_source_replay,"
|
||||
"fdr_reader,sitl_observer.prepare_sitl_no_gps} — currently AZ-441 / "
|
||||
"AZ-407 leftovers. Pure-logic AC-3 covered by "
|
||||
"e2e/_unit_tests/helpers/test_cold_start_evaluator.py."
|
||||
"FT-P-11 AC-3 full scenario requires `E2E_SITL_REPLAY_DIR` to point "
|
||||
"at a prepared SITL replay fixture (AZ-595). Pure-logic AC-3 "
|
||||
"covered by e2e/_unit_tests/helpers/test_cold_start_evaluator.py."
|
||||
)
|
||||
|
||||
from runner.helpers import fdr_reader, sitl_observer
|
||||
|
||||
Reference in New Issue
Block a user