mirror of
https://github.com/azaion/gps-denied-onboard.git
synced 2026-06-22 13:01:14 +00:00
[AZ-416] [AZ-417] [AZ-419] Test batch 72: FT-P-09 AP/iNav + FT-P-11 cold start
- AZ-416 (FT-P-09-AP): fills mavproxy_tlog_reader.iter_messages with pymavlink body (AZ-406 surface kept); adds ap_contract_evaluator covering AC-1 (signing handshake <=5s), AC-2 (GPS_INPUT >=4.5 Hz), AC-3 (EK3_SRC1_POSXY=3), AC-4 (GPS_RAW_INT health >=80%); scenario forces fc_adapter=ardupilot. - AZ-417 (FT-P-09-iNav): msp_frame_observer covering AC-2 (MSP rate) and AC-3 (fix_type/provider/numSat); scenario forces fc_adapter=inav. - AZ-419 (FT-P-11): cold_start_evaluator covering AC-1 (operator manifest origin), AC-2 (FC EKF fallback), AC-3 (no-origin abort), AC-4 (bounded-delta conflict, ADR-010 Principle #11 amended); scenario parametrized on origin_source plus dedicated no-origin abort scenario. - All scenarios skip-gated on upstream frame_source_replay / imu_replay / fdr_reader / sitl_observer extensions. - +67 unit tests; full e2e unit suite: 460 passed. - K=3 cumulative review fired: PASS for batches 70-72. See _docs/03_implementation/batch_72_report.md, _docs/03_implementation/reviews/batch_72_review.md, _docs/03_implementation/cumulative_review_batches_70-72_cycle1_report.md. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -0,0 +1,184 @@
|
||||
"""FT-P-09-AP — ArduPilot GPS_INPUT contract + MAVLink 2.0 signing (AZ-416 / AC-4.3).
|
||||
|
||||
The full scenario:
|
||||
|
||||
1. Force ``fc_adapter=ardupilot``; load ``mavlink-test-passkey.txt``
|
||||
as the docker secret feeding the SUT signing channel.
|
||||
2. Start the SUT against the ArduPilot SITL container; mavproxy-listener
|
||||
captures the wire traffic to a ``.tlog``.
|
||||
3. AC-1: parse the ``.tlog``; first signed frame must arrive within
|
||||
≤5 s of the first observed message; no ``BAD_SIGNATURE`` STATUSTEXT
|
||||
in that window.
|
||||
4. Replay 60 s of Derkachi through the SUT (signed GPS_INPUT flow).
|
||||
5. AC-2: GPS_INPUT cadence over the full ``.tlog`` ≥4.5 Hz.
|
||||
6. AC-3: ``EK3_SRC1_POSXY`` (read via mavproxy parameter request) ==
|
||||
3 (GPS source).
|
||||
7. AC-4: GPS_RAW_INT health (``fix_type ≥ 3`` AND ``eph ≤ 200``)
|
||||
for ≥80 % of the window.
|
||||
8. AC-5: parameterised per ``vio_strategy`` (``fc_adapter`` fixed to
|
||||
``ardupilot``).
|
||||
|
||||
Gated on:
|
||||
* ``runner.helpers.frame_source_replay`` — owned by AZ-441
|
||||
* ``runner.helpers.sitl_observer`` — owned by AZ-407 (AP-side leg
|
||||
``capture_ap_tlog`` + ``read_ap_parameter``)
|
||||
|
||||
Pure-logic AC-1/AC-2/AC-3/AC-4 coverage lives in
|
||||
``e2e/_unit_tests/helpers/test_ap_contract_evaluator.py`` and
|
||||
``e2e/_unit_tests/helpers/test_mavproxy_tlog_reader.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from runner.helpers import ap_contract_evaluator as ace
|
||||
from runner.helpers import mavproxy_tlog_reader as mtr
|
||||
|
||||
DERKACHI_DIR = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "_docs"
|
||||
/ "00_problem"
|
||||
/ "input_data"
|
||||
/ "flight_derkachi"
|
||||
)
|
||||
DERKACHI_MP4 = DERKACHI_DIR / "flight_derkachi.mp4"
|
||||
MAVLINK_PASSKEY_FIXTURE = (
|
||||
Path(__file__).resolve().parents[2]
|
||||
/ "fixtures"
|
||||
/ "secrets"
|
||||
/ "mavlink-test-passkey.txt"
|
||||
)
|
||||
|
||||
REPLAY_WINDOW_S = 60
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def _ap_harness_implemented() -> bool:
|
||||
"""True iff frame_source_replay + sitl_observer AP-side leg are real."""
|
||||
from runner.helpers import sitl_observer
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
try:
|
||||
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
|
||||
try:
|
||||
replayer.replay_video(Path("/tmp/non-existent.mp4"))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
sitl_observer.capture_ap_tlog(host="ardupilot-sitl", duration_s=0.01)
|
||||
except (NotImplementedError, AttributeError):
|
||||
return False
|
||||
try:
|
||||
sitl_observer.read_ap_parameter(host="ardupilot-sitl", name="EK3_SRC1_POSXY")
|
||||
except (NotImplementedError, AttributeError):
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class _NullSink:
|
||||
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
@pytest.mark.traces_to("AC-4.3,AC-1,AC-2,AC-3,AC-4,AC-5,D-C8-9")
|
||||
def test_ft_p_09_ap_signing(
|
||||
vio_strategy: str,
|
||||
evidence_dir, # type: ignore[no-untyped-def]
|
||||
run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
request, # type: ignore[no-untyped-def]
|
||||
_ap_harness_implemented: bool,
|
||||
) -> None:
|
||||
"""Full FT-P-09-AP scenario; parameterized per vio_strategy."""
|
||||
fc_adapter = request.getfixturevalue("fc_adapter")
|
||||
if fc_adapter != "ardupilot":
|
||||
pytest.skip("FT-P-09-AP is ArduPilot-only; iNav variant is FT-P-09-iNav (AZ-417)")
|
||||
|
||||
if not MAVLINK_PASSKEY_FIXTURE.exists():
|
||||
pytest.fail(
|
||||
f"mavlink-test-passkey fixture missing at {MAVLINK_PASSKEY_FIXTURE} — "
|
||||
"AZ-407 / AZ-408 owns the on-disk fixture."
|
||||
)
|
||||
|
||||
if not _ap_harness_implemented:
|
||||
pytest.skip(
|
||||
"FT-P-09-AP full scenario requires runner.helpers.{frame_source_replay,"
|
||||
"sitl_observer.capture_ap_tlog,sitl_observer.read_ap_parameter} — "
|
||||
"currently AZ-441 / AZ-407 leftovers. Pure-logic AC-1..AC-4 covered by "
|
||||
"e2e/_unit_tests/helpers/test_ap_contract_evaluator.py."
|
||||
)
|
||||
|
||||
from runner.helpers import sitl_observer
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
# 1. Drive replay (captures tlog continuously via mavproxy-listener).
|
||||
FrameSourceReplayer(_resolve_frame_sink()).replay_video(DERKACHI_MP4)
|
||||
tlog_path = sitl_observer.capture_ap_tlog(
|
||||
host="ardupilot-sitl", duration_s=REPLAY_WINDOW_S,
|
||||
)
|
||||
|
||||
# 2. Materialise the tlog ONCE (iter_messages is single-pass).
|
||||
msgs = ace.collect_messages_to_list(mtr.iter_messages(tlog_path))
|
||||
if not msgs:
|
||||
pytest.fail(f"FT-P-09-AP: empty tlog at {tlog_path}")
|
||||
|
||||
# 3. AC-1: signing handshake.
|
||||
handshake = ace.observe_signing_handshake(msgs)
|
||||
|
||||
# 4. AC-2: GPS_INPUT rate.
|
||||
rate = ace.compute_gps_input_rate(msgs)
|
||||
|
||||
# 5. AC-3: EK3_SRC1_POSXY param read.
|
||||
ek3_value = int(sitl_observer.read_ap_parameter(
|
||||
host="ardupilot-sitl", name="EK3_SRC1_POSXY"
|
||||
))
|
||||
ek3_ok = ace.validate_ek3_src1_posxy(ek3_value)
|
||||
|
||||
# 6. AC-4: GPS_RAW_INT health.
|
||||
health = ace.evaluate_gps_raw_int_health(msgs)
|
||||
|
||||
# 7. NFR metrics + assertions.
|
||||
if handshake.lag_s is not None:
|
||||
nfr_recorder.record_metric(
|
||||
"ft_p_09_ap.signing_handshake_s", handshake.lag_s, ac_id="AC-1"
|
||||
)
|
||||
nfr_recorder.record_metric(
|
||||
"ft_p_09_ap.gps_input_rate_hz", rate.observed_rate_hz, ac_id="AC-2"
|
||||
)
|
||||
nfr_recorder.record_metric(
|
||||
"ft_p_09_ap.ek3_src1_posxy", float(ek3_value), ac_id="AC-3"
|
||||
)
|
||||
nfr_recorder.record_metric(
|
||||
"ft_p_09_ap.gps_raw_int_healthy_fraction", health.healthy_fraction, ac_id="AC-4"
|
||||
)
|
||||
|
||||
assert handshake.passes, (
|
||||
f"AC-1 (signing handshake ≤{ace.HANDSHAKE_BUDGET_S} s, no BAD_SIGNATURE) failed: "
|
||||
f"first_signed_us={handshake.first_signed_us}, lag_s={handshake.lag_s}, "
|
||||
f"bad_signature_count={handshake.bad_signature_count}"
|
||||
)
|
||||
assert rate.passes, (
|
||||
f"AC-2 (GPS_INPUT ≥{ace.GPS_INPUT_MIN_RATE_HZ} Hz for "
|
||||
f"{ace.GPS_INPUT_TARGET_RATE_HZ} Hz target) failed: "
|
||||
f"observed_rate_hz={rate.observed_rate_hz:.3f}, frames={rate.frame_count}"
|
||||
)
|
||||
assert ek3_ok, (
|
||||
f"AC-3 (EK3_SRC1_POSXY = {ace.EK3_SRC1_POSXY_REQUIRED}) failed: got {ek3_value}"
|
||||
)
|
||||
assert health.passes, (
|
||||
f"AC-4 (GPS_RAW_INT healthy fraction ≥"
|
||||
f"{ace.GPS_RAW_INT_HEALTHY_FRACTION_REQUIRED:.0%}) failed: "
|
||||
f"observed={health.healthy_fraction:.4f}, "
|
||||
f"healthy={health.healthy_samples}/{health.total_samples}"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_frame_sink(): # type: ignore[no-untyped-def]
|
||||
raise NotImplementedError(
|
||||
"frame sink resolution is owned by AZ-441 / runner.helpers.frame_source_replay"
|
||||
)
|
||||
@@ -0,0 +1,171 @@
|
||||
"""FT-P-09-iNav — iNav MSP2_SENSOR_GPS contract conformance (AZ-417 / AC-4.3).
|
||||
|
||||
The full scenario:
|
||||
|
||||
1. Force ``fc_adapter=inav``; start the SUT against the iNav SITL
|
||||
container on ``inav-sitl:5760``.
|
||||
2. AC-1: probe the TCP connection establishment from the SUT side
|
||||
within ≤5 s (observable via the SITL observer's connection event).
|
||||
3. Replay 60 s of Derkachi through the SUT.
|
||||
4. AC-2: count MSP2_SENSOR_GPS (function ID 0x1F03) frame arrivals at
|
||||
iNav; assert ≥4.5 Hz observed.
|
||||
5. AC-3: query iNav GPS state via ``msp_gps_toy`` subprocess; assert
|
||||
``gpsSol.fixType ≥ 3``, ``provider = "MSP"``, ``gpsSol.numSat``
|
||||
matches the emitted value.
|
||||
6. AC-4: parameterise per ``vio_strategy`` (``fc_adapter`` fixed to
|
||||
``inav``).
|
||||
|
||||
Gated on:
|
||||
* ``runner.helpers.frame_source_replay`` — owned by AZ-441
|
||||
* ``runner.helpers.sitl_observer`` — owned by AZ-407 (iNav probe leg
|
||||
is part of the iNav-side `inav_msp_observer` follow-up)
|
||||
|
||||
Pure-logic AC-2/AC-3 coverage lives in
|
||||
``e2e/_unit_tests/helpers/test_msp_frame_observer.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from runner.helpers import msp_frame_observer as mfo
|
||||
|
||||
DERKACHI_DIR = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "_docs"
|
||||
/ "00_problem"
|
||||
/ "input_data"
|
||||
/ "flight_derkachi"
|
||||
)
|
||||
DERKACHI_MP4 = DERKACHI_DIR / "flight_derkachi.mp4"
|
||||
|
||||
REPLAY_WINDOW_S = 60
|
||||
TCP_HANDSHAKE_BUDGET_S = 5
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def _inav_harness_implemented() -> bool:
|
||||
"""True iff frame_source_replay + sitl_observer iNav leg are real."""
|
||||
from runner.helpers import sitl_observer
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
try:
|
||||
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
|
||||
try:
|
||||
replayer.replay_video(Path("/tmp/non-existent.mp4"))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
sitl_observer.observe_inav_tcp_handshake(host="inav-sitl", port=5760, timeout_s=0.01)
|
||||
except (NotImplementedError, AttributeError):
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class _NullSink:
|
||||
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
@pytest.mark.traces_to("AC-4.3,AC-1,AC-2,AC-3,AC-4")
|
||||
def test_ft_p_09_inav(
|
||||
vio_strategy: str,
|
||||
evidence_dir, # type: ignore[no-untyped-def]
|
||||
run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
request, # type: ignore[no-untyped-def]
|
||||
_inav_harness_implemented: bool,
|
||||
) -> None:
|
||||
"""Full FT-P-09-iNav scenario; parameterized per vio_strategy.
|
||||
|
||||
`fc_adapter` is FORCED to ``inav`` (AC-4) — the test skips on any
|
||||
other adapter so the conftest matrix doesn't double-run it under
|
||||
``ardupilot``.
|
||||
"""
|
||||
fc_adapter = request.getfixturevalue("fc_adapter")
|
||||
if fc_adapter != "inav":
|
||||
pytest.skip("FT-P-09-iNav is iNav-only; ardupilot variant is FT-P-09-AP (AZ-416)")
|
||||
|
||||
if not _inav_harness_implemented:
|
||||
pytest.skip(
|
||||
"FT-P-09-iNav full scenario requires runner.helpers.{frame_source_replay,"
|
||||
"sitl_observer.observe_inav_tcp_handshake} — currently AZ-441 / AZ-407 leftovers. "
|
||||
"Pure-logic AC-2/AC-3 covered by "
|
||||
"e2e/_unit_tests/helpers/test_msp_frame_observer.py."
|
||||
)
|
||||
|
||||
from runner.helpers import sitl_observer
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
# 1. AC-1: TCP handshake.
|
||||
handshake = sitl_observer.observe_inav_tcp_handshake(
|
||||
host="inav-sitl", port=5760, timeout_s=TCP_HANDSHAKE_BUDGET_S,
|
||||
)
|
||||
assert handshake.established_within_s is not None, (
|
||||
f"AC-1 (TCP connect ≤{TCP_HANDSHAKE_BUDGET_S} s) failed: no connection event"
|
||||
)
|
||||
assert handshake.established_within_s <= TCP_HANDSHAKE_BUDGET_S, (
|
||||
f"AC-1 (TCP connect ≤{TCP_HANDSHAKE_BUDGET_S} s) failed: "
|
||||
f"established_within_s={handshake.established_within_s}"
|
||||
)
|
||||
|
||||
# 2. Drive replay.
|
||||
FrameSourceReplayer(_resolve_frame_sink()).replay_video(DERKACHI_MP4)
|
||||
|
||||
# 3. Collect MSP frame arrivals from the iNav observer.
|
||||
capture = sitl_observer.collect_inav_msp_frames(
|
||||
host="inav-sitl", port=5760, window_s=REPLAY_WINDOW_S,
|
||||
)
|
||||
samples = [
|
||||
mfo.MspFrameSample(monotonic_ms=int(f.monotonic_ms), function_id=int(f.function_id))
|
||||
for f in capture.frames
|
||||
]
|
||||
|
||||
# 4. AC-2: rate.
|
||||
rate_report = mfo.compute_rate_hz(samples)
|
||||
|
||||
# 5. AC-3: iNav GPS state via msp_gps_toy.
|
||||
state = sitl_observer.query_inav_gps_state(host="inav-sitl")
|
||||
gps_report = mfo.evaluate_inav_gps_state(
|
||||
mfo.InavGpsSnapshot(
|
||||
fix_type=int(state.fix_type),
|
||||
num_sat=int(state.num_sat),
|
||||
provider=str(state.provider),
|
||||
),
|
||||
expected_num_sat=int(capture.expected_num_sat),
|
||||
)
|
||||
|
||||
# 6. NFR metrics + assertions.
|
||||
nfr_recorder.record_metric(
|
||||
"ft_p_09_inav.frame_count", float(rate_report.frame_count), ac_id="AC-2"
|
||||
)
|
||||
nfr_recorder.record_metric(
|
||||
"ft_p_09_inav.observed_rate_hz", rate_report.observed_rate_hz, ac_id="AC-2"
|
||||
)
|
||||
nfr_recorder.record_metric(
|
||||
"ft_p_09_inav.tcp_handshake_s", float(handshake.established_within_s), ac_id="AC-1"
|
||||
)
|
||||
nfr_recorder.record_metric(
|
||||
"ft_p_09_inav.fix_type", float(gps_report.snapshot.fix_type), ac_id="AC-3"
|
||||
)
|
||||
|
||||
assert rate_report.passes, (
|
||||
f"AC-2 (≥{mfo.MIN_OBSERVED_RATE_HZ} Hz for {mfo.DEFAULT_TARGET_RATE_HZ} Hz target) failed: "
|
||||
f"observed_rate_hz={rate_report.observed_rate_hz:.3f}, "
|
||||
f"frames={rate_report.frame_count}, window_ms={rate_report.window_ms}"
|
||||
)
|
||||
assert gps_report.passes, (
|
||||
f"AC-3 failed: fix_type_ok={gps_report.fix_type_ok}, "
|
||||
f"provider_ok={gps_report.provider_ok}, num_sat_ok={gps_report.num_sat_ok}; "
|
||||
f"snapshot={gps_report.snapshot}, expected_num_sat={gps_report.expected_num_sat}"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_frame_sink(): # type: ignore[no-untyped-def]
|
||||
raise NotImplementedError(
|
||||
"frame sink resolution is owned by AZ-441 / runner.helpers.frame_source_replay"
|
||||
)
|
||||
@@ -0,0 +1,309 @@
|
||||
"""FT-P-11 — Cold-start initialization (AZ-419 / ADR-010 / AC-5.1).
|
||||
|
||||
Three parametrized origin_source variants share one scenario module:
|
||||
|
||||
* ``operator_manifest`` (primary path, ADR-010 / AZ-490): Manifest
|
||||
carries ``flight.takeoff_origin = A``; SITL FC has NO valid GPS;
|
||||
SUT cold-starts; first outbound estimate within ±50 m of A;
|
||||
FDR has ``c5.cold_start_origin.set(source="manifest")``.
|
||||
* ``fc_ekf`` (secondary path, legacy AC-5.1): Manifest has no
|
||||
``takeoff_origin``; ``cold-boot-fixture`` JSON loaded into SITL;
|
||||
first outbound estimate within ±50 m of FC EKF snapshot;
|
||||
FDR has ``c5.cold_start_origin.set(source="fc_ekf")``.
|
||||
* ``bounded_delta_conflict`` (ADR-010 Principle #11 amended): Manifest
|
||||
carries ``takeoff_origin = A``; FC EKF reports B with
|
||||
``vincenty(A, B) > 200 m``; first outbound estimate within ±50 m of
|
||||
A; source_label is NOT ``satellite_anchored``; FDR has
|
||||
``c5.gps_bounded_delta.reject`` naming both A and B.
|
||||
|
||||
The fourth variant exercised by AC-3 (no origin available → SUT
|
||||
refuses takeoff) lives in a separate scenario function in the same
|
||||
module so the parametrize matrix for the other three stays clean.
|
||||
|
||||
Gated on the upstream replay + SITL observer + FDR helpers; pure
|
||||
logic is covered by
|
||||
``e2e/_unit_tests/helpers/test_cold_start_evaluator.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from runner.helpers import cold_start_evaluator as cse
|
||||
|
||||
DERKACHI_DIR = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "_docs"
|
||||
/ "00_problem"
|
||||
/ "input_data"
|
||||
/ "flight_derkachi"
|
||||
)
|
||||
DERKACHI_MP4 = DERKACHI_DIR / "flight_derkachi.mp4"
|
||||
COLD_BOOT_FIXTURE = (
|
||||
Path(__file__).resolve().parents[2]
|
||||
/ "fixtures"
|
||||
/ "cold-boot"
|
||||
/ "cold_boot_fixture.json"
|
||||
)
|
||||
|
||||
OPERATOR_ORIGIN = cse.LatLonAlt(lat_deg=50.0, lon_deg=36.2, alt_m=200.0)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def _cold_start_harness_implemented() -> bool:
|
||||
"""True iff frame_source_replay + sitl_observer + fdr_reader are real.
|
||||
|
||||
Cold start adds two specific SITL-observer surfaces beyond the
|
||||
common replay path: ``prepare_sitl_cold_boot`` (parameter-load
|
||||
path) and ``prepare_sitl_no_gps`` (``SIM_GPS_DISABLE = 1``).
|
||||
"""
|
||||
from runner.helpers import fdr_reader, sitl_observer
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
try:
|
||||
replayer = FrameSourceReplayer(sink=_NullSink()) # type: ignore[arg-type]
|
||||
try:
|
||||
replayer.replay_video(Path("/tmp/non-existent.mp4"))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
list(fdr_reader.iter_records(Path("/tmp/non-existent")))
|
||||
except NotImplementedError:
|
||||
return False
|
||||
try:
|
||||
sitl_observer.prepare_sitl_cold_boot(host="ardupilot-sitl", fixture_path=COLD_BOOT_FIXTURE)
|
||||
except (NotImplementedError, AttributeError):
|
||||
return False
|
||||
try:
|
||||
sitl_observer.prepare_sitl_no_gps(host="ardupilot-sitl")
|
||||
except (NotImplementedError, AttributeError):
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class _NullSink:
|
||||
def write_frame(self, jpeg_bytes: bytes, timestamp_ms: int) -> None:
|
||||
return None
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _cold_run_id(run_id: str) -> str:
|
||||
"""Return a fresh run_id — Cold-start REQUIRES an empty fdr-output volume.
|
||||
|
||||
The runner's ``run_id`` is per-invocation already, but cold-start
|
||||
additionally relies on the volume being empty. The actual volume
|
||||
wipe is part of the docker-compose lifecycle owned by AZ-407 and
|
||||
is therefore implicit in the scenario being skipped until the
|
||||
harness is real.
|
||||
"""
|
||||
return run_id
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"origin_source",
|
||||
["operator_manifest", "fc_ekf", "bounded_delta_conflict"],
|
||||
)
|
||||
@pytest.mark.traces_to("AC-5.1,AC-1,AC-2,AC-4,AC-5,ADR-010")
|
||||
def test_ft_p_11_cold_start_origin_variants(
|
||||
origin_source: str,
|
||||
fc_adapter: str,
|
||||
vio_strategy: str,
|
||||
evidence_dir, # type: ignore[no-untyped-def]
|
||||
_cold_run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
tmp_path: Path,
|
||||
_cold_start_harness_implemented: bool,
|
||||
) -> None:
|
||||
"""FT-P-11 AC-1 / AC-2 / AC-4 across the three origin_source variants."""
|
||||
if not _cold_start_harness_implemented:
|
||||
pytest.skip(
|
||||
"FT-P-11 full scenario requires runner.helpers.{frame_source_replay,"
|
||||
"fdr_reader,sitl_observer.prepare_sitl_cold_boot,"
|
||||
"sitl_observer.prepare_sitl_no_gps} — currently AZ-441 / AZ-407 "
|
||||
"leftovers. Pure-logic AC-1/2/3/4 covered by "
|
||||
"e2e/_unit_tests/helpers/test_cold_start_evaluator.py."
|
||||
)
|
||||
|
||||
from runner.helpers import fdr_reader, sitl_observer
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
# 1. Stage the fixture per variant.
|
||||
manifest_path = tmp_path / f"ft-p-11-{origin_source}-manifest.json"
|
||||
if origin_source == "operator_manifest":
|
||||
cse.write_manifest(manifest_path, OPERATOR_ORIGIN)
|
||||
sitl_observer.prepare_sitl_no_gps(host=f"{fc_adapter}-sitl")
|
||||
expected_origin = OPERATOR_ORIGIN
|
||||
elif origin_source == "fc_ekf":
|
||||
cse.write_manifest(manifest_path, None)
|
||||
snap = cse.read_cold_boot_fixture(COLD_BOOT_FIXTURE)
|
||||
sitl_observer.prepare_sitl_cold_boot(host=f"{fc_adapter}-sitl", fixture_path=COLD_BOOT_FIXTURE)
|
||||
expected_origin = cse.LatLonAlt(snap.lat_deg, snap.lon_deg, snap.alt_m)
|
||||
elif origin_source == "bounded_delta_conflict":
|
||||
cse.write_manifest(manifest_path, OPERATOR_ORIGIN)
|
||||
snap = cse.read_cold_boot_fixture(COLD_BOOT_FIXTURE)
|
||||
assert (
|
||||
cse.bounded_delta_distance_m(
|
||||
OPERATOR_ORIGIN,
|
||||
cse.LatLonAlt(snap.lat_deg, snap.lon_deg, snap.alt_m),
|
||||
)
|
||||
> cse.BOUNDED_DELTA_TRIGGER_M
|
||||
), (
|
||||
"Test fixture invariant broken: cold-boot snapshot and operator origin "
|
||||
"must be > 200 m apart for bounded_delta_conflict variant."
|
||||
)
|
||||
sitl_observer.prepare_sitl_cold_boot(host=f"{fc_adapter}-sitl", fixture_path=COLD_BOOT_FIXTURE)
|
||||
expected_origin = OPERATOR_ORIGIN
|
||||
else:
|
||||
pytest.fail(f"Unknown origin_source {origin_source!r}")
|
||||
|
||||
# 2. Cold-start SUT + push the first frame.
|
||||
FrameSourceReplayer(_resolve_frame_sink()).replay_video(
|
||||
DERKACHI_MP4, manifest_path=manifest_path, frame_limit=1,
|
||||
)
|
||||
|
||||
# 3. Collect first outbound estimate + FDR audit records.
|
||||
fdr_root = Path(evidence_dir).parent / f"run-{_cold_run_id}" / "fdr"
|
||||
first_estimate: cse.OutboundEstimate | None = None
|
||||
fdr_records: list[cse.FdrAuditRecord] = []
|
||||
for rec in fdr_reader.iter_records(fdr_root):
|
||||
if (
|
||||
first_estimate is None
|
||||
and rec.record_type == "outbound_estimate"
|
||||
):
|
||||
payload = rec.payload
|
||||
first_estimate = cse.OutboundEstimate(
|
||||
monotonic_ms=int(rec.monotonic_ms),
|
||||
lat_deg=float(payload["lat_deg"]), # type: ignore[arg-type]
|
||||
lon_deg=float(payload["lon_deg"]), # type: ignore[arg-type]
|
||||
source_label=str(payload["source_label"]), # type: ignore[arg-type]
|
||||
)
|
||||
if rec.record_type in {
|
||||
cse.FDR_RECORD_ORIGIN_SET,
|
||||
cse.FDR_RECORD_ORIGIN_UNAVAILABLE,
|
||||
cse.FDR_RECORD_BOUNDED_DELTA_REJECT,
|
||||
}:
|
||||
fdr_records.append(
|
||||
cse.FdrAuditRecord(
|
||||
monotonic_ms=int(rec.monotonic_ms),
|
||||
record_type=rec.record_type,
|
||||
payload=rec.payload,
|
||||
)
|
||||
)
|
||||
|
||||
# 4. Evaluate + assert per variant.
|
||||
report = cse.evaluate_first_estimate(
|
||||
origin_source=origin_source,
|
||||
expected_origin=expected_origin,
|
||||
first_estimate=first_estimate,
|
||||
fdr_records=fdr_records,
|
||||
)
|
||||
|
||||
if report.distance_m is not None:
|
||||
nfr_recorder.record_metric(
|
||||
f"ft_p_11.{origin_source}.distance_m", report.distance_m, ac_id="AC-1"
|
||||
)
|
||||
|
||||
assert report.passes_distance, (
|
||||
f"FT-P-11 {origin_source}: distance check failed "
|
||||
f"(budget {cse.ACCURACY_BUDGET_M} m): got distance_m={report.distance_m}"
|
||||
)
|
||||
|
||||
if origin_source == "operator_manifest":
|
||||
assert report.fdr_origin_set_source == "manifest", (
|
||||
f"AC-1: FDR must record c5.cold_start_origin.set(source='manifest'); "
|
||||
f"got source={report.fdr_origin_set_source!r}"
|
||||
)
|
||||
elif origin_source == "fc_ekf":
|
||||
assert report.fdr_origin_set_source == "fc_ekf", (
|
||||
f"AC-2: FDR must record c5.cold_start_origin.set(source='fc_ekf'); "
|
||||
f"got source={report.fdr_origin_set_source!r}"
|
||||
)
|
||||
elif origin_source == "bounded_delta_conflict":
|
||||
assert report.source_label_ok, (
|
||||
f"AC-4: source_label MUST NOT be "
|
||||
f"{cse.FORBIDDEN_FIRST_LABEL_BOUNDED_DELTA!r}; got "
|
||||
f"{report.actual_estimate.source_label if report.actual_estimate else None!r}"
|
||||
)
|
||||
assert report.fdr_bounded_delta_seen, (
|
||||
"AC-4: FDR must record c5.gps_bounded_delta.reject naming A and B"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.traces_to("AC-3,AC-NEW-1")
|
||||
def test_ft_p_11_cold_start_no_origin_aborts(
|
||||
fc_adapter: str,
|
||||
vio_strategy: str,
|
||||
evidence_dir, # type: ignore[no-untyped-def]
|
||||
_cold_run_id: str,
|
||||
nfr_recorder, # type: ignore[no-untyped-def]
|
||||
tmp_path: Path,
|
||||
_cold_start_harness_implemented: bool,
|
||||
) -> None:
|
||||
"""AC-3: Manifest empty + SITL no GPS → SUT MUST refuse takeoff."""
|
||||
if not _cold_start_harness_implemented:
|
||||
pytest.skip(
|
||||
"FT-P-11 AC-3 full scenario requires runner.helpers.{frame_source_replay,"
|
||||
"fdr_reader,sitl_observer.prepare_sitl_no_gps} — currently AZ-441 / "
|
||||
"AZ-407 leftovers. Pure-logic AC-3 covered by "
|
||||
"e2e/_unit_tests/helpers/test_cold_start_evaluator.py."
|
||||
)
|
||||
|
||||
from runner.helpers import fdr_reader, sitl_observer
|
||||
from runner.helpers.frame_source_replay import FrameSourceReplayer
|
||||
|
||||
manifest_path = tmp_path / "ft-p-11-no-origin-manifest.json"
|
||||
cse.write_manifest(manifest_path, None)
|
||||
sitl_observer.prepare_sitl_no_gps(host=f"{fc_adapter}-sitl")
|
||||
|
||||
FrameSourceReplayer(_resolve_frame_sink()).replay_video(
|
||||
DERKACHI_MP4, manifest_path=manifest_path, frame_limit=1,
|
||||
)
|
||||
|
||||
fdr_root = Path(evidence_dir).parent / f"run-{_cold_run_id}" / "fdr"
|
||||
first_estimate: cse.OutboundEstimate | None = None
|
||||
fdr_records: list[cse.FdrAuditRecord] = []
|
||||
for rec in fdr_reader.iter_records(fdr_root):
|
||||
if first_estimate is None and rec.record_type == "outbound_estimate":
|
||||
payload = rec.payload
|
||||
first_estimate = cse.OutboundEstimate(
|
||||
monotonic_ms=int(rec.monotonic_ms),
|
||||
lat_deg=float(payload["lat_deg"]), # type: ignore[arg-type]
|
||||
lon_deg=float(payload["lon_deg"]), # type: ignore[arg-type]
|
||||
source_label=str(payload["source_label"]), # type: ignore[arg-type]
|
||||
)
|
||||
if rec.record_type == cse.FDR_RECORD_ORIGIN_UNAVAILABLE:
|
||||
fdr_records.append(
|
||||
cse.FdrAuditRecord(
|
||||
monotonic_ms=int(rec.monotonic_ms),
|
||||
record_type=rec.record_type,
|
||||
payload=rec.payload,
|
||||
)
|
||||
)
|
||||
|
||||
report = cse.evaluate_no_origin_path(
|
||||
first_estimate=first_estimate, fdr_records=fdr_records,
|
||||
)
|
||||
|
||||
nfr_recorder.record_metric(
|
||||
"ft_p_11.no_origin.estimate_emitted",
|
||||
1.0 if report.estimate_within_budget else 0.0,
|
||||
ac_id="AC-3",
|
||||
)
|
||||
|
||||
assert report.passes, (
|
||||
f"AC-3: SUT must NOT emit any estimate AND FDR must record "
|
||||
f"{cse.FDR_RECORD_ORIGIN_UNAVAILABLE} within "
|
||||
f"{cse.FIRST_EMISSION_BUDGET_S} s. "
|
||||
f"estimate_emitted={report.estimate_within_budget}, "
|
||||
f"fdr_unavailable_seen={report.fdr_origin_unavailable_seen}"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_frame_sink(): # type: ignore[no-untyped-def]
|
||||
raise NotImplementedError(
|
||||
"frame sink resolution is owned by AZ-441 / runner.helpers.frame_source_replay"
|
||||
)
|
||||
Reference in New Issue
Block a user