[AZ-234] [AZ-235] [AZ-236] [AZ-237] Add replay tests

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-06-22 13:21:13 +00:00 · 2026-05-05 06:24:10 +03:00
parent c30fd4f67d
commit 5acd14b792
12 changed files with 616 additions and 3 deletions
@@ -0,0 +1,117 @@
+from e2e.replay.harness import mavlink_source_is_authorized
+from mavlink_gcs_integration import InMemoryMavlinkGateway, OperatorStatusMessage
+from safety_anchor_wrapper import SafetyAnchorStateMachine, SafetyStateConfig, TelemetryContext
+from shared.contracts import VioStatePacket
+
+
+def test_blackout_trace_transitions_to_dead_reckoned_then_no_fix() -> None:
+    # Arrange
+    state_machine = SafetyAnchorStateMachine(
+        SafetyStateConfig(
+            initial_covariance_m=2.0,
+            dead_reckoning_growth_m=125.0,
+            no_fix_covariance_threshold_m=500.0,
+        )
+    )
+    state_machine.update_vio(
+        VioStatePacket(
+            timestamp_ns=1_000_000_000,
+            relative_pose={"x_m": 0.0},
+            velocity_mps=(0.0, 0.0, 0.0),
+            tracking_quality=0.9,
+            covariance_hint=[[2.0, 0.0], [0.0, 2.0]],
+        ),
+        TelemetryContext(
+            timestamp_ns=1_000_000_000,
+            latitude_hint_deg=48.0,
+            longitude_hint_deg=37.0,
+            altitude_m=400.0,
+        ),
+    )
+
+    # Act
+    snapshots = tuple(
+        state_machine.propagate_blackout(1_000_000_000 + index * 1_000_000_000)
+        for index in range(1, 6)
+    )
+
+    # Assert
+    assert snapshots[0].mode == "dead_reckoned"
+    assert snapshots[-1].mode == "no_fix"
+    covariances = tuple(snapshot.estimate.covariance_semimajor_m for snapshot in snapshots)
+    assert covariances == tuple(sorted(covariances))
+    assert snapshots[-1].estimate.fix_type == 0
+    assert snapshots[-1].estimate.horizontal_accuracy_m >= 999.0
+
+
+def test_no_fix_estimate_is_not_emitted_as_confident_gps_input() -> None:
+    # Arrange
+    state_machine = SafetyAnchorStateMachine(
+        SafetyStateConfig(dead_reckoning_growth_m=600.0, no_fix_covariance_threshold_m=500.0)
+    )
+    gateway = InMemoryMavlinkGateway(status_rate_limit_ns=1_000_000_000)
+    state_machine.update_vio(
+        VioStatePacket(
+            timestamp_ns=1,
+            relative_pose={"x_m": 0.0},
+            velocity_mps=(0.0, 0.0, 0.0),
+            tracking_quality=0.5,
+        ),
+        TelemetryContext(
+            timestamp_ns=1,
+            latitude_hint_deg=48.0,
+            longitude_hint_deg=37.0,
+            altitude_m=400.0,
+        ),
+    )
+    no_fix_snapshot = state_machine.propagate_blackout(2)
+
+    # Act
+    emission = gateway.emit_gps_input(no_fix_snapshot.estimate)
+
+    # Assert
+    assert emission.emitted is False
+    assert emission.error is not None
+    assert "unsafe for GPS_INPUT" in emission.error.message
+
+
+def test_unauthorized_mavlink_sources_are_rejected_by_test_assertion() -> None:
+    # Arrange
+    allowed_source_system_ids = {1, 42}
+
+    # Act / Assert
+    assert mavlink_source_is_authorized(42, allowed_source_system_ids) is True
+    assert mavlink_source_is_authorized(99, allowed_source_system_ids) is False
+
+
+def test_qgc_status_and_fdr_evidence_are_visible_and_rate_limited() -> None:
+    # Arrange
+    gateway = InMemoryMavlinkGateway(status_rate_limit_ns=2_000_000_000)
+    messages = [
+        OperatorStatusMessage(
+            timestamp_ns=1_000_000_000,
+            severity="warning",
+            text="VISUAL_BLACKOUT_IMU_ONLY",
+        ),
+        OperatorStatusMessage(
+            timestamp_ns=2_000_000_000,
+            severity="warning",
+            text="VISUAL_BLACKOUT_IMU_ONLY",
+        ),
+        OperatorStatusMessage(
+            timestamp_ns=4_000_000_000,
+            severity="critical",
+            text="VISUAL_BLACKOUT_FAILSAFE",
+        ),
+    ]
+
+    # Act
+    result = gateway.emit_status(messages)
+
+    # Assert
+    assert [message.text for message in result.emitted] == [
+        "VISUAL_BLACKOUT_IMU_ONLY",
+        "VISUAL_BLACKOUT_FAILSAFE",
+    ]
+    assert len(result.suppressed) == 1
+    assert all(message.visible_to_qgc for message in result.emitted)
@@ -0,0 +1,123 @@
+from anchor_verification import AnchorFrame, CandidateTile, GeometryGatedAnchorVerifier
+from e2e.replay.harness import SatelliteCacheStub
+from satellite_service import (
+    LocalVprIndexPackage,
+    LocalVprRetriever,
+    RelocalizationRequest,
+    SatelliteSyncBoundary,
+    VprDescriptorRecord,
+)
+from shared.contracts import VprCandidate
+from tile_manager import GeneratedTileSyncPackage
+
+
+def test_verified_anchor_includes_retrieval_matching_and_provenance_evidence() -> None:
+    # Arrange
+    retriever = LocalVprRetriever()
+    retriever.load_index(
+        LocalVprIndexPackage(
+            package_id="fixture-index",
+            records=(
+                VprDescriptorRecord(
+                    chunk_id="chunk-001",
+                    tile_id="tile-001",
+                    descriptor=(1.0, 0.0, 0.0),
+                    footprint={"min_lat": 48.0, "max_lat": 48.1, "min_lon": 37.0, "max_lon": 37.1},
+                    freshness_status="fresh",
+                ),
+            ),
+        )
+    )
+    retrieval = retriever.retrieve(
+        RelocalizationRequest(
+            frame_id="frame-001",
+            image_ref="AD000001.jpg",
+            trigger_reason="cold_start",
+            top_k=1,
+            query_descriptor=(1.0, 0.0, 0.0),
+        )
+    )
+    keypoints = tuple((float(index), float(index % 5)) for index in range(24))
+    shifted_keypoints = tuple((x + 1.0, y + 1.0) for x, y in keypoints)
+    verifier = GeometryGatedAnchorVerifier()
+
+    # Act
+    verification = verifier.verify_candidate(
+        AnchorFrame(frame_id="frame-001", image_ref="AD000001.jpg", keypoints=keypoints),
+        CandidateTile(
+            candidate=retrieval.candidates[0],
+            image_ref="tile-001.cog",
+            keypoints=shifted_keypoints,
+            provenance_trusted=True,
+        ),
+    )
+
+    # Assert
+    assert retrieval.ready is True
+    assert retrieval.latency_ms is not None
+    assert verification.decision.accepted is True
+    assert verification.decision.candidate_id == "chunk-001"
+    assert verification.decision.inliers >= 20
+    assert verification.decision.mean_reprojection_error_px <= 3.0
+    assert verification.homography is not None
+    assert verification.freshness_status == "fresh"
+
+
+def test_unsafe_cache_or_low_texture_candidates_never_emit_trusted_anchor() -> None:
+    # Arrange
+    verifier = GeometryGatedAnchorVerifier()
+    frame = AnchorFrame(
+        frame_id="frame-low-texture",
+        image_ref="low-texture.jpg",
+        usable_for_anchor=False,
+        keypoints=((0.0, 0.0), (1.0, 1.0), (2.0, 2.0), (3.0, 3.0)),
+    )
+    candidate = VprCandidate(
+        chunk_id="chunk-stale",
+        tile_id="tile-stale",
+        score=0.9,
+        footprint={"min_lat": 48.0, "max_lat": 48.1, "min_lon": 37.0, "max_lon": 37.1},
+        freshness_status="stale",
+    )
+
+    # Act
+    verification = verifier.verify_candidate(
+        frame,
+        CandidateTile(
+            candidate=candidate,
+            image_ref="tile-stale.cog",
+            keypoints=((0.0, 0.0), (1.0, 1.0), (2.0, 2.0), (3.0, 3.0)),
+            provenance_trusted=False,
+        ),
+    )
+
+    # Assert
+    assert verification.decision.accepted is False
+    assert verification.decision.rejection_reason == "frame_not_usable"
+
+
+def test_flight_mode_missing_cache_does_not_attempt_external_access() -> None:
+    # Arrange
+    cache_stub = SatelliteCacheStub()
+    sync_boundary = SatelliteSyncBoundary()
+
+    # Act
+    cache_response = cache_stub.query_manifest("NFT-SEC-04", "missing")
+    sync_result = sync_boundary.upload_generated_tiles(
+        GeneratedTileSyncPackage(
+            package_ref="generated-empty",
+            mission_id="mission-001",
+            manifest_delta=(),
+            sidecars=(),
+        ),
+        phase="in_flight",
+    )
+
+    # Assert
+    assert cache_response["network_fetch_attempted"] is False
+    assert cache_response["trusted"] is False
+    assert int(str(cache_response["fixture_size_bytes"])) < int(
+        str(cache_response["storage_budget_bytes"])
+    )
+    assert sync_result.error is not None
+    assert sync_result.error.cause == "mid_flight_network_blocked"
@@ -0,0 +1,68 @@
+from pathlib import Path
+
+import pytest
+
+from e2e.replay.harness import (
+    ReplayEstimate,
+    evaluate_still_image_estimates,
+    load_expected_coordinates,
+)
+
+
+def test_expected_coordinate_loader_rejects_invalid_wgs84_rows(tmp_path: Path) -> None:
+    # Arrange
+    coordinates_path = tmp_path / "coordinates.csv"
+    coordinates_path.write_text("image, lat, lon\nAD000001.jpg, 120.0, 37.0\n", encoding="utf-8")
+
+    # Act / Assert
+    with pytest.raises(ValueError, match="outside WGS84 bounds"):
+        load_expected_coordinates(coordinates_path)
+
+
+def test_still_image_replay_reports_coordinate_thresholds_and_latency() -> None:
+    # Arrange
+    expected = load_expected_coordinates(Path("_docs/00_problem/input_data/coordinates.csv"))
+    estimates = tuple(
+        ReplayEstimate(
+            image_ref=coordinate.image_ref,
+            latitude_deg=coordinate.latitude_deg + 0.00001,
+            longitude_deg=coordinate.longitude_deg + 0.00001,
+            covariance_95_semi_major_m=8.0,
+            source_label="satellite_anchored",
+            anchor_age_ms=150,
+            capture_to_output_latency_ms=40.0 + index,
+        )
+        for index, coordinate in enumerate(expected)
+    )
+
+    # Act
+    metrics = evaluate_still_image_estimates(expected, estimates)
+
+    # Assert
+    assert metrics["threshold_passed"] is True
+    assert metrics["within_50_m_rate"] >= 0.80
+    assert metrics["within_20_m_rate"] >= 0.50
+    assert metrics["p50_latency_ms"] > 0.0
+    assert metrics["p95_latency_ms"] >= metrics["p50_latency_ms"]
+    assert metrics["p99_latency_ms"] >= metrics["p95_latency_ms"]
+    assert metrics["dropped_frame_rate"] == 0.0
+
+
+def test_confidence_contract_validation_fails_missing_source_label() -> None:
+    # Arrange
+    expected = load_expected_coordinates(Path("_docs/00_problem/input_data/coordinates.csv"))[:1]
+    estimates = (
+        ReplayEstimate(
+            image_ref=expected[0].image_ref,
+            latitude_deg=expected[0].latitude_deg,
+            longitude_deg=expected[0].longitude_deg,
+            covariance_95_semi_major_m=8.0,
+            source_label="",
+            anchor_age_ms=0,
+            capture_to_output_latency_ms=10.0,
+        ),
+    )
+
+    # Act / Assert
+    with pytest.raises(ValueError, match="source label is missing"):
+        evaluate_still_image_estimates(expected, estimates)
@@ -0,0 +1,88 @@
+from pathlib import Path
+
+import pytest
+
+from e2e.replay.harness import (
+    BlackboxReplayRunner,
+    ScenarioConfig,
+    ScenarioGroup,
+    ScenarioResult,
+    validate_derkachi_alignment,
+)
+from shared.contracts import FramePacket, TelemetrySample
+from vio_adapter import LocalVioAdapter, VioInputPacket
+
+
+def test_derkachi_alignment_validator_accepts_expected_fixture_shape() -> None:
+    # Act
+    metrics = validate_derkachi_alignment(
+        video_duration_s=490.07,
+        telemetry_duration_s=490.07,
+        telemetry_rows=4_900,
+    )
+
+    # Assert
+    assert metrics["alignment_valid"] is True
+    assert metrics["duration_delta_s"] == 0.0
+    assert metrics["frames_per_telemetry"] == pytest.approx(3.0, abs=0.05)
+
+
+def test_derkachi_alignment_validator_blocks_duration_drift() -> None:
+    # Act / Assert
+    with pytest.raises(ValueError, match="more than 250 ms"):
+        validate_derkachi_alignment(
+            video_duration_s=490.07,
+            telemetry_duration_s=489.50,
+            telemetry_rows=4_900,
+        )
+
+
+def test_public_vio_replay_boundary_emits_frame_by_frame_estimate() -> None:
+    # Arrange
+    adapter = LocalVioAdapter()
+    frame = FramePacket(
+        frame_id="derkachi-0001",
+        timestamp_ns=1_000_000_000,
+        image_ref="_docs/00_problem/input_data/flight_derkachi/flight_derkachi.mp4#0",
+        calibration_id="derkachi-calibration-gated",
+        occlusion="clear",
+        quality=0.9,
+    )
+    telemetry = (
+        TelemetrySample(
+            timestamp_ns=1_000_000_000,
+            imu={"accel_x": 0.0, "accel_y": 0.0, "accel_z": -9.8},
+            attitude={"roll": 0.0, "pitch": 0.0, "yaw": 1.0},
+            altitude_m=400.0,
+            airspeed_mps=22.0,
+            gps_health="healthy",
+        ),
+    )
+
+    # Act
+    result = adapter.process(VioInputPacket(frame=frame, telemetry_samples=telemetry))
+
+    # Assert
+    assert result.state_packet is not None
+    assert result.health.state == "ready"
+    assert result.state_packet.timestamp_ns == frame.timestamp_ns
+    assert result.state_packet.tracking_quality > 0.0
+
+
+def test_public_dataset_and_calibration_prerequisites_are_reported_blocked(tmp_path: Path) -> None:
+    # Arrange
+    scenario = ScenarioConfig(
+        scenario_id="FT-P-03-CALIBRATION",
+        name="Calibration-gated public VIO dataset",
+        group=ScenarioGroup.PERFORMANCE,
+        input_dataset="public_nadir_vio_candidates",
+        required_paths=(tmp_path / "camera_intrinsics.yaml",),
+    )
+
+    # Act
+    result = BlackboxReplayRunner(output_root=tmp_path, scenarios=(scenario,)).run()
+
+    # Assert
+    report = result.reports[0]
+    assert report.result == ScenarioResult.BLOCKED
+    assert "camera_intrinsics.yaml" in report.error_message