[AZ-329] [AZ-330] [AZ-523] [AZ-524] Batch 44 atomic refactor

Implements two new C12 services and rebalances the C11/C12 boundary
in one atomic commit:

* AZ-329 PostLandingUploadOrchestrator — gates C11 upload on the
  `flight_footer` FDR record's `clean_shutdown` field; 4 refusal
  modes; new FdrFooterReader Protocol + LocalFdrFooterReader.
* AZ-330 OperatorReLocService — AC-3.4 visual-loss re-localization
  hint; reuses shared LatLonAlt; OperatorCommandTransport Protocol
  cut (E-C8 owns the future pymavlink concrete); new FDR record
  kind `c12.reloc.requested`; log redaction (lat/lon 5 decimals,
  reason 200 chars).
* AZ-523 C11 internal flight-state gate removed (SRP refactor):
  `confirm_flight_state` / `FlightStateSignal` use /
  `FlightStateNotOnGroundError` deleted from C11; TileUploader
  contract bumped to v2.0.0 (frozen) with migration note; AZ-317
  superseded.
* AZ-524 Package rename `c12_operator_tooling` →
  `c12_operator_orchestrator` across source, tests, pyproject,
  CMake, Dockerfile, compose, CI, runtime-root services class
  (`OperatorOrchestratorServices`) + factory function
  (`build_operator_orchestrator`), logger namespaces, config slug,
  docs, and the E-C12 epic title.

Tests: 1543 passed, 80 skipped (all environment gates). Targeted
AC suite (AZ-329 + AZ-330 + FdrFooterReader): 37 passed. Cold-start
NFR-perf still ≤ 500 ms p99.

Tracker: AZ-317 → Done (superseded); AZ-319 v2.0.0 contract bump
comment; AZ-329/AZ-330 → In Testing; AZ-253 epic renamed; AZ-523
+ AZ-524 created and closed as audit-trail tickets.

See `_docs/03_implementation/batch_44_cycle1_report.md`.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Oleksandr Bezdieniezhnykh
2026-05-13 19:42:46 +03:00
parent 2d88d3d674
commit 5fe67023b2
112 changed files with 3409 additions and 1311 deletions
@@ -1,297 +0,0 @@
"""AZ-317 ``FlightStateGate`` unit tests.
Covers all eight acceptance criteria + NFRs from
``_docs/02_tasks/done/AZ-317_c11_flight_state_gate.md`` (after the
batch-38 archive). Uses a hand-rolled fake :class:`FlightStateSource`
and a list-backed log handler so assertions stay close to the
captured records.
"""
from __future__ import annotations
import logging
import time
from datetime import datetime, timezone
import pytest
from gps_denied_onboard.components.c11_tile_manager import (
FlightStateGate,
FlightStateNotOnGroundError,
FlightStateSignal,
FlightStateSource,
)
# ----------------------------------------------------------------------
# Helpers
# ----------------------------------------------------------------------
class _FakeSource:
"""Hand-rolled :class:`FlightStateSource` returning a fixed signal.
Spies on every ``current_flight_state`` call so AC-8 can assert
the gate calls the source exactly once per ``confirm_on_ground``.
"""
def __init__(self, signal: FlightStateSignal) -> None:
self._signal = signal
self.call_count = 0
def current_flight_state(self) -> FlightStateSignal:
self.call_count += 1
return self._signal
class _RaisingSource:
""":class:`FlightStateSource` whose ``current_flight_state`` raises."""
def __init__(self, exc: Exception) -> None:
self._exc = exc
self.call_count = 0
def current_flight_state(self) -> FlightStateSignal:
self.call_count += 1
raise self._exc
class _PartialFake:
"""Type stub WITHOUT ``current_flight_state`` for AC-6 negative case."""
def something_else(self) -> str:
return "noop"
def _build_gate(
*,
source: FlightStateSource,
) -> tuple[FlightStateGate, list[logging.LogRecord]]:
records: list[logging.LogRecord] = []
class _ListHandler(logging.Handler):
def emit(self, record: logging.LogRecord) -> None:
records.append(record)
logger = logging.getLogger(f"test_az317_{id(records)}")
logger.handlers.clear()
logger.addHandler(_ListHandler())
logger.setLevel(logging.DEBUG)
logger.propagate = False
return FlightStateGate(source=source, logger=logger), records
def _kinds(records: list[logging.LogRecord]) -> list[str]:
return [getattr(r, "kind", None) for r in records]
# ----------------------------------------------------------------------
# AC-1: ON_GROUND passes
# ----------------------------------------------------------------------
def test_ac1_on_ground_returns_signal_and_emits_info_log() -> None:
# Arrange
source = _FakeSource(FlightStateSignal.ON_GROUND)
gate, records = _build_gate(source=source)
# Act
result = gate.confirm_on_ground()
# Assert
assert result is FlightStateSignal.ON_GROUND
assert _kinds(records) == ["c11.upload.flight_state_confirmed"]
assert records[0].levelname == "INFO"
assert source.call_count == 1
# ----------------------------------------------------------------------
# AC-2: IN_FLIGHT raises
# ----------------------------------------------------------------------
def test_ac2_in_flight_raises_with_observed_and_error_log() -> None:
# Arrange
source = _FakeSource(FlightStateSignal.IN_FLIGHT)
gate, records = _build_gate(source=source)
# Act + Assert
with pytest.raises(FlightStateNotOnGroundError) as excinfo:
gate.confirm_on_ground()
assert excinfo.value.observed is FlightStateSignal.IN_FLIGHT
assert "IN_FLIGHT" in str(excinfo.value)
assert _kinds(records) == ["c11.upload.refused.flight_state"]
assert records[0].levelname == "ERROR"
# ----------------------------------------------------------------------
# AC-3: UNKNOWN raises (fail-closed)
# ----------------------------------------------------------------------
def test_ac3_unknown_raises_fail_closed() -> None:
# Arrange
source = _FakeSource(FlightStateSignal.UNKNOWN)
gate, records = _build_gate(source=source)
# Act + Assert
with pytest.raises(FlightStateNotOnGroundError) as excinfo:
gate.confirm_on_ground()
assert excinfo.value.observed is FlightStateSignal.UNKNOWN
assert _kinds(records) == ["c11.upload.refused.flight_state"]
# ----------------------------------------------------------------------
# AC-4: TAKING_OFF and LANDING raise
# ----------------------------------------------------------------------
@pytest.mark.parametrize(
"transition_signal",
[FlightStateSignal.TAKING_OFF, FlightStateSignal.LANDING],
)
def test_ac4_transition_states_raise(
transition_signal: FlightStateSignal,
) -> None:
# Arrange
source = _FakeSource(transition_signal)
gate, records = _build_gate(source=source)
# Act + Assert
with pytest.raises(FlightStateNotOnGroundError) as excinfo:
gate.confirm_on_ground()
assert excinfo.value.observed is transition_signal
assert _kinds(records) == ["c11.upload.refused.flight_state"]
# ----------------------------------------------------------------------
# AC-5: source exception → UNKNOWN with __cause__ chained
# ----------------------------------------------------------------------
def test_ac5_source_exception_maps_to_unknown_and_preserves_cause() -> None:
# Arrange
original = RuntimeError("FC disconnected")
source = _RaisingSource(original)
gate, records = _build_gate(source=source)
# Act + Assert
with pytest.raises(FlightStateNotOnGroundError) as excinfo:
gate.confirm_on_ground()
assert excinfo.value.observed is FlightStateSignal.UNKNOWN
assert excinfo.value.__cause__ is original
assert _kinds(records) == ["c11.upload.refused.flight_state"]
assert records[0].levelname == "ERROR"
assert "FC disconnected" in records[0].kv["source_error"]
# ----------------------------------------------------------------------
# AC-6: FlightStateSource Protocol is conformance-checkable
# ----------------------------------------------------------------------
def test_ac6_protocol_isinstance_check_distinguishes_conforming_from_partial() -> None:
# Arrange
conforming = _FakeSource(FlightStateSignal.ON_GROUND)
non_conforming = _PartialFake()
# Assert
assert isinstance(conforming, FlightStateSource)
assert not isinstance(non_conforming, FlightStateSource)
# ----------------------------------------------------------------------
# AC-7: Error carries diagnostic fields
# ----------------------------------------------------------------------
def test_ac7_error_carries_observed_and_observed_at_with_message_format() -> None:
# Arrange
source = _FakeSource(FlightStateSignal.IN_FLIGHT)
gate, _ = _build_gate(source=source)
# Act
with pytest.raises(FlightStateNotOnGroundError) as excinfo:
gate.confirm_on_ground()
# Assert
assert excinfo.value.observed is FlightStateSignal.IN_FLIGHT
assert isinstance(excinfo.value.observed_at, datetime)
assert excinfo.value.observed_at.tzinfo == timezone.utc
assert excinfo.value.observed_at.microsecond == 0
assert str(excinfo.value).startswith("Upload refused: flight state is ")
# ----------------------------------------------------------------------
# AC-8: Gate calls source exactly once
# ----------------------------------------------------------------------
def test_ac8_gate_calls_source_exactly_once_no_retry() -> None:
# Arrange
source = _FakeSource(FlightStateSignal.IN_FLIGHT)
gate, _ = _build_gate(source=source)
# Act
with pytest.raises(FlightStateNotOnGroundError):
gate.confirm_on_ground()
# Assert
assert source.call_count == 1
# ----------------------------------------------------------------------
# NFR-perf: confirm_on_ground microbench p99 ≤ 1 ms
# ----------------------------------------------------------------------
def test_nfr_perf_microbench_under_one_ms_p99() -> None:
# Arrange
source = _FakeSource(FlightStateSignal.ON_GROUND)
gate, _ = _build_gate(source=source)
iterations = 5_000
# Act
samples_ns: list[int] = []
for _ in range(iterations):
start = time.perf_counter_ns()
gate.confirm_on_ground()
samples_ns.append(time.perf_counter_ns() - start)
# Assert
samples_ns.sort()
p99_ns = samples_ns[int(iterations * 0.99) - 1]
assert p99_ns < 1_000_000, (
f"p99 latency {p99_ns} ns exceeds 1 ms (1_000_000 ns) NFR budget"
)
# ----------------------------------------------------------------------
# NFR-reliability-fail-closed: every non-ON_GROUND state raises
# ----------------------------------------------------------------------
@pytest.mark.parametrize(
"non_on_ground_signal",
[
FlightStateSignal.IN_FLIGHT,
FlightStateSignal.TAKING_OFF,
FlightStateSignal.LANDING,
FlightStateSignal.UNKNOWN,
],
)
def test_nfr_reliability_fail_closed_matrix_complete(
non_on_ground_signal: FlightStateSignal,
) -> None:
# Arrange
source = _FakeSource(non_on_ground_signal)
gate, _ = _build_gate(source=source)
# Act + Assert
with pytest.raises(FlightStateNotOnGroundError):
gate.confirm_on_ground()
@@ -26,8 +26,6 @@ import pytest
from gps_denied_onboard.components.c11_tile_manager import (
C11RetryConfig,
FlightStateNotOnGroundError,
FlightStateSignal,
IdempotentRetryTileUploader,
IngestStatus,
PerTileStatus,
@@ -76,7 +74,6 @@ class _ScriptedInner:
self.raises = list(raise_on_call or [])
self.calls: list[UploadRequest] = []
self.enumerate_calls: list[Any] = []
self.confirm_calls: int = 0
def upload_pending_tiles(self, request: UploadRequest) -> UploadBatchReport:
self.calls.append(request)
@@ -94,10 +91,6 @@ class _ScriptedInner:
self.enumerate_calls.append(flight_id)
return [{"sentinel": True, "flight_id": flight_id}]
def confirm_flight_state(self) -> FlightStateSignal:
self.confirm_calls += 1
return FlightStateSignal.ON_GROUND
@dataclass
class _FakeMetadataStore:
@@ -388,39 +381,11 @@ def test_ac11_enumerate_pending_passes_through() -> None:
assert out == [{"sentinel": True, "flight_id": fid}]
def test_ac11_confirm_flight_state_passes_through() -> None:
# Arrange
inner = _ScriptedInner(reports=[_success(0)])
(decorator, _logs, _store, _clk, _fdr) = _build_decorator(inner=inner)
# Act
state = decorator.confirm_flight_state()
# Assert
assert state == FlightStateSignal.ON_GROUND
assert inner.confirm_calls == 1
# ----------------------------------------------------------------------
# AC-12 — inner exception propagates without retry
# ----------------------------------------------------------------------
def test_ac12_flight_state_not_on_ground_propagates_without_retry() -> None:
# Arrange
from datetime import datetime, timezone
err = FlightStateNotOnGroundError(FlightStateSignal.IN_FLIGHT, datetime.now(timezone.utc))
inner = _ScriptedInner(raise_on_call=[err])
(decorator, _logs, _store, clk, _fdr) = _build_decorator(inner=inner)
# Act / Assert
with pytest.raises(FlightStateNotOnGroundError):
decorator.upload_pending_tiles(_request())
assert clk.sleep_calls == []
assert len(inner.calls) == 1
def test_ac12_satellite_provider_error_propagates_without_retry() -> None:
# Arrange
inner = _ScriptedInner(raise_on_call=[SatelliteProviderError("boom")])
@@ -493,7 +458,6 @@ def test_ac10_factory_returns_decorated_uploader_by_default() -> None:
http_client=_httpx.Client(transport=transport),
tile_store=object(),
tile_metadata_store=object(),
flight_state_gate=object(), # type: ignore[arg-type]
key_manager=object(), # type: ignore[arg-type]
)
@@ -516,7 +480,6 @@ def test_ac10_factory_bypasses_decorator_when_flag_set() -> None:
http_client=_httpx.Client(transport=transport),
tile_store=object(),
tile_metadata_store=object(),
flight_state_gate=object(), # type: ignore[arg-type]
key_manager=object(), # type: ignore[arg-type]
)
@@ -33,17 +33,12 @@ class _NullSleep:
return None
class _PartialFakeMissingConfirm:
"""Conformance counterexample: missing ``confirm_flight_state``."""
class _PartialFakeMissingEnumerate:
"""Conformance counterexample: missing ``enumerate_pending_tiles``."""
def upload_pending_tiles(self, request: object) -> object: # noqa: ARG002
return None
def enumerate_pending_tiles(
self, flight_id: object | None = None
) -> list[object]: # noqa: ARG002
return []
class _PartialDownloaderMissingEnumerate:
"""Conformance counterexample: missing ``enumerate_remote_coverage``."""
@@ -67,7 +62,6 @@ def test_ac12_concrete_uploader_satisfies_protocol() -> None:
http_client=httpx.Client(transport=transport),
tile_store=object(), # type: ignore[arg-type]
tile_metadata_store=object(), # type: ignore[arg-type]
flight_state_gate=object(), # type: ignore[arg-type]
key_manager=object(), # type: ignore[arg-type]
fdr_client=FakeFdrSink(_PRODUCER_ID), # type: ignore[arg-type]
logger=logging.getLogger("test_az319_conformance"),
@@ -81,7 +75,7 @@ def test_ac12_concrete_uploader_satisfies_protocol() -> None:
def test_ac12_partial_fake_is_not_protocol_conformant() -> None:
# Assert
assert not isinstance(_PartialFakeMissingConfirm(), TileUploader)
assert not isinstance(_PartialFakeMissingEnumerate(), TileUploader)
def test_ac10_concrete_downloader_satisfies_protocol() -> None:
@@ -129,7 +123,6 @@ def test_ac9_idempotent_retry_decorator_satisfies_uploader_protocol() -> None:
http_client=httpx.Client(transport=transport),
tile_store=object(), # type: ignore[arg-type]
tile_metadata_store=object(), # type: ignore[arg-type]
flight_state_gate=object(), # type: ignore[arg-type]
key_manager=object(), # type: ignore[arg-type]
fdr_client=FakeFdrSink(_PRODUCER_ID), # type: ignore[arg-type]
logger=logging.getLogger("test_az320_inner"),
@@ -1,12 +1,14 @@
"""AZ-319 ``HttpTileUploader`` unit tests.
Covers AC-1 .. AC-14 and the upload-throughput NFR from
``_docs/02_tasks/todo/AZ-319_c11_tile_uploader.md``.
Covers AC-1, AC-3 .. AC-14 and the upload-throughput NFR from
``_docs/02_tasks/done/AZ-319_c11_tile_uploader.md``. AC-2 (the legacy
ON_GROUND gate) was removed in batch 44 — gating is now C12's
``PostLandingUploadOrchestrator`` responsibility.
Uses :class:`httpx.MockTransport` for deterministic HTTP responses,
:class:`FakeFdrSink` for FDR capture, a list-backed ``logging.Handler``
for log capture, and stub C6 stores / gate / key manager so this
suite never drags in AZ-303 / AZ-305 / AZ-317 / AZ-318 internals.
for log capture, and stub C6 stores / key manager so this suite never
drags in AZ-303 / AZ-305 / AZ-318 internals.
"""
from __future__ import annotations
@@ -25,8 +27,6 @@ import pytest
from gps_denied_onboard.components.c11_tile_manager import (
C11Config,
FlightStateNotOnGroundError,
FlightStateSignal,
HttpTileUploader,
IngestStatus,
PerFlightKeyManager,
@@ -37,9 +37,6 @@ from gps_denied_onboard.components.c11_tile_manager import (
UploadRequest,
canonical_payload_bytes,
)
from gps_denied_onboard.components.c11_tile_manager.flight_state_gate import (
FlightStateGate,
)
from gps_denied_onboard.fdr_client import FdrRecord
from gps_denied_onboard.fdr_client.fakes import FakeFdrSink
@@ -125,25 +122,6 @@ class _FakeMetadataStore:
self.mark_calls.append((tile_id, uploaded_at))
class _StubGate:
"""Stand-in for AZ-317 ``FlightStateGate``."""
def __init__(
self, signal: FlightStateSignal = FlightStateSignal.ON_GROUND
) -> None:
self._signal = signal
self.confirm_calls: int = 0
def confirm_on_ground(self) -> FlightStateSignal:
self.confirm_calls += 1
if self._signal != FlightStateSignal.ON_GROUND:
raise FlightStateNotOnGroundError(
self._signal,
datetime.now(timezone.utc),
)
return self._signal
class _StubKeyManager:
"""Stand-in for AZ-318 ``PerFlightKeyManager``.
@@ -222,7 +200,6 @@ def _build_uploader(
transport: httpx.MockTransport,
pending: list[_FakeTile] | None = None,
blobs: dict[str, bytes] | None = None,
gate_signal: FlightStateSignal = FlightStateSignal.ON_GROUND,
fingerprint_hex: str = "0123456789abcdef",
config: C11Config | None = None,
sleep_recorder: list[float] | None = None,
@@ -230,7 +207,6 @@ def _build_uploader(
HttpTileUploader,
FakeFdrSink,
list[logging.LogRecord],
_StubGate,
_StubKeyManager,
_FakeTileStore,
_FakeMetadataStore,
@@ -249,7 +225,6 @@ def _build_uploader(
logger.setLevel(logging.DEBUG)
logger.propagate = False
gate = _StubGate(signal=gate_signal)
key_manager = _StubKeyManager(fingerprint_hex=fingerprint_hex)
tile_store = _FakeTileStore(blobs=blobs)
metadata_store = _FakeMetadataStore(pending=pending)
@@ -272,14 +247,13 @@ def _build_uploader(
http_client=client,
tile_store=tile_store,
tile_metadata_store=metadata_store,
flight_state_gate=gate, # type: ignore[arg-type]
key_manager=key_manager, # type: ignore[arg-type]
fdr_client=fdr, # type: ignore[arg-type]
logger=logger,
config=cfg,
sleep=_sleep,
)
return uploader, fdr, log_records, gate, key_manager, tile_store, metadata_store, sleeps
return uploader, fdr, log_records, key_manager, tile_store, metadata_store, sleeps
def _make_request(*, batch_size: int = 10, flight_id: UUID | None = None) -> UploadRequest:
@@ -361,7 +335,6 @@ def test_ac1_50_tile_happy_path_marks_all_uploaded() -> None:
uploader,
fdr,
_logs,
_gate,
key_manager,
_tile_store,
metadata_store,
@@ -385,48 +358,6 @@ def test_ac1_50_tile_happy_path_marks_all_uploaded() -> None:
assert key_manager.end_calls == 1
# ----------------------------------------------------------------------
# AC-2: gate blocks before any read or POST
# ----------------------------------------------------------------------
def test_ac2_gate_blocks_before_any_read_or_post() -> None:
# Arrange
pending = [_make_tile()]
posted: list[httpx.Request] = []
def _handler(request: httpx.Request) -> httpx.Response:
posted.append(request)
return httpx.Response(202, json={"batch_uuid": str(uuid4()), "per_tile_status": []})
transport = httpx.MockTransport(_handler)
(
uploader,
_fdr,
_logs,
gate,
key_manager,
tile_store,
metadata_store,
_sleeps,
) = _build_uploader(
transport=transport,
pending=pending,
gate_signal=FlightStateSignal.IN_FLIGHT,
)
# Act / Assert
with pytest.raises(FlightStateNotOnGroundError):
uploader.upload_pending_tiles(_make_request())
assert gate.confirm_calls == 1
assert metadata_store.pending_calls == 0
assert tile_store.read_calls == []
assert key_manager.start_calls == []
assert key_manager.end_calls == 0
assert posted == []
# ----------------------------------------------------------------------
# AC-3: signature rejection — record + skip mark_uploaded; outcome=partial
# ----------------------------------------------------------------------
@@ -457,7 +388,6 @@ def test_ac3_signature_rejection_records_and_keeps_pending() -> None:
uploader,
fdr,
_logs,
_gate,
key_manager,
_tile_store,
metadata_store,
@@ -504,7 +434,6 @@ def test_ac4_duplicate_and_superseded_are_success() -> None:
uploader,
_fdr,
_logs,
_gate,
_key_manager,
_tile_store,
metadata_store,
@@ -536,7 +465,6 @@ def test_ac5_signing_key_zeroised_on_success() -> None:
uploader,
_fdr,
_logs,
_gate,
key_manager,
_tile_store,
_metadata_store,
@@ -570,7 +498,6 @@ def test_ac6_signing_key_zeroised_on_failure() -> None:
uploader,
_fdr,
_logs,
_gate,
key_manager,
_tile_store,
metadata_store,
@@ -605,7 +532,6 @@ def test_ac7_public_key_fdr_precedes_tile_fdr() -> None:
uploader,
fdr,
_logs,
_gate,
_key_manager,
_tile_store,
_metadata_store,
@@ -661,7 +587,6 @@ def test_ac8_429_honours_retry_after_seconds() -> None:
uploader,
_fdr,
_logs,
_gate,
_key_manager,
_tile_store,
_metadata_store,
@@ -697,7 +622,6 @@ def test_ac9_persistent_5xx_raises_satellite_provider_error() -> None:
uploader,
_fdr,
_logs,
_gate,
key_manager,
_tile_store,
_metadata_store,
@@ -731,7 +655,6 @@ def test_ac10_401_fails_fast_no_retry() -> None:
uploader,
_fdr,
log_records,
_gate,
_key_manager,
_tile_store,
_metadata_store,
@@ -766,7 +689,6 @@ def test_ac11_empty_pending_set_is_success_no_posts() -> None:
uploader,
fdr,
_logs,
_gate,
key_manager,
_tile_store,
_metadata_store,
@@ -868,7 +790,6 @@ def test_ac14_partial_success_batch_does_not_raise() -> None:
uploader,
_fdr,
_logs,
_gate,
_key_manager,
_tile_store,
metadata_store,
@@ -910,7 +831,6 @@ def test_429_budget_exhaustion_raises_rate_limited_error() -> None:
uploader,
_fdr,
_logs,
_gate,
key_manager,
_tile_store,
_metadata_store,
@@ -951,7 +871,7 @@ def test_nfr_throughput_1000_tiles_under_budget() -> None:
)
transport = httpx.MockTransport(_handler)
(uploader, _fdr, _logs, _gate, _km, _ts, _ms, _sleeps) = _build_uploader(
(uploader, _fdr, _logs, _km, _ts, _ms, _sleeps) = _build_uploader(
transport=transport, pending=pending
)