mirror of
https://github.com/azaion/gps-denied-onboard.git
synced 2026-06-21 21:41:12 +00:00
b5dd6031d2
AZ-291 — FileFdrWriter: single writer thread draining every registered FdrClient SPSC ring buffer to per-flight segment files; per-segment size rotation; cross-process fcntl.flock filelock on flight_root; ENOSPC degraded mode with rate-capped ERROR logs and one GCS alert. AZ-292 — FlightHeader/FlightFooter dataclasses + open_flight / close_flight lifecycle methods; four per-flight monotonic counters (records_written, records_dropped_overrun, bytes_written, rollover_count) reported by the footer; flight_id mismatch and close-without-open are typed errors. AZ-293 — CapacityCapPolicy (post-rotation hook): walks the flight directory, drops the oldest CLOSED segment when total > cap (default 64 GiB), emits a kind="segment_rollover" record per drop. Never drops the currently-open segment or segment 0 alone; cap_misconfigured path logs ERROR + GCS alert. No config flag disables emission (C13-ST-01). Schema: bumped fdr_record_schema flight_header / flight_footer payload key sets to match the AZ-292 task spec (effective 1.0.0 -> 1.1.0; no prior producer); KNOWN_PAYLOAD_KEYS updated. Added FdrWriterConfig nested in FdrConfig (segment_size_bytes, batch_size, flight_cap_bytes, debug_log_per_record). Tests: 29 new unit tests (8 AC + 1 invariant per task); full suite 323 passed, 2 pre-existing skips, 0 regressions. Co-authored-by: Cursor <cursoragent@cursor.com>
321 lines
9.7 KiB
Python
321 lines
9.7 KiB
Python
"""AZ-292 — FlightHeader / FlightFooter + per-flight counters.
|
|
|
|
Covers AC-1..AC-8.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import dataclasses
|
|
import struct
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from uuid import UUID, uuid4
|
|
|
|
import pytest
|
|
|
|
from gps_denied_onboard.components.c13_fdr import (
|
|
FdrCloseWithoutOpenError,
|
|
FdrOpenError,
|
|
FileFdrWriter,
|
|
FlightFooter,
|
|
FlightHeader,
|
|
)
|
|
from gps_denied_onboard.config import FdrWriterConfig
|
|
from gps_denied_onboard.fdr_client.client import FdrClient
|
|
from gps_denied_onboard.fdr_client.records import (
|
|
OVERRUN_KIND,
|
|
OVERRUN_PRODUCER_ID,
|
|
FdrRecord,
|
|
parse,
|
|
)
|
|
|
|
_LENGTH_PREFIX = struct.Struct("<I")
|
|
|
|
|
|
def _make_header(flight_id: UUID) -> FlightHeader:
|
|
return FlightHeader(
|
|
flight_id=flight_id,
|
|
flight_started_at_iso=datetime.now(tz=timezone.utc).isoformat(),
|
|
flight_started_at_monotonic_ns=time.monotonic_ns(),
|
|
config_snapshot={"tier": 2, "fdr": {"queue_size": 4096}},
|
|
signing_key_rotation_event={"current_key_id": "k1"},
|
|
manifest_content_hashes={"foo/bar.bin": "deadbeef" * 8},
|
|
build_info={"commit": "abc1234", "build_date": "2026-05-11"},
|
|
)
|
|
|
|
|
|
def _make_client(producer_id: str = "c1_vio") -> FdrClient:
|
|
return FdrClient(producer_id=producer_id, capacity=256, _emit_diag_log=False)
|
|
|
|
|
|
def _payload(i: int) -> FdrRecord:
|
|
return FdrRecord(
|
|
schema_version=1,
|
|
ts=datetime.now(tz=timezone.utc).isoformat(),
|
|
producer_id="c1_vio",
|
|
kind="vio.tick",
|
|
payload={
|
|
"frame_id": i,
|
|
"R": [[1, 0, 0], [0, 1, 0], [0, 0, 1]],
|
|
"t": [0, 0, 0],
|
|
"P": [],
|
|
"last_anchor_age_ms": 0,
|
|
},
|
|
)
|
|
|
|
|
|
def _read_records(flight_dir: Path) -> list[FdrRecord]:
|
|
records: list[FdrRecord] = []
|
|
for seg in sorted(flight_dir.glob("segment-*.fdr")):
|
|
data = seg.read_bytes()
|
|
offset = 0
|
|
while offset < len(data):
|
|
(length,) = _LENGTH_PREFIX.unpack_from(data, offset)
|
|
offset += _LENGTH_PREFIX.size
|
|
records.append(parse(data[offset : offset + length]))
|
|
offset += length
|
|
return records
|
|
|
|
|
|
def _build_writer(
|
|
tmp_path: Path, flight_id: UUID, config: FdrWriterConfig | None = None
|
|
) -> tuple[FileFdrWriter, FdrClient, list[str]]:
|
|
config = config or FdrWriterConfig()
|
|
alerts: list[str] = []
|
|
client = _make_client()
|
|
writer = FileFdrWriter(
|
|
flight_root=tmp_path / "fdr",
|
|
flight_id=flight_id,
|
|
config=config,
|
|
fdr_clients=[client],
|
|
gcs_alert=alerts.append,
|
|
)
|
|
return writer, client, alerts
|
|
|
|
|
|
def test_ac1_flight_header_is_first_record(tmp_path: Path) -> None:
|
|
# Arrange
|
|
flight_id = uuid4()
|
|
writer, _client, _alerts = _build_writer(tmp_path, flight_id)
|
|
header = _make_header(flight_id)
|
|
writer.start()
|
|
|
|
# Act
|
|
writer.open_flight(header)
|
|
footer = writer.close_flight()
|
|
|
|
# Assert
|
|
records = _read_records(writer.flight_dir)
|
|
assert records[0].kind == "flight_header"
|
|
assert records[0].payload["flight_id"] == str(flight_id)
|
|
assert footer.flight_id == flight_id
|
|
|
|
|
|
def test_ac2_flight_footer_is_last_record(tmp_path: Path) -> None:
|
|
# Arrange
|
|
flight_id = uuid4()
|
|
writer, client, _alerts = _build_writer(tmp_path, flight_id)
|
|
writer.start()
|
|
writer.open_flight(_make_header(flight_id))
|
|
for i in range(20):
|
|
client.enqueue(_payload(i))
|
|
|
|
# Act
|
|
deadline = time.monotonic() + 3.0
|
|
while time.monotonic() < deadline and client._buffer_size() > 0:
|
|
time.sleep(0.01)
|
|
footer = writer.close_flight()
|
|
|
|
# Assert
|
|
records = _read_records(writer.flight_dir)
|
|
assert records[-1].kind == "flight_footer"
|
|
assert records[-1].payload["clean_shutdown"] is True
|
|
# Returned footer matches the on-disk payload (modulo flight_id stringification).
|
|
on_disk = records[-1].payload
|
|
assert on_disk["records_written"] == footer.records_written
|
|
assert on_disk["records_dropped_overrun"] == footer.records_dropped_overrun
|
|
assert on_disk["bytes_written"] == footer.bytes_written
|
|
assert on_disk["rollover_count"] == footer.rollover_count
|
|
|
|
|
|
def test_ac3_counters_reflect_on_disk_reality(tmp_path: Path) -> None:
|
|
# Arrange
|
|
flight_id = uuid4()
|
|
config = FdrWriterConfig(segment_size_bytes=2048, batch_size=8, flight_cap_bytes=1024**3)
|
|
writer, client, _alerts = _build_writer(tmp_path, flight_id, config)
|
|
writer.start()
|
|
writer.open_flight(_make_header(flight_id))
|
|
R = 30
|
|
for i in range(R):
|
|
client.enqueue(_payload(i))
|
|
|
|
# Act
|
|
deadline = time.monotonic() + 5.0
|
|
while time.monotonic() < deadline and client._buffer_size() > 0:
|
|
time.sleep(0.01)
|
|
footer = writer.close_flight()
|
|
|
|
# Assert
|
|
expected_written = R + 2
|
|
assert footer.records_written == expected_written
|
|
assert footer.records_dropped_overrun == 0
|
|
# rollover_count: per-segment rotations (counter is monotonic; the test
|
|
# writer's small cap forces ≥1 rotation).
|
|
assert footer.rollover_count >= 1
|
|
# Cross-check: footer.rollover_count == observed segment_index at close.
|
|
assert footer.rollover_count == writer.rollover_count
|
|
|
|
|
|
def test_ac4_open_flight_fdrerror_on_disk_failure(tmp_path: Path) -> None:
|
|
# Arrange — start the writer normally, then point segment 0 to a
|
|
# read-only directory.
|
|
flight_id = uuid4()
|
|
config = FdrWriterConfig()
|
|
alerts: list[str] = []
|
|
client = _make_client()
|
|
ro_root = tmp_path / "ro"
|
|
ro_root.mkdir(parents=True)
|
|
# Create a stand-in flight_dir read-only file pretending to be the segment.
|
|
ro_flight = ro_root / str(flight_id)
|
|
ro_flight.mkdir()
|
|
seg = ro_flight / "segment-0000.fdr"
|
|
seg.write_bytes(b"")
|
|
seg.chmod(0o400)
|
|
|
|
writer = FileFdrWriter(
|
|
flight_root=ro_root,
|
|
flight_id=flight_id,
|
|
config=config,
|
|
fdr_clients=[client],
|
|
gcs_alert=alerts.append,
|
|
)
|
|
# Block the path from being opened for write — chmod the parent dir.
|
|
ro_flight.chmod(0o500)
|
|
try:
|
|
writer.start()
|
|
except Exception:
|
|
# If start fails outright (cannot create flight_dir), that satisfies the contract.
|
|
ro_flight.chmod(0o700)
|
|
return
|
|
|
|
# If start did succeed (writer created the file before chmod took effect),
|
|
# explicitly fail open_flight by writing the header via a forced OSError.
|
|
seg.chmod(0o400)
|
|
with pytest.raises(FdrOpenError):
|
|
from unittest import mock
|
|
|
|
with mock.patch(
|
|
"gps_denied_onboard.components.c13_fdr.writer.os.write",
|
|
side_effect=PermissionError("read-only"),
|
|
):
|
|
writer.open_flight(_make_header(flight_id))
|
|
|
|
# Cleanup
|
|
ro_flight.chmod(0o700)
|
|
|
|
|
|
def test_ac5_open_flight_rejects_flight_id_mismatch(tmp_path: Path) -> None:
|
|
# Arrange
|
|
writer_flight_id = uuid4()
|
|
other_flight_id = uuid4()
|
|
writer, _client, _alerts = _build_writer(tmp_path, writer_flight_id)
|
|
writer.start()
|
|
bad_header = _make_header(other_flight_id)
|
|
|
|
# Act, Assert
|
|
with pytest.raises(FdrOpenError, match="does not match"):
|
|
writer.open_flight(bad_header)
|
|
|
|
|
|
def test_ac6_close_without_open_raises(tmp_path: Path) -> None:
|
|
# Arrange
|
|
flight_id = uuid4()
|
|
writer, _client, _alerts = _build_writer(tmp_path, flight_id)
|
|
writer.start()
|
|
|
|
# Act, Assert
|
|
with pytest.raises(FdrCloseWithoutOpenError):
|
|
writer.close_flight()
|
|
writer.stop()
|
|
|
|
|
|
def test_ac7_uncleansed_teardown_no_clean_shutdown(tmp_path: Path) -> None:
|
|
# Arrange
|
|
flight_id = uuid4()
|
|
writer, _client, _alerts = _build_writer(tmp_path, flight_id)
|
|
writer.start()
|
|
writer.open_flight(_make_header(flight_id))
|
|
|
|
# Act — stop without close_flight.
|
|
writer.stop()
|
|
|
|
# Assert — no flight_footer record exists, so post-flight tooling marks the flight truncated.
|
|
records = _read_records(writer.flight_dir)
|
|
has_footer = any(r.kind == "flight_footer" for r in records)
|
|
assert not has_footer
|
|
|
|
|
|
def test_ac8_records_dropped_overrun_aggregates_dropped_counts(tmp_path: Path) -> None:
|
|
# Arrange
|
|
flight_id = uuid4()
|
|
writer, client, _alerts = _build_writer(tmp_path, flight_id)
|
|
writer.start()
|
|
writer.open_flight(_make_header(flight_id))
|
|
drops = [3, 7, 2, 11, 4]
|
|
for d in drops:
|
|
client.enqueue(
|
|
FdrRecord(
|
|
schema_version=1,
|
|
ts=datetime.now(tz=timezone.utc).isoformat(),
|
|
producer_id=OVERRUN_PRODUCER_ID,
|
|
kind=OVERRUN_KIND,
|
|
payload={"producer_id": "c1_vio", "dropped_count": d},
|
|
)
|
|
)
|
|
|
|
# Act
|
|
deadline = time.monotonic() + 3.0
|
|
while time.monotonic() < deadline and client._buffer_size() > 0:
|
|
time.sleep(0.01)
|
|
footer = writer.close_flight()
|
|
|
|
# Assert
|
|
assert footer.records_dropped_overrun == sum(drops)
|
|
|
|
|
|
def test_close_flight_idempotent(tmp_path: Path) -> None:
|
|
# Arrange
|
|
flight_id = uuid4()
|
|
writer, _client, _alerts = _build_writer(tmp_path, flight_id)
|
|
writer.start()
|
|
writer.open_flight(_make_header(flight_id))
|
|
|
|
# Act
|
|
footer_a = writer.close_flight()
|
|
footer_b = writer.close_flight()
|
|
|
|
# Assert
|
|
assert footer_a == footer_b
|
|
|
|
|
|
def test_flight_header_and_footer_are_frozen() -> None:
|
|
# Arrange
|
|
header = _make_header(uuid4())
|
|
footer = FlightFooter(
|
|
flight_id=uuid4(),
|
|
flight_ended_at_iso="2026-05-11T00:00:00+00:00",
|
|
flight_ended_at_monotonic_ns=0,
|
|
records_written=1,
|
|
records_dropped_overrun=0,
|
|
bytes_written=0,
|
|
rollover_count=0,
|
|
clean_shutdown=True,
|
|
)
|
|
|
|
# Assert
|
|
with pytest.raises(dataclasses.FrozenInstanceError):
|
|
header.flight_id = uuid4() # type: ignore[misc]
|
|
with pytest.raises(dataclasses.FrozenInstanceError):
|
|
footer.records_written = 999 # type: ignore[misc]
|