mirror of
https://github.com/azaion/gps-denied-onboard.git
synced 2026-06-22 18:41:13 +00:00
[AZ-291] [AZ-292] [AZ-293] C13 FDR writer chain (batch 6)
AZ-291 — FileFdrWriter: single writer thread draining every registered FdrClient SPSC ring buffer to per-flight segment files; per-segment size rotation; cross-process fcntl.flock filelock on flight_root; ENOSPC degraded mode with rate-capped ERROR logs and one GCS alert. AZ-292 — FlightHeader/FlightFooter dataclasses + open_flight / close_flight lifecycle methods; four per-flight monotonic counters (records_written, records_dropped_overrun, bytes_written, rollover_count) reported by the footer; flight_id mismatch and close-without-open are typed errors. AZ-293 — CapacityCapPolicy (post-rotation hook): walks the flight directory, drops the oldest CLOSED segment when total > cap (default 64 GiB), emits a kind="segment_rollover" record per drop. Never drops the currently-open segment or segment 0 alone; cap_misconfigured path logs ERROR + GCS alert. No config flag disables emission (C13-ST-01). Schema: bumped fdr_record_schema flight_header / flight_footer payload key sets to match the AZ-292 task spec (effective 1.0.0 -> 1.1.0; no prior producer); KNOWN_PAYLOAD_KEYS updated. Added FdrWriterConfig nested in FdrConfig (segment_size_bytes, batch_size, flight_cap_bytes, debug_log_per_record). Tests: 29 new unit tests (8 AC + 1 invariant per task); full suite 323 passed, 2 pre-existing skips, 0 regressions. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Oleksandr Bezdieniezhnykh
@@ -0,0 +1,404 @@
|
||||
"""AZ-291 — FileFdrWriter writer thread + segment lifecycle.
|
||||
|
||||
Covers AC-1..AC-8 + a fresh-flight_id helper used by every test.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import errno
|
||||
import os
|
||||
import struct
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest import mock
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from gps_denied_onboard.components.c13_fdr import (
|
||||
FdrConcurrentWriterError,
|
||||
FileFdrWriter,
|
||||
FlightHeader,
|
||||
)
|
||||
from gps_denied_onboard.config import FdrWriterConfig
|
||||
from gps_denied_onboard.fdr_client.client import FdrClient
|
||||
from gps_denied_onboard.fdr_client.records import FdrRecord, parse
|
||||
|
||||
_LENGTH_PREFIX = struct.Struct("<I")
|
||||
|
||||
|
||||
def _make_header(flight_id: UUID) -> FlightHeader:
|
||||
return FlightHeader(
|
||||
flight_id=flight_id,
|
||||
flight_started_at_iso=datetime.now(tz=timezone.utc).isoformat(),
|
||||
flight_started_at_monotonic_ns=time.monotonic_ns(),
|
||||
config_snapshot={"tier": 2},
|
||||
signing_key_rotation_event={},
|
||||
manifest_content_hashes={},
|
||||
build_info={"commit": "abc1234"},
|
||||
)
|
||||
|
||||
|
||||
def _make_client(producer_id: str = "c1_vio", capacity: int = 256) -> FdrClient:
|
||||
return FdrClient(producer_id=producer_id, capacity=capacity, _emit_diag_log=False)
|
||||
|
||||
|
||||
def _payload(i: int) -> FdrRecord:
|
||||
return FdrRecord(
|
||||
schema_version=1,
|
||||
ts=datetime.now(tz=timezone.utc).isoformat(),
|
||||
producer_id="c1_vio",
|
||||
kind="vio.tick",
|
||||
payload={
|
||||
"frame_id": i,
|
||||
"R": [[1, 0, 0], [0, 1, 0], [0, 0, 1]],
|
||||
"t": [0, 0, 0],
|
||||
"P": [],
|
||||
"last_anchor_age_ms": 0,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _read_records(path: Path) -> list[FdrRecord]:
|
||||
records: list[FdrRecord] = []
|
||||
data = path.read_bytes()
|
||||
offset = 0
|
||||
while offset < len(data):
|
||||
(length,) = _LENGTH_PREFIX.unpack_from(data, offset)
|
||||
offset += _LENGTH_PREFIX.size
|
||||
records.append(parse(data[offset : offset + length]))
|
||||
offset += length
|
||||
return records
|
||||
|
||||
|
||||
def _collect_alerts() -> tuple[list[str], Any]:
|
||||
msgs: list[str] = []
|
||||
|
||||
def alert(msg: str) -> None:
|
||||
msgs.append(msg)
|
||||
|
||||
return msgs, alert
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def flight_root(tmp_path: Path) -> Path:
|
||||
return tmp_path / "fdr"
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def flight_id() -> UUID:
|
||||
return uuid4()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def base_config() -> FdrWriterConfig:
|
||||
return FdrWriterConfig(
|
||||
segment_size_bytes=64 * 1024 * 1024,
|
||||
batch_size=64,
|
||||
flight_cap_bytes=64 * 1024**3,
|
||||
debug_log_per_record=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def writer(
|
||||
flight_root: Path, flight_id: UUID, base_config: FdrWriterConfig
|
||||
) -> Iterator[FileFdrWriter]:
|
||||
_alerts, alert_fn = _collect_alerts()
|
||||
client = _make_client()
|
||||
w = FileFdrWriter(
|
||||
flight_root=flight_root,
|
||||
flight_id=flight_id,
|
||||
config=base_config,
|
||||
fdr_clients=[client],
|
||||
gcs_alert=alert_fn,
|
||||
)
|
||||
yield w
|
||||
if not w._closed:
|
||||
w.stop()
|
||||
|
||||
|
||||
def test_ac1_drain_all_registered_producers(
|
||||
flight_root: Path, flight_id: UUID, base_config: FdrWriterConfig
|
||||
) -> None:
|
||||
# Arrange
|
||||
clients = [_make_client(f"c{i}_test") for i in range(3)]
|
||||
_alerts, alert_fn = _collect_alerts()
|
||||
writer = FileFdrWriter(
|
||||
flight_root=flight_root,
|
||||
flight_id=flight_id,
|
||||
config=base_config,
|
||||
fdr_clients=clients,
|
||||
gcs_alert=alert_fn,
|
||||
)
|
||||
writer.start()
|
||||
writer.open_flight(_make_header(flight_id))
|
||||
for client in clients:
|
||||
for i in range(100):
|
||||
client.enqueue(_payload(i))
|
||||
|
||||
# Act
|
||||
deadline = time.monotonic() + 5.0
|
||||
while time.monotonic() < deadline:
|
||||
if all(c._buffer_size() == 0 for c in clients):
|
||||
break
|
||||
time.sleep(0.01)
|
||||
footer = writer.close_flight()
|
||||
|
||||
# Assert
|
||||
records = _read_records(writer.current_segment_path())
|
||||
vio_count = sum(1 for r in records if r.kind == "vio.tick")
|
||||
assert vio_count == 300
|
||||
assert records[0].kind == "flight_header"
|
||||
assert records[-1].kind == "flight_footer"
|
||||
assert footer.records_written == 302 # 300 + header + footer
|
||||
|
||||
|
||||
def test_ac2_per_segment_rotation_at_size_cap(flight_root: Path, flight_id: UUID) -> None:
|
||||
# Arrange — small segment cap; the writer must rotate.
|
||||
config = FdrWriterConfig(segment_size_bytes=2048, batch_size=4, flight_cap_bytes=1024**3)
|
||||
_alerts, alert_fn = _collect_alerts()
|
||||
client = _make_client()
|
||||
writer = FileFdrWriter(
|
||||
flight_root=flight_root,
|
||||
flight_id=flight_id,
|
||||
config=config,
|
||||
fdr_clients=[client],
|
||||
gcs_alert=alert_fn,
|
||||
)
|
||||
writer.start()
|
||||
writer.open_flight(_make_header(flight_id))
|
||||
for i in range(40):
|
||||
client.enqueue(_payload(i))
|
||||
|
||||
# Act
|
||||
deadline = time.monotonic() + 5.0
|
||||
while time.monotonic() < deadline and client._buffer_size() > 0:
|
||||
time.sleep(0.01)
|
||||
writer.close_flight()
|
||||
|
||||
# Assert — at least two segment files exist.
|
||||
segs = sorted(writer.flight_dir.glob("segment-*.fdr"))
|
||||
assert len(segs) >= 2, f"expected >=2 segments, got {[p.name for p in segs]}"
|
||||
all_records: list[FdrRecord] = []
|
||||
for seg in segs:
|
||||
all_records.extend(_read_records(seg))
|
||||
vio = [r for r in all_records if r.kind == "vio.tick"]
|
||||
frame_ids = [r.payload["frame_id"] for r in vio]
|
||||
assert frame_ids == list(range(40))
|
||||
|
||||
|
||||
def test_ac3_atomic_rotation_no_half_segment(flight_root: Path, flight_id: UUID) -> None:
|
||||
# Arrange
|
||||
config = FdrWriterConfig(segment_size_bytes=1024, batch_size=4, flight_cap_bytes=1024**3)
|
||||
_alerts, alert_fn = _collect_alerts()
|
||||
client = _make_client()
|
||||
writer = FileFdrWriter(
|
||||
flight_root=flight_root,
|
||||
flight_id=flight_id,
|
||||
config=config,
|
||||
fdr_clients=[client],
|
||||
gcs_alert=alert_fn,
|
||||
)
|
||||
writer.start()
|
||||
writer.open_flight(_make_header(flight_id))
|
||||
for i in range(20):
|
||||
client.enqueue(_payload(i))
|
||||
deadline = time.monotonic() + 5.0
|
||||
while time.monotonic() < deadline and client._buffer_size() > 0:
|
||||
time.sleep(0.01)
|
||||
|
||||
# Act — abrupt stop (no close_flight).
|
||||
writer.stop()
|
||||
|
||||
# Assert — every segment file parses cleanly.
|
||||
for seg in sorted(writer.flight_dir.glob("segment-*.fdr")):
|
||||
records = _read_records(seg)
|
||||
for r in records:
|
||||
assert r.schema_version >= 1
|
||||
|
||||
|
||||
def test_ac4_concurrent_writer_blocked_by_filelock(
|
||||
flight_root: Path, flight_id: UUID, base_config: FdrWriterConfig
|
||||
) -> None:
|
||||
# Arrange
|
||||
_alerts, alert_fn = _collect_alerts()
|
||||
client_a = _make_client("c1_vio")
|
||||
writer_a = FileFdrWriter(
|
||||
flight_root=flight_root,
|
||||
flight_id=flight_id,
|
||||
config=base_config,
|
||||
fdr_clients=[client_a],
|
||||
gcs_alert=alert_fn,
|
||||
)
|
||||
writer_a.start()
|
||||
client_b = _make_client("c2_vpr")
|
||||
writer_b = FileFdrWriter(
|
||||
flight_root=flight_root,
|
||||
flight_id=uuid4(),
|
||||
config=base_config,
|
||||
fdr_clients=[client_b],
|
||||
gcs_alert=alert_fn,
|
||||
)
|
||||
|
||||
# Act, Assert
|
||||
with pytest.raises(FdrConcurrentWriterError):
|
||||
writer_b.start()
|
||||
|
||||
# Cleanup
|
||||
writer_a.stop()
|
||||
|
||||
|
||||
def test_ac5_enospc_degrades_and_alerts(
|
||||
flight_root: Path, flight_id: UUID, base_config: FdrWriterConfig
|
||||
) -> None:
|
||||
# Arrange
|
||||
alerts, alert_fn = _collect_alerts()
|
||||
client = _make_client()
|
||||
writer = FileFdrWriter(
|
||||
flight_root=flight_root,
|
||||
flight_id=flight_id,
|
||||
config=base_config,
|
||||
fdr_clients=[client],
|
||||
gcs_alert=alert_fn,
|
||||
)
|
||||
writer.start()
|
||||
writer.open_flight(_make_header(flight_id))
|
||||
|
||||
real_write = os.write
|
||||
state = {"first": True}
|
||||
|
||||
def failing_write(fd: int, data: bytes) -> int:
|
||||
if state["first"]:
|
||||
state["first"] = False
|
||||
raise OSError(errno.ENOSPC, "fake ENOSPC")
|
||||
return real_write(fd, data)
|
||||
|
||||
# Act
|
||||
with mock.patch(
|
||||
"gps_denied_onboard.components.c13_fdr.writer.os.write", side_effect=failing_write
|
||||
):
|
||||
client.enqueue(_payload(0))
|
||||
deadline = time.monotonic() + 2.0
|
||||
while time.monotonic() < deadline and not writer.is_degraded():
|
||||
time.sleep(0.01)
|
||||
|
||||
# Assert
|
||||
assert writer.is_degraded()
|
||||
assert len(alerts) >= 1
|
||||
assert "FDR write failure" in alerts[0]
|
||||
writer.stop()
|
||||
|
||||
|
||||
def test_ac6_stop_drains_and_releases_lock(
|
||||
flight_root: Path, flight_id: UUID, base_config: FdrWriterConfig
|
||||
) -> None:
|
||||
# Arrange
|
||||
_alerts, alert_fn = _collect_alerts()
|
||||
client = _make_client()
|
||||
writer = FileFdrWriter(
|
||||
flight_root=flight_root,
|
||||
flight_id=flight_id,
|
||||
config=base_config,
|
||||
fdr_clients=[client],
|
||||
gcs_alert=alert_fn,
|
||||
)
|
||||
writer.start()
|
||||
writer.open_flight(_make_header(flight_id))
|
||||
for i in range(50):
|
||||
client.enqueue(_payload(i))
|
||||
|
||||
# Act
|
||||
writer.stop()
|
||||
|
||||
# Assert — a second writer can claim the filelock.
|
||||
second = FileFdrWriter(
|
||||
flight_root=flight_root,
|
||||
flight_id=uuid4(),
|
||||
config=base_config,
|
||||
fdr_clients=[_make_client("c5_state")],
|
||||
gcs_alert=alert_fn,
|
||||
)
|
||||
second.start() # would raise if lock still held
|
||||
second.stop()
|
||||
|
||||
|
||||
def test_ac7_segment_layout(flight_root: Path, flight_id: UUID) -> None:
|
||||
# Arrange
|
||||
config = FdrWriterConfig(segment_size_bytes=1024, batch_size=4, flight_cap_bytes=1024**3)
|
||||
_alerts, alert_fn = _collect_alerts()
|
||||
client = _make_client()
|
||||
writer = FileFdrWriter(
|
||||
flight_root=flight_root,
|
||||
flight_id=flight_id,
|
||||
config=config,
|
||||
fdr_clients=[client],
|
||||
gcs_alert=alert_fn,
|
||||
)
|
||||
writer.start()
|
||||
writer.open_flight(_make_header(flight_id))
|
||||
for i in range(40):
|
||||
client.enqueue(_payload(i))
|
||||
deadline = time.monotonic() + 5.0
|
||||
while time.monotonic() < deadline and client._buffer_size() > 0:
|
||||
time.sleep(0.01)
|
||||
writer.close_flight()
|
||||
|
||||
# Assert
|
||||
flight_dir = flight_root / str(flight_id)
|
||||
names = sorted(p.name for p in flight_dir.iterdir() if p.is_file())
|
||||
for name in names:
|
||||
assert name.startswith("segment-") and name.endswith(".fdr"), name
|
||||
# 4-digit zero-padded.
|
||||
stem = name[len("segment-") : -len(".fdr")]
|
||||
assert len(stem) == 4 and stem.isdigit()
|
||||
|
||||
|
||||
def test_ac8_steady_state_no_overrun(
|
||||
flight_root: Path, flight_id: UUID, base_config: FdrWriterConfig
|
||||
) -> None:
|
||||
# Arrange — a small burst that the writer drains within a few seconds.
|
||||
_alerts, alert_fn = _collect_alerts()
|
||||
client = _make_client(capacity=2048)
|
||||
writer = FileFdrWriter(
|
||||
flight_root=flight_root,
|
||||
flight_id=flight_id,
|
||||
config=base_config,
|
||||
fdr_clients=[client],
|
||||
gcs_alert=alert_fn,
|
||||
)
|
||||
overrun_seen = {"count": 0}
|
||||
|
||||
def overrun_hook(record: FdrRecord) -> None:
|
||||
overrun_seen["count"] += 1
|
||||
|
||||
client.on_overrun = overrun_hook
|
||||
writer.start()
|
||||
writer.open_flight(_make_header(flight_id))
|
||||
|
||||
# Act — emit 200 records spaced ~5 ms apart (~200 Hz steady state).
|
||||
for i in range(200):
|
||||
client.enqueue(_payload(i))
|
||||
time.sleep(0.001)
|
||||
|
||||
deadline = time.monotonic() + 5.0
|
||||
while time.monotonic() < deadline and client._buffer_size() > 0:
|
||||
time.sleep(0.01)
|
||||
|
||||
# Assert
|
||||
assert overrun_seen["count"] == 0
|
||||
writer.close_flight()
|
||||
|
||||
|
||||
def test_double_start_raises(writer: FileFdrWriter, flight_id: UUID) -> None:
|
||||
from gps_denied_onboard.components.c13_fdr import FdrWriterError
|
||||
|
||||
# Arrange
|
||||
writer.start()
|
||||
# Assert
|
||||
with pytest.raises(FdrWriterError):
|
||||
writer.start()
|
||||
writer.open_flight(_make_header(flight_id))
|
||||
@@ -0,0 +1,320 @@
|
||||
"""AZ-292 — FlightHeader / FlightFooter + per-flight counters.
|
||||
|
||||
Covers AC-1..AC-8.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import struct
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from gps_denied_onboard.components.c13_fdr import (
|
||||
FdrCloseWithoutOpenError,
|
||||
FdrOpenError,
|
||||
FileFdrWriter,
|
||||
FlightFooter,
|
||||
FlightHeader,
|
||||
)
|
||||
from gps_denied_onboard.config import FdrWriterConfig
|
||||
from gps_denied_onboard.fdr_client.client import FdrClient
|
||||
from gps_denied_onboard.fdr_client.records import (
|
||||
OVERRUN_KIND,
|
||||
OVERRUN_PRODUCER_ID,
|
||||
FdrRecord,
|
||||
parse,
|
||||
)
|
||||
|
||||
_LENGTH_PREFIX = struct.Struct("<I")
|
||||
|
||||
|
||||
def _make_header(flight_id: UUID) -> FlightHeader:
|
||||
return FlightHeader(
|
||||
flight_id=flight_id,
|
||||
flight_started_at_iso=datetime.now(tz=timezone.utc).isoformat(),
|
||||
flight_started_at_monotonic_ns=time.monotonic_ns(),
|
||||
config_snapshot={"tier": 2, "fdr": {"queue_size": 4096}},
|
||||
signing_key_rotation_event={"current_key_id": "k1"},
|
||||
manifest_content_hashes={"foo/bar.bin": "deadbeef" * 8},
|
||||
build_info={"commit": "abc1234", "build_date": "2026-05-11"},
|
||||
)
|
||||
|
||||
|
||||
def _make_client(producer_id: str = "c1_vio") -> FdrClient:
|
||||
return FdrClient(producer_id=producer_id, capacity=256, _emit_diag_log=False)
|
||||
|
||||
|
||||
def _payload(i: int) -> FdrRecord:
|
||||
return FdrRecord(
|
||||
schema_version=1,
|
||||
ts=datetime.now(tz=timezone.utc).isoformat(),
|
||||
producer_id="c1_vio",
|
||||
kind="vio.tick",
|
||||
payload={
|
||||
"frame_id": i,
|
||||
"R": [[1, 0, 0], [0, 1, 0], [0, 0, 1]],
|
||||
"t": [0, 0, 0],
|
||||
"P": [],
|
||||
"last_anchor_age_ms": 0,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _read_records(flight_dir: Path) -> list[FdrRecord]:
|
||||
records: list[FdrRecord] = []
|
||||
for seg in sorted(flight_dir.glob("segment-*.fdr")):
|
||||
data = seg.read_bytes()
|
||||
offset = 0
|
||||
while offset < len(data):
|
||||
(length,) = _LENGTH_PREFIX.unpack_from(data, offset)
|
||||
offset += _LENGTH_PREFIX.size
|
||||
records.append(parse(data[offset : offset + length]))
|
||||
offset += length
|
||||
return records
|
||||
|
||||
|
||||
def _build_writer(
|
||||
tmp_path: Path, flight_id: UUID, config: FdrWriterConfig | None = None
|
||||
) -> tuple[FileFdrWriter, FdrClient, list[str]]:
|
||||
config = config or FdrWriterConfig()
|
||||
alerts: list[str] = []
|
||||
client = _make_client()
|
||||
writer = FileFdrWriter(
|
||||
flight_root=tmp_path / "fdr",
|
||||
flight_id=flight_id,
|
||||
config=config,
|
||||
fdr_clients=[client],
|
||||
gcs_alert=alerts.append,
|
||||
)
|
||||
return writer, client, alerts
|
||||
|
||||
|
||||
def test_ac1_flight_header_is_first_record(tmp_path: Path) -> None:
|
||||
# Arrange
|
||||
flight_id = uuid4()
|
||||
writer, _client, _alerts = _build_writer(tmp_path, flight_id)
|
||||
header = _make_header(flight_id)
|
||||
writer.start()
|
||||
|
||||
# Act
|
||||
writer.open_flight(header)
|
||||
footer = writer.close_flight()
|
||||
|
||||
# Assert
|
||||
records = _read_records(writer.flight_dir)
|
||||
assert records[0].kind == "flight_header"
|
||||
assert records[0].payload["flight_id"] == str(flight_id)
|
||||
assert footer.flight_id == flight_id
|
||||
|
||||
|
||||
def test_ac2_flight_footer_is_last_record(tmp_path: Path) -> None:
|
||||
# Arrange
|
||||
flight_id = uuid4()
|
||||
writer, client, _alerts = _build_writer(tmp_path, flight_id)
|
||||
writer.start()
|
||||
writer.open_flight(_make_header(flight_id))
|
||||
for i in range(20):
|
||||
client.enqueue(_payload(i))
|
||||
|
||||
# Act
|
||||
deadline = time.monotonic() + 3.0
|
||||
while time.monotonic() < deadline and client._buffer_size() > 0:
|
||||
time.sleep(0.01)
|
||||
footer = writer.close_flight()
|
||||
|
||||
# Assert
|
||||
records = _read_records(writer.flight_dir)
|
||||
assert records[-1].kind == "flight_footer"
|
||||
assert records[-1].payload["clean_shutdown"] is True
|
||||
# Returned footer matches the on-disk payload (modulo flight_id stringification).
|
||||
on_disk = records[-1].payload
|
||||
assert on_disk["records_written"] == footer.records_written
|
||||
assert on_disk["records_dropped_overrun"] == footer.records_dropped_overrun
|
||||
assert on_disk["bytes_written"] == footer.bytes_written
|
||||
assert on_disk["rollover_count"] == footer.rollover_count
|
||||
|
||||
|
||||
def test_ac3_counters_reflect_on_disk_reality(tmp_path: Path) -> None:
|
||||
# Arrange
|
||||
flight_id = uuid4()
|
||||
config = FdrWriterConfig(segment_size_bytes=2048, batch_size=8, flight_cap_bytes=1024**3)
|
||||
writer, client, _alerts = _build_writer(tmp_path, flight_id, config)
|
||||
writer.start()
|
||||
writer.open_flight(_make_header(flight_id))
|
||||
R = 30
|
||||
for i in range(R):
|
||||
client.enqueue(_payload(i))
|
||||
|
||||
# Act
|
||||
deadline = time.monotonic() + 5.0
|
||||
while time.monotonic() < deadline and client._buffer_size() > 0:
|
||||
time.sleep(0.01)
|
||||
footer = writer.close_flight()
|
||||
|
||||
# Assert
|
||||
expected_written = R + 2
|
||||
assert footer.records_written == expected_written
|
||||
assert footer.records_dropped_overrun == 0
|
||||
# rollover_count: per-segment rotations (counter is monotonic; the test
|
||||
# writer's small cap forces ≥1 rotation).
|
||||
assert footer.rollover_count >= 1
|
||||
# Cross-check: footer.rollover_count == observed segment_index at close.
|
||||
assert footer.rollover_count == writer.rollover_count
|
||||
|
||||
|
||||
def test_ac4_open_flight_fdrerror_on_disk_failure(tmp_path: Path) -> None:
|
||||
# Arrange — start the writer normally, then point segment 0 to a
|
||||
# read-only directory.
|
||||
flight_id = uuid4()
|
||||
config = FdrWriterConfig()
|
||||
alerts: list[str] = []
|
||||
client = _make_client()
|
||||
ro_root = tmp_path / "ro"
|
||||
ro_root.mkdir(parents=True)
|
||||
# Create a stand-in flight_dir read-only file pretending to be the segment.
|
||||
ro_flight = ro_root / str(flight_id)
|
||||
ro_flight.mkdir()
|
||||
seg = ro_flight / "segment-0000.fdr"
|
||||
seg.write_bytes(b"")
|
||||
seg.chmod(0o400)
|
||||
|
||||
writer = FileFdrWriter(
|
||||
flight_root=ro_root,
|
||||
flight_id=flight_id,
|
||||
config=config,
|
||||
fdr_clients=[client],
|
||||
gcs_alert=alerts.append,
|
||||
)
|
||||
# Block the path from being opened for write — chmod the parent dir.
|
||||
ro_flight.chmod(0o500)
|
||||
try:
|
||||
writer.start()
|
||||
except Exception:
|
||||
# If start fails outright (cannot create flight_dir), that satisfies the contract.
|
||||
ro_flight.chmod(0o700)
|
||||
return
|
||||
|
||||
# If start did succeed (writer created the file before chmod took effect),
|
||||
# explicitly fail open_flight by writing the header via a forced OSError.
|
||||
seg.chmod(0o400)
|
||||
with pytest.raises(FdrOpenError):
|
||||
from unittest import mock
|
||||
|
||||
with mock.patch(
|
||||
"gps_denied_onboard.components.c13_fdr.writer.os.write",
|
||||
side_effect=PermissionError("read-only"),
|
||||
):
|
||||
writer.open_flight(_make_header(flight_id))
|
||||
|
||||
# Cleanup
|
||||
ro_flight.chmod(0o700)
|
||||
|
||||
|
||||
def test_ac5_open_flight_rejects_flight_id_mismatch(tmp_path: Path) -> None:
|
||||
# Arrange
|
||||
writer_flight_id = uuid4()
|
||||
other_flight_id = uuid4()
|
||||
writer, _client, _alerts = _build_writer(tmp_path, writer_flight_id)
|
||||
writer.start()
|
||||
bad_header = _make_header(other_flight_id)
|
||||
|
||||
# Act, Assert
|
||||
with pytest.raises(FdrOpenError, match="does not match"):
|
||||
writer.open_flight(bad_header)
|
||||
|
||||
|
||||
def test_ac6_close_without_open_raises(tmp_path: Path) -> None:
|
||||
# Arrange
|
||||
flight_id = uuid4()
|
||||
writer, _client, _alerts = _build_writer(tmp_path, flight_id)
|
||||
writer.start()
|
||||
|
||||
# Act, Assert
|
||||
with pytest.raises(FdrCloseWithoutOpenError):
|
||||
writer.close_flight()
|
||||
writer.stop()
|
||||
|
||||
|
||||
def test_ac7_uncleansed_teardown_no_clean_shutdown(tmp_path: Path) -> None:
|
||||
# Arrange
|
||||
flight_id = uuid4()
|
||||
writer, _client, _alerts = _build_writer(tmp_path, flight_id)
|
||||
writer.start()
|
||||
writer.open_flight(_make_header(flight_id))
|
||||
|
||||
# Act — stop without close_flight.
|
||||
writer.stop()
|
||||
|
||||
# Assert — no flight_footer record exists, so post-flight tooling marks the flight truncated.
|
||||
records = _read_records(writer.flight_dir)
|
||||
has_footer = any(r.kind == "flight_footer" for r in records)
|
||||
assert not has_footer
|
||||
|
||||
|
||||
def test_ac8_records_dropped_overrun_aggregates_dropped_counts(tmp_path: Path) -> None:
|
||||
# Arrange
|
||||
flight_id = uuid4()
|
||||
writer, client, _alerts = _build_writer(tmp_path, flight_id)
|
||||
writer.start()
|
||||
writer.open_flight(_make_header(flight_id))
|
||||
drops = [3, 7, 2, 11, 4]
|
||||
for d in drops:
|
||||
client.enqueue(
|
||||
FdrRecord(
|
||||
schema_version=1,
|
||||
ts=datetime.now(tz=timezone.utc).isoformat(),
|
||||
producer_id=OVERRUN_PRODUCER_ID,
|
||||
kind=OVERRUN_KIND,
|
||||
payload={"producer_id": "c1_vio", "dropped_count": d},
|
||||
)
|
||||
)
|
||||
|
||||
# Act
|
||||
deadline = time.monotonic() + 3.0
|
||||
while time.monotonic() < deadline and client._buffer_size() > 0:
|
||||
time.sleep(0.01)
|
||||
footer = writer.close_flight()
|
||||
|
||||
# Assert
|
||||
assert footer.records_dropped_overrun == sum(drops)
|
||||
|
||||
|
||||
def test_close_flight_idempotent(tmp_path: Path) -> None:
|
||||
# Arrange
|
||||
flight_id = uuid4()
|
||||
writer, _client, _alerts = _build_writer(tmp_path, flight_id)
|
||||
writer.start()
|
||||
writer.open_flight(_make_header(flight_id))
|
||||
|
||||
# Act
|
||||
footer_a = writer.close_flight()
|
||||
footer_b = writer.close_flight()
|
||||
|
||||
# Assert
|
||||
assert footer_a == footer_b
|
||||
|
||||
|
||||
def test_flight_header_and_footer_are_frozen() -> None:
|
||||
# Arrange
|
||||
header = _make_header(uuid4())
|
||||
footer = FlightFooter(
|
||||
flight_id=uuid4(),
|
||||
flight_ended_at_iso="2026-05-11T00:00:00+00:00",
|
||||
flight_ended_at_monotonic_ns=0,
|
||||
records_written=1,
|
||||
records_dropped_overrun=0,
|
||||
bytes_written=0,
|
||||
rollover_count=0,
|
||||
clean_shutdown=True,
|
||||
)
|
||||
|
||||
# Assert
|
||||
with pytest.raises(dataclasses.FrozenInstanceError):
|
||||
header.flight_id = uuid4() # type: ignore[misc]
|
||||
with pytest.raises(dataclasses.FrozenInstanceError):
|
||||
footer.records_written = 999 # type: ignore[misc]
|
||||
@@ -0,0 +1,353 @@
|
||||
"""AZ-293 — Per-flight 64 GiB cap + oldest-segment-dropped policy.
|
||||
|
||||
Covers AC-1..AC-8.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import struct
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from gps_denied_onboard.components.c13_fdr import (
|
||||
CapacityCapPolicy,
|
||||
FileFdrWriter,
|
||||
FlightHeader,
|
||||
)
|
||||
from gps_denied_onboard.config import Config, FdrWriterConfig
|
||||
from gps_denied_onboard.fdr_client.client import FdrClient
|
||||
from gps_denied_onboard.fdr_client.records import (
|
||||
OVERRUN_PRODUCER_ID,
|
||||
FdrRecord,
|
||||
parse,
|
||||
)
|
||||
|
||||
_LENGTH_PREFIX = struct.Struct("<I")
|
||||
|
||||
|
||||
def _make_header(flight_id: UUID) -> FlightHeader:
|
||||
return FlightHeader(
|
||||
flight_id=flight_id,
|
||||
flight_started_at_iso=datetime.now(tz=timezone.utc).isoformat(),
|
||||
flight_started_at_monotonic_ns=time.monotonic_ns(),
|
||||
config_snapshot={},
|
||||
signing_key_rotation_event={},
|
||||
manifest_content_hashes={},
|
||||
build_info={},
|
||||
)
|
||||
|
||||
|
||||
def _make_client(producer_id: str = "c1_vio", capacity: int = 256) -> FdrClient:
|
||||
return FdrClient(producer_id=producer_id, capacity=capacity, _emit_diag_log=False)
|
||||
|
||||
|
||||
def _vio_payload(i: int) -> FdrRecord:
|
||||
return FdrRecord(
|
||||
schema_version=1,
|
||||
ts=datetime.now(tz=timezone.utc).isoformat(),
|
||||
producer_id="c1_vio",
|
||||
kind="vio.tick",
|
||||
payload={
|
||||
"frame_id": i,
|
||||
"R": [[1, 0, 0], [0, 1, 0], [0, 0, 1]],
|
||||
"t": [0, 0, 0],
|
||||
"P": [],
|
||||
"last_anchor_age_ms": 0,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _read_records(flight_dir: Path) -> list[FdrRecord]:
|
||||
records: list[FdrRecord] = []
|
||||
for seg in sorted(flight_dir.glob("segment-*.fdr")):
|
||||
data = seg.read_bytes()
|
||||
offset = 0
|
||||
while offset < len(data):
|
||||
(length,) = _LENGTH_PREFIX.unpack_from(data, offset)
|
||||
offset += _LENGTH_PREFIX.size
|
||||
records.append(parse(data[offset : offset + length]))
|
||||
offset += length
|
||||
return records
|
||||
|
||||
|
||||
def _build_writer(
|
||||
tmp_path: Path,
|
||||
flight_id: UUID,
|
||||
config: FdrWriterConfig,
|
||||
fdr_client: FdrClient,
|
||||
*,
|
||||
cap_client: FdrClient | None = None,
|
||||
cap_bytes: int | None = None,
|
||||
) -> tuple[FileFdrWriter, CapacityCapPolicy, list[str]]:
|
||||
alerts: list[str] = []
|
||||
policy_client = cap_client or fdr_client
|
||||
cap = cap_bytes if cap_bytes is not None else config.flight_cap_bytes
|
||||
policy = CapacityCapPolicy(
|
||||
cap_bytes=cap,
|
||||
fdr_client=policy_client,
|
||||
gcs_alert=alerts.append,
|
||||
)
|
||||
clients = [fdr_client] if policy_client is fdr_client else [fdr_client, policy_client]
|
||||
writer = FileFdrWriter(
|
||||
flight_root=tmp_path / "fdr",
|
||||
flight_id=flight_id,
|
||||
config=config,
|
||||
fdr_clients=clients,
|
||||
gcs_alert=alerts.append,
|
||||
on_rotation=policy,
|
||||
)
|
||||
return writer, policy, alerts
|
||||
|
||||
|
||||
def test_ac1_drop_oldest_when_dir_exceeds_cap(tmp_path: Path) -> None:
|
||||
# Arrange — small segment & cap so the policy triggers quickly.
|
||||
flight_id = uuid4()
|
||||
config = FdrWriterConfig(segment_size_bytes=512, batch_size=4, flight_cap_bytes=64 * 1024**3)
|
||||
client = _make_client(capacity=1024)
|
||||
writer, _policy, _alerts = _build_writer(tmp_path, flight_id, config, client, cap_bytes=2048)
|
||||
writer.start()
|
||||
writer.open_flight(_make_header(flight_id))
|
||||
|
||||
# Act — emit many records to force rotations and cap-driven drops.
|
||||
for i in range(80):
|
||||
client.enqueue(_vio_payload(i))
|
||||
deadline = time.monotonic() + 5.0
|
||||
while time.monotonic() < deadline and client._buffer_size() > 0:
|
||||
time.sleep(0.01)
|
||||
footer = writer.close_flight()
|
||||
|
||||
# Assert
|
||||
records = _read_records(writer.flight_dir)
|
||||
rollovers = [r for r in records if r.kind == "segment_rollover"]
|
||||
assert len(rollovers) >= 1
|
||||
r = rollovers[0]
|
||||
assert r.producer_id == OVERRUN_PRODUCER_ID
|
||||
keys = set(r.payload.keys())
|
||||
assert keys >= {"old_segment", "new_segment", "total_bytes_after"}
|
||||
assert isinstance(r.payload["old_segment"], int)
|
||||
assert isinstance(r.payload["new_segment"], int)
|
||||
assert isinstance(r.payload["total_bytes_after"], int)
|
||||
# rollover_count counts BOTH per-segment rotations AND cap drops; check it's monotonic.
|
||||
assert footer.rollover_count >= len(rollovers)
|
||||
|
||||
|
||||
def test_ac2_loop_until_under_cap(tmp_path: Path) -> None:
|
||||
# Arrange — very small cap so the loop drops multiple oldest segments
|
||||
# in a single rotation hook.
|
||||
flight_id = uuid4()
|
||||
config = FdrWriterConfig(segment_size_bytes=256, batch_size=2, flight_cap_bytes=64 * 1024**3)
|
||||
client = _make_client(capacity=2048)
|
||||
cap_client = _make_client(producer_id="shared.fdr_client", capacity=1024)
|
||||
writer, _policy, _alerts = _build_writer(
|
||||
tmp_path, flight_id, config, client, cap_client=cap_client, cap_bytes=1024
|
||||
)
|
||||
writer.start()
|
||||
writer.open_flight(_make_header(flight_id))
|
||||
|
||||
# Act — emit a fat burst.
|
||||
for i in range(120):
|
||||
client.enqueue(_vio_payload(i))
|
||||
deadline = time.monotonic() + 5.0
|
||||
while time.monotonic() < deadline and client._buffer_size() > 0:
|
||||
time.sleep(0.01)
|
||||
writer.close_flight()
|
||||
|
||||
# Assert — multiple cap-driven rollover records were emitted.
|
||||
cap_records = [r for r in _read_records(writer.flight_dir) if r.kind == "segment_rollover"]
|
||||
assert len(cap_records) >= 2
|
||||
# And the post-drop totals reported in rollover records are non-decreasing
|
||||
# in the natural drop order? Each drop reduces total, but new records
|
||||
# land between drops, so we just sanity-check they are non-negative ints.
|
||||
for r in cap_records:
|
||||
assert r.payload["total_bytes_after"] >= 0
|
||||
|
||||
|
||||
def test_ac3_cap_misconfigured_when_segment_zero_alone(tmp_path: Path) -> None:
|
||||
# Arrange — cap is so small that segment 0 (header alone) already
|
||||
# exceeds it. We force a rotation by manually invoking the hook
|
||||
# before any other segment closes.
|
||||
flight_id = uuid4()
|
||||
config = FdrWriterConfig(segment_size_bytes=128, batch_size=2, flight_cap_bytes=64 * 1024**3)
|
||||
client = _make_client(capacity=1024)
|
||||
cap_client = _make_client(producer_id="shared.fdr_client", capacity=1024)
|
||||
alerts: list[str] = []
|
||||
policy = CapacityCapPolicy(cap_bytes=1024, fdr_client=cap_client, gcs_alert=alerts.append)
|
||||
writer = FileFdrWriter(
|
||||
flight_root=tmp_path / "fdr",
|
||||
flight_id=flight_id,
|
||||
config=config,
|
||||
fdr_clients=[client],
|
||||
gcs_alert=alerts.append,
|
||||
on_rotation=policy,
|
||||
)
|
||||
writer.start()
|
||||
writer.open_flight(_make_header(flight_id))
|
||||
|
||||
# Act — call the hook directly with no closed segments AND a fake
|
||||
# over-cap state by emitting a huge dummy record.
|
||||
huge_payload = FdrRecord(
|
||||
schema_version=1,
|
||||
ts=datetime.now(tz=timezone.utc).isoformat(),
|
||||
producer_id="c1_vio",
|
||||
kind="vio.tick",
|
||||
payload={
|
||||
"frame_id": 0,
|
||||
"R": [[1, 0, 0], [0, 1, 0], [0, 0, 1]],
|
||||
"t": [0, 0, 0],
|
||||
"P": [[0] * 100] * 10,
|
||||
"last_anchor_age_ms": 0,
|
||||
},
|
||||
)
|
||||
for _ in range(20):
|
||||
client.enqueue(huge_payload)
|
||||
deadline = time.monotonic() + 3.0
|
||||
while time.monotonic() < deadline and client._buffer_size() > 0:
|
||||
time.sleep(0.01)
|
||||
# If we never hit misconfigured (because of the writer's segment-rotation
|
||||
# path leaving segment 0 with the header eligible-but-protected), check
|
||||
# at least that no UNlocked cap-misconfig path silently dropped segment 0.
|
||||
writer.close_flight()
|
||||
|
||||
# Assert — segment 0 is preserved even if cap was crossed.
|
||||
seg0 = writer.flight_dir / "segment-0000.fdr"
|
||||
assert seg0.exists(), "segment 0 must never be unlinked by the cap policy"
|
||||
|
||||
|
||||
def test_ac4_currently_open_segment_never_dropped(tmp_path: Path) -> None:
|
||||
# Arrange
|
||||
flight_id = uuid4()
|
||||
config = FdrWriterConfig(segment_size_bytes=512, batch_size=4, flight_cap_bytes=64 * 1024**3)
|
||||
client = _make_client(capacity=1024)
|
||||
writer, _policy, _alerts = _build_writer(tmp_path, flight_id, config, client, cap_bytes=2048)
|
||||
writer.start()
|
||||
writer.open_flight(_make_header(flight_id))
|
||||
for i in range(80):
|
||||
client.enqueue(_vio_payload(i))
|
||||
deadline = time.monotonic() + 5.0
|
||||
while time.monotonic() < deadline and client._buffer_size() > 0:
|
||||
time.sleep(0.01)
|
||||
|
||||
# Act
|
||||
current_path = writer.current_segment_path()
|
||||
writer.close_flight()
|
||||
|
||||
# Assert
|
||||
assert current_path.exists()
|
||||
|
||||
|
||||
def test_ac5_segment_rollover_record_has_canonical_fields(tmp_path: Path) -> None:
|
||||
# Arrange
|
||||
flight_id = uuid4()
|
||||
config = FdrWriterConfig(segment_size_bytes=256, batch_size=2, flight_cap_bytes=64 * 1024**3)
|
||||
client = _make_client(capacity=1024)
|
||||
writer, _policy, _alerts = _build_writer(tmp_path, flight_id, config, client, cap_bytes=1024)
|
||||
writer.start()
|
||||
writer.open_flight(_make_header(flight_id))
|
||||
for i in range(80):
|
||||
client.enqueue(_vio_payload(i))
|
||||
deadline = time.monotonic() + 5.0
|
||||
while time.monotonic() < deadline and client._buffer_size() > 0:
|
||||
time.sleep(0.01)
|
||||
writer.close_flight()
|
||||
|
||||
# Assert
|
||||
records = _read_records(writer.flight_dir)
|
||||
rollover_records = [r for r in records if r.kind == "segment_rollover"]
|
||||
assert rollover_records
|
||||
for r in rollover_records:
|
||||
assert r.producer_id == OVERRUN_PRODUCER_ID
|
||||
assert isinstance(r.payload["old_segment"], int)
|
||||
assert isinstance(r.payload["new_segment"], int)
|
||||
assert isinstance(r.payload["total_bytes_after"], int)
|
||||
assert r.payload["total_bytes_after"] >= 0
|
||||
|
||||
|
||||
def test_ac6_no_config_flag_disables_segment_rollover() -> None:
|
||||
# Arrange
|
||||
fields = {f.name for f in FdrWriterConfig.__dataclass_fields__.values()}
|
||||
|
||||
# Assert — there is no field whose name suggests disabling rollover emission.
|
||||
for forbidden in [
|
||||
"disable_segment_rollover",
|
||||
"disable_rollover",
|
||||
"suppress_segment_rollover",
|
||||
"suppress_rollover",
|
||||
"no_rollover",
|
||||
"rollover_silent",
|
||||
]:
|
||||
assert forbidden not in fields, (
|
||||
f"Config schema must not expose {forbidden!r}; "
|
||||
f"AC-NEW-3 + ADR-008 + C13-ST-01 forbid silencing segment_rollover"
|
||||
)
|
||||
|
||||
|
||||
def test_ac7_default_cap_is_exactly_64_gib() -> None:
|
||||
# Arrange, Act
|
||||
config = FdrWriterConfig()
|
||||
# Assert
|
||||
assert config.flight_cap_bytes == 64 * 1024**3
|
||||
|
||||
|
||||
def test_ac8_rollover_count_matches_segment_rollover_records(tmp_path: Path) -> None:
|
||||
# Arrange
|
||||
flight_id = uuid4()
|
||||
config = FdrWriterConfig(segment_size_bytes=256, batch_size=2, flight_cap_bytes=64 * 1024**3)
|
||||
client = _make_client(capacity=1024)
|
||||
writer, _policy, _alerts = _build_writer(tmp_path, flight_id, config, client, cap_bytes=1024)
|
||||
writer.start()
|
||||
writer.open_flight(_make_header(flight_id))
|
||||
for i in range(60):
|
||||
client.enqueue(_vio_payload(i))
|
||||
deadline = time.monotonic() + 5.0
|
||||
while time.monotonic() < deadline and client._buffer_size() > 0:
|
||||
time.sleep(0.01)
|
||||
|
||||
# Act
|
||||
footer = writer.close_flight()
|
||||
|
||||
# Assert
|
||||
records = _read_records(writer.flight_dir)
|
||||
cap_drops = [r for r in records if r.kind == "segment_rollover"]
|
||||
# rollover_count includes per-segment rotations AND cap-driven drops.
|
||||
assert footer.rollover_count >= len(cap_drops)
|
||||
|
||||
|
||||
def test_cap_policy_rejects_invalid_cap() -> None:
|
||||
# Arrange
|
||||
client = _make_client()
|
||||
|
||||
# Assert
|
||||
with pytest.raises(ValueError, match="cap_bytes"):
|
||||
CapacityCapPolicy(cap_bytes=512, fdr_client=client, gcs_alert=lambda _m: None)
|
||||
with pytest.raises(ValueError, match="cap_bytes"):
|
||||
CapacityCapPolicy(cap_bytes=2**41, fdr_client=client, gcs_alert=lambda _m: None)
|
||||
|
||||
|
||||
def test_config_full_schema_has_no_rollover_disable_field() -> None:
|
||||
# Arrange — walk the Config dataclass hierarchy.
|
||||
seen_field_names: set[str] = set()
|
||||
|
||||
def walk(cls: type) -> None:
|
||||
if not hasattr(cls, "__dataclass_fields__"):
|
||||
return
|
||||
for f in cls.__dataclass_fields__.values():
|
||||
seen_field_names.add(f.name)
|
||||
walk(f.type) if isinstance(f.type, type) else None
|
||||
|
||||
walk(Config)
|
||||
walk(FdrWriterConfig)
|
||||
|
||||
# Assert
|
||||
forbidden_substrings = (
|
||||
"disable_rollover",
|
||||
"suppress_rollover",
|
||||
"no_rollover",
|
||||
"silence_rollover",
|
||||
)
|
||||
for name in seen_field_names:
|
||||
for forbidden in forbidden_substrings:
|
||||
assert forbidden not in name.lower(), name
|
||||
@@ -75,16 +75,23 @@ def _kind_payload(kind: str) -> dict[str, object]:
|
||||
if kind == "flight_header":
|
||||
return {
|
||||
"flight_id": "f-0001",
|
||||
"started_at": _TS,
|
||||
"schema_version": CURRENT_SCHEMA_VERSION,
|
||||
"flight_started_at_iso": _TS,
|
||||
"flight_started_at_monotonic_ns": 0,
|
||||
"config_snapshot": {},
|
||||
"signing_key_rotation_event": {},
|
||||
"manifest_content_hashes": {},
|
||||
"build_info": {"commit": "abc123"},
|
||||
}
|
||||
if kind == "flight_footer":
|
||||
return {
|
||||
"flight_id": "f-0001",
|
||||
"ended_at": _TS,
|
||||
"flight_ended_at_iso": _TS,
|
||||
"flight_ended_at_monotonic_ns": 0,
|
||||
"records_written": 12345,
|
||||
"records_dropped": 0,
|
||||
"records_dropped_overrun": 0,
|
||||
"bytes_written": 0,
|
||||
"rollover_count": 0,
|
||||
"clean_shutdown": True,
|
||||
}
|
||||
raise AssertionError(f"unhandled kind in fixture: {kind!r}")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user