mirror of
https://github.com/azaion/gps-denied-onboard.git
synced 2026-06-22 18:31:13 +00:00
[AZ-270] [AZ-272] [AZ-279] [AZ-281] [AZ-283] Compose root + FDR schema + 3 Layer-1 helpers
AZ-270: composition root with strategy registry, tier-gated lookup, topo-order construction, all-or-nothing teardown, StrategyNotLinkedError payload. AZ-272: orjson-backed FdrRecord serialise/parse with forward-compat for unknown payload + top-level fields and canonical overrun-record shape. AZ-279: pyproj-backed WGS84/ECEF/ENU + OSM slippy-map tile math with WgsConversionError for shape/range/zoom guards. AZ-281: strict EngineFilenameSchema build/parse/matches_host with anchored regex + enum validation; round-trip identity by construction. AZ-283: dtype-preserving (fp16/fp32) single + batch L2 normaliser with zero-norm safety and descriptor_metric() source-of-truth. pyproject.toml pins pyproj>=3.6 and orjson>=3.9 (named-backend deps per the AZ-272 / AZ-279 contracts). New DTOs LatLonAlt + BoundingBox and EngineCacheKey + HostCapabilities land in _types/ to back the helper contracts. 203 unit tests pass (64 new). Review verdict: PASS_WITH_WARNINGS; findings are perf-NFR deferrals + dep amendment + minor docstring polish. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -5,6 +5,27 @@ Producer-side API used by every component. Consumer-side writer lives in
|
||||
"""
|
||||
|
||||
from gps_denied_onboard.fdr_client.client import FdrClient
|
||||
from gps_denied_onboard.fdr_client.records import FdrRecord
|
||||
from gps_denied_onboard.fdr_client.records import (
|
||||
CURRENT_SCHEMA_VERSION,
|
||||
KNOWN_KINDS,
|
||||
MAX_INLINE_BLOB_BYTES,
|
||||
OVERRUN_KIND,
|
||||
OVERRUN_PRODUCER_ID,
|
||||
FdrRecord,
|
||||
FdrSchemaError,
|
||||
parse,
|
||||
serialise,
|
||||
)
|
||||
|
||||
__all__ = ["FdrClient", "FdrRecord"]
|
||||
__all__ = [
|
||||
"CURRENT_SCHEMA_VERSION",
|
||||
"KNOWN_KINDS",
|
||||
"MAX_INLINE_BLOB_BYTES",
|
||||
"OVERRUN_KIND",
|
||||
"OVERRUN_PRODUCER_ID",
|
||||
"FdrClient",
|
||||
"FdrRecord",
|
||||
"FdrSchemaError",
|
||||
"parse",
|
||||
"serialise",
|
||||
]
|
||||
|
||||
@@ -1,25 +1,226 @@
|
||||
"""FDR record schema — STUB.
|
||||
"""FDR record schema + versioned (de)serialiser (AZ-272 / E-CC-FDR-CLIENT).
|
||||
|
||||
Concrete schema (estimates / IMU / MAVLink / health / tile / thumbnail discriminated
|
||||
record types) is owned by AZ-272. Bootstrap declares the umbrella DTO so every
|
||||
producer can import it.
|
||||
Public surface frozen by
|
||||
`_docs/02_document/contracts/shared_fdr_client/fdr_record_schema.md` v1.0.0.
|
||||
|
||||
The library backing ``serialise`` / ``parse`` (``orjson``) is pinned in
|
||||
``pyproject.toml`` and intentionally hidden from the public API — callers
|
||||
trade ``FdrRecord <-> bytes`` only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from typing import Any, Final
|
||||
|
||||
import orjson
|
||||
|
||||
__all__ = [
|
||||
"CURRENT_SCHEMA_VERSION",
|
||||
"KNOWN_KINDS",
|
||||
"MAX_INLINE_BLOB_BYTES",
|
||||
"OVERRUN_KIND",
|
||||
"OVERRUN_PRODUCER_ID",
|
||||
"FdrRecord",
|
||||
"FdrSchemaError",
|
||||
"parse",
|
||||
"serialise",
|
||||
]
|
||||
|
||||
CURRENT_SCHEMA_VERSION: Final[int] = 1
|
||||
|
||||
OVERRUN_KIND: Final[str] = "overrun"
|
||||
OVERRUN_PRODUCER_ID: Final[str] = "shared.fdr_client"
|
||||
|
||||
# Per-kind allowed payload keys. The parser uses this to route unknown future
|
||||
# fields into ``payload["extra"]`` (forward-compat AC-2). Unknown ``kind`` values
|
||||
# bypass the table and are returned opaquely (AC-3).
|
||||
KNOWN_PAYLOAD_KEYS: Final[dict[str, frozenset[str]]] = {
|
||||
"log": frozenset({"level", "component", "frame_id", "kind", "msg", "kv", "exc"}),
|
||||
"vio.tick": frozenset(
|
||||
{"frame_id", "R", "t", "P", "last_anchor_age_ms", "mre_px", "imu_bias_norm"}
|
||||
),
|
||||
"state.tick": frozenset({"frame_id", "fused_pose", "covariance_2x2", "estimator_label"}),
|
||||
"tile_match": frozenset({"frame_id", "tile_id", "score", "match_count", "ransac_inliers"}),
|
||||
"overrun": frozenset({"producer_id", "dropped_count"}),
|
||||
"segment_rollover": frozenset({"old_segment", "new_segment", "total_bytes_after"}),
|
||||
"failed_tile_thumbnail": frozenset({"frame_id", "tile_id", "jpeg_bytes_b64"}),
|
||||
"mid_flight_tile_snapshot": frozenset({"snapshot_path", "captured_at"}),
|
||||
"flight_header": frozenset({"flight_id", "started_at", "schema_version", "build_info"}),
|
||||
"flight_footer": frozenset({"flight_id", "ended_at", "records_written", "records_dropped"}),
|
||||
}
|
||||
|
||||
KNOWN_KINDS: Final[frozenset[str]] = frozenset(KNOWN_PAYLOAD_KEYS.keys())
|
||||
|
||||
# Inline binary blob cap; bigger payloads must reference a sidecar path.
|
||||
MAX_INLINE_BLOB_BYTES: Final[int] = 4 * 1024
|
||||
|
||||
|
||||
class FdrSchemaError(ValueError):
|
||||
"""Raised on schema-violation in ``serialise`` / ``parse`` (AZ-272).
|
||||
|
||||
The ONLY exception type either function raises on schema-violation; orjson
|
||||
/ library-specific errors are wrapped at the public boundary.
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FdrRecord:
|
||||
"""A single FDR record (record-type discriminator + payload).
|
||||
"""Frozen FDR record envelope per contract v1.0.0."""
|
||||
|
||||
The full discriminated-union of record types is defined by AZ-272.
|
||||
"""
|
||||
|
||||
record_type: str
|
||||
timestamp: datetime
|
||||
producer: str
|
||||
schema_version: int
|
||||
ts: str
|
||||
producer_id: str
|
||||
kind: str
|
||||
payload: dict[str, Any] = field(default_factory=dict)
|
||||
extra: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
_ENVELOPE_REQUIRED: Final[tuple[str, ...]] = (
|
||||
"schema_version",
|
||||
"ts",
|
||||
"producer_id",
|
||||
"kind",
|
||||
"payload",
|
||||
)
|
||||
|
||||
|
||||
def _validate_envelope_outgoing(record: FdrRecord) -> None:
|
||||
if not isinstance(record.schema_version, int) or isinstance(record.schema_version, bool):
|
||||
raise FdrSchemaError(
|
||||
f"FdrRecord.schema_version must be a non-bool integer; got {record.schema_version!r}"
|
||||
)
|
||||
if record.schema_version < 1:
|
||||
raise FdrSchemaError(f"FdrRecord.schema_version must be >= 1; got {record.schema_version}")
|
||||
if not isinstance(record.ts, str) or not record.ts:
|
||||
raise FdrSchemaError(f"FdrRecord.ts must be a non-empty string; got {record.ts!r}")
|
||||
if not isinstance(record.producer_id, str) or not record.producer_id:
|
||||
raise FdrSchemaError(
|
||||
f"FdrRecord.producer_id must be a non-empty string; got {record.producer_id!r}"
|
||||
)
|
||||
if not isinstance(record.kind, str) or not record.kind:
|
||||
raise FdrSchemaError(f"FdrRecord.kind must be a non-empty string; got {record.kind!r}")
|
||||
if not isinstance(record.payload, dict):
|
||||
raise FdrSchemaError(
|
||||
f"FdrRecord.payload must be a dict; got {type(record.payload).__name__}"
|
||||
)
|
||||
if record.extra:
|
||||
raise FdrSchemaError(
|
||||
"FdrRecord.extra is populated only by the parser; producers must leave it empty"
|
||||
)
|
||||
_validate_payload_size(record.payload)
|
||||
if record.kind == OVERRUN_KIND:
|
||||
_validate_overrun_payload(record.payload)
|
||||
|
||||
|
||||
def _validate_payload_size(payload: dict[str, Any]) -> None:
|
||||
"""Reject any single binary blob >MAX_INLINE_BLOB_BYTES inside the payload."""
|
||||
for key, value in payload.items():
|
||||
if isinstance(value, (bytes, bytearray)) and len(value) > MAX_INLINE_BLOB_BYTES:
|
||||
raise FdrSchemaError(
|
||||
f"FdrRecord.payload[{key!r}] is {len(value)} bytes; "
|
||||
f"max inline blob is {MAX_INLINE_BLOB_BYTES} bytes — use a sidecar path"
|
||||
)
|
||||
|
||||
|
||||
def _validate_overrun_payload(payload: dict[str, Any]) -> None:
|
||||
inner_producer_id = payload.get("producer_id")
|
||||
if not isinstance(inner_producer_id, str) or not inner_producer_id:
|
||||
raise FdrSchemaError(
|
||||
"overrun record: payload.producer_id must be a non-empty string identifying the "
|
||||
"originating producer"
|
||||
)
|
||||
dropped_count = payload.get("dropped_count")
|
||||
if not isinstance(dropped_count, int) or isinstance(dropped_count, bool):
|
||||
raise FdrSchemaError(
|
||||
"overrun record: payload.dropped_count must be a non-bool integer; "
|
||||
f"got {dropped_count!r}"
|
||||
)
|
||||
if dropped_count <= 0:
|
||||
raise FdrSchemaError(
|
||||
f"overrun record: payload.dropped_count must be > 0; got {dropped_count}"
|
||||
)
|
||||
|
||||
|
||||
def serialise(record: FdrRecord) -> bytes:
|
||||
"""Encode ``record`` to wire bytes (single-line UTF-8 JSON, ``orjson``-backed)."""
|
||||
_validate_envelope_outgoing(record)
|
||||
envelope: dict[str, Any] = {
|
||||
"schema_version": record.schema_version,
|
||||
"ts": record.ts,
|
||||
"producer_id": record.producer_id,
|
||||
"kind": record.kind,
|
||||
"payload": record.payload,
|
||||
}
|
||||
try:
|
||||
return orjson.dumps(envelope)
|
||||
except (TypeError, orjson.JSONEncodeError) as exc:
|
||||
raise FdrSchemaError(f"failed to serialise FdrRecord: {exc}") from exc
|
||||
|
||||
|
||||
def parse(buf: bytes) -> FdrRecord:
|
||||
"""Decode wire bytes to an ``FdrRecord``; forward-compatible per contract AC-2/3."""
|
||||
if not isinstance(buf, (bytes, bytearray)):
|
||||
raise FdrSchemaError(f"parse expects bytes; got {type(buf).__name__}")
|
||||
try:
|
||||
decoded = orjson.loads(buf)
|
||||
except orjson.JSONDecodeError as exc:
|
||||
raise FdrSchemaError(f"failed to decode FdrRecord bytes: {exc}") from exc
|
||||
|
||||
if not isinstance(decoded, dict):
|
||||
raise FdrSchemaError(
|
||||
f"FdrRecord wire payload must decode to a dict; got {type(decoded).__name__}"
|
||||
)
|
||||
missing = [k for k in _ENVELOPE_REQUIRED if k not in decoded]
|
||||
if missing:
|
||||
raise FdrSchemaError(f"FdrRecord missing required field(s): {', '.join(missing)}")
|
||||
|
||||
schema_version = decoded.pop("schema_version")
|
||||
if not isinstance(schema_version, int) or isinstance(schema_version, bool):
|
||||
raise FdrSchemaError(
|
||||
f"FdrRecord.schema_version must be a non-bool integer; got {schema_version!r}"
|
||||
)
|
||||
if schema_version < 1:
|
||||
raise FdrSchemaError(f"FdrRecord.schema_version must be >= 1; got {schema_version}")
|
||||
|
||||
ts = decoded.pop("ts")
|
||||
if not isinstance(ts, str) or not ts:
|
||||
raise FdrSchemaError(f"FdrRecord.ts must be a non-empty string; got {ts!r}")
|
||||
producer_id = decoded.pop("producer_id")
|
||||
if not isinstance(producer_id, str) or not producer_id:
|
||||
raise FdrSchemaError(
|
||||
f"FdrRecord.producer_id must be a non-empty string; got {producer_id!r}"
|
||||
)
|
||||
kind = decoded.pop("kind")
|
||||
if not isinstance(kind, str) or not kind:
|
||||
raise FdrSchemaError(f"FdrRecord.kind must be a non-empty string; got {kind!r}")
|
||||
payload = decoded.pop("payload")
|
||||
if not isinstance(payload, dict):
|
||||
raise FdrSchemaError(f"FdrRecord.payload must be a dict; got {type(payload).__name__}")
|
||||
|
||||
# Anything left at the top level after popping required + payload is forward-compat extra.
|
||||
extra = dict(decoded)
|
||||
|
||||
# Forward-compat payload sweep: for known kinds, keys outside the v1.0.0
|
||||
# set are stashed in payload["extra"] so future v1.x producers can add new
|
||||
# fields without breaking v1.0 readers.
|
||||
known_keys = KNOWN_PAYLOAD_KEYS.get(kind)
|
||||
if known_keys is not None:
|
||||
unknown_keys = [k for k in payload.keys() if k not in known_keys and k != "extra"]
|
||||
if unknown_keys:
|
||||
extra_bucket = dict(payload.get("extra") or {})
|
||||
for key in unknown_keys:
|
||||
extra_bucket[key] = payload.pop(key)
|
||||
payload["extra"] = extra_bucket
|
||||
|
||||
if kind == OVERRUN_KIND:
|
||||
_validate_overrun_payload(payload)
|
||||
|
||||
return FdrRecord(
|
||||
schema_version=schema_version,
|
||||
ts=ts,
|
||||
producer_id=producer_id,
|
||||
kind=kind,
|
||||
payload=payload,
|
||||
extra=extra,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user