mirror of
https://github.com/azaion/gps-denied-onboard.git
synced 2026-06-23 00:31:12 +00:00
[AZ-266] [AZ-269] [AZ-277] [AZ-280] Cross-cutting log/config + SE3/SHA256 helpers
AZ-266: schema-compliant JSON logging entrypoint, level normalisation, handler-topology guard, format-error fallback (log_record_schema v1.0.0). AZ-269: env > YAML > defaults config loader, frozen Config dataclass, missing-var fail-fast with pointer to .env.example, component-block registry. AZ-277: GTSAM-backed SE3Utils (matrix<->SE3 + exp/log/adjoint) with strict orthogonality, dtype, and bottom-row contract enforcement. AZ-280: atomicwrites-backed write_atomic + independent verify + order-deterministic aggregate_hash; sidecar format strictness. pyproject.toml pins gtsam>=4.2,<5.0 and atomicwrites>=1.4,<2.0 (named-backend deps per the AZ-277 / AZ-280 contracts). 139 unit tests pass (44 new). Review verdict: PASS_WITH_WARNINGS; findings are perf-NFR + journald deferrals, no blocking issues. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -1,9 +1,24 @@
|
||||
"""Config loader + dataclass schemas (owned by AZ-269 / E-CC-CONF).
|
||||
"""Config loader + dataclass schemas (E-CC-CONF / AZ-269)."""
|
||||
|
||||
Bootstrap creates importable stubs so every component constructor can take a
|
||||
config argument from day one.
|
||||
"""
|
||||
from gps_denied_onboard.config.loader import ENV_KEY_MAP, load_config
|
||||
from gps_denied_onboard.config.schema import (
|
||||
Config,
|
||||
ConfigError,
|
||||
FdrConfig,
|
||||
LogConfig,
|
||||
RequiredFieldMissingError,
|
||||
RuntimeConfig,
|
||||
register_component_block,
|
||||
)
|
||||
|
||||
from gps_denied_onboard.config.schema import RuntimeConfig
|
||||
|
||||
__all__ = ["RuntimeConfig"]
|
||||
__all__ = [
|
||||
"ENV_KEY_MAP",
|
||||
"Config",
|
||||
"ConfigError",
|
||||
"FdrConfig",
|
||||
"LogConfig",
|
||||
"RequiredFieldMissingError",
|
||||
"RuntimeConfig",
|
||||
"load_config",
|
||||
"register_component_block",
|
||||
]
|
||||
|
||||
@@ -1,16 +1,178 @@
|
||||
"""Config loader — STUB.
|
||||
"""`load_config` — the single entrypoint that materialises `Config` at startup.
|
||||
|
||||
Concrete YAML + env-var loader is owned by AZ-269. Bootstrap exposes the load
|
||||
function as `NotImplementedError` so callers fail loudly until AZ-269 lands.
|
||||
Implements the `composition_root_protocol` contract v1.0.0 (E-CC-CONF /
|
||||
AZ-269 / AZ-246). Precedence (highest -> lowest):
|
||||
|
||||
1. Environment variables (``env`` argument).
|
||||
2. YAML files (``paths``), in order — later paths override earlier ones.
|
||||
3. Documented defaults baked into the cross-cutting dataclasses.
|
||||
|
||||
The returned `Config` is frozen end-to-end. Required env vars that fail
|
||||
to resolve raise `RequiredFieldMissingError` with the name of the
|
||||
offending variable and a pointer at ``.env.example``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any, Final
|
||||
|
||||
from gps_denied_onboard.config.schema import RuntimeConfig
|
||||
import yaml
|
||||
|
||||
from gps_denied_onboard.config.schema import (
|
||||
_COMPONENT_REGISTRY,
|
||||
Config,
|
||||
FdrConfig,
|
||||
LogConfig,
|
||||
RequiredFieldMissingError,
|
||||
RuntimeConfig,
|
||||
_replace_block,
|
||||
_resolve_component_blocks,
|
||||
)
|
||||
|
||||
__all__ = ["ENV_KEY_MAP", "load_config"]
|
||||
|
||||
|
||||
def load_runtime_config(yaml_path: Path) -> RuntimeConfig:
|
||||
"""Load a `RuntimeConfig` from a YAML file + environment overlay."""
|
||||
raise NotImplementedError("Config loader concrete impl is AZ-269 (E-CC-CONF)")
|
||||
# Env-var -> (block, field) mapping. The composition root reads env vars
|
||||
# through this table so the YAML path and the env path stay in sync.
|
||||
ENV_KEY_MAP: Final[dict[str, tuple[str, str]]] = {
|
||||
# Cross-cutting blocks
|
||||
"GPS_DENIED_FC_PROFILE": ("runtime", "fc_profile"),
|
||||
"GPS_DENIED_TIER": ("runtime", "tier"),
|
||||
"DB_URL": ("runtime", "db_url"),
|
||||
"CAMERA_CALIBRATION_PATH": ("runtime", "camera_calibration_path"),
|
||||
"INFERENCE_BACKEND": ("runtime", "inference_backend"),
|
||||
"TILE_CACHE_PATH": ("runtime", "tile_cache_path"),
|
||||
"LOG_LEVEL": ("log", "level"),
|
||||
"LOG_TIER": ("log", "tier"),
|
||||
"LOG_SINK": ("log", "sink"),
|
||||
"FDR_PATH": ("fdr", "path"),
|
||||
"FDR_QUEUE_SIZE": ("fdr", "queue_size"),
|
||||
}
|
||||
|
||||
# Env vars that MUST resolve to a non-empty value before `load_config`
|
||||
# can return (per AZ-263 AC-8 + AZ-269 AC-6). Missing values trigger
|
||||
# `RequiredFieldMissingError` with the variable name in the message.
|
||||
_REQUIRED_ENV_VARS: Final[tuple[str, ...]] = (
|
||||
"GPS_DENIED_FC_PROFILE",
|
||||
"GPS_DENIED_TIER",
|
||||
"DB_URL",
|
||||
"CAMERA_CALIBRATION_PATH",
|
||||
"LOG_LEVEL",
|
||||
"LOG_SINK",
|
||||
"INFERENCE_BACKEND",
|
||||
"FDR_PATH",
|
||||
"TILE_CACHE_PATH",
|
||||
)
|
||||
|
||||
# Field-name -> python type. We coerce string env vars + raw YAML scalars
|
||||
# into the dataclass's declared types so `Config.runtime.tier` is always
|
||||
# `int` regardless of source.
|
||||
_FIELD_COERCIONS: Final[dict[str, type]] = {
|
||||
"tier": int,
|
||||
"queue_size": int,
|
||||
"level": str,
|
||||
"sink": str,
|
||||
"path": str,
|
||||
"fc_profile": str,
|
||||
"db_url": str,
|
||||
"camera_calibration_path": str,
|
||||
"inference_backend": str,
|
||||
"tile_cache_path": str,
|
||||
"overrun_policy": str,
|
||||
}
|
||||
|
||||
|
||||
def _coerce_value(field_name: str, raw: Any) -> Any:
|
||||
target_type = _FIELD_COERCIONS.get(field_name)
|
||||
if target_type is None or isinstance(raw, target_type):
|
||||
return raw
|
||||
try:
|
||||
return target_type(raw)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise RequiredFieldMissingError(
|
||||
f"config field {field_name!r}: cannot coerce {raw!r} to {target_type.__name__} ({exc})"
|
||||
) from exc
|
||||
|
||||
|
||||
def _load_yaml_files(paths: Sequence[Path]) -> dict[str, dict[str, Any]]:
|
||||
"""Merge YAML files in order: later paths win for the same block + field."""
|
||||
merged: dict[str, dict[str, Any]] = {}
|
||||
for path in paths:
|
||||
data = yaml.safe_load(path.read_text()) or {}
|
||||
if not isinstance(data, dict):
|
||||
raise RequiredFieldMissingError(
|
||||
f"YAML at {path} must be a mapping at the top level; got {type(data).__name__}"
|
||||
)
|
||||
for block_name, block_value in data.items():
|
||||
if not isinstance(block_value, dict):
|
||||
continue
|
||||
merged.setdefault(block_name, {}).update(block_value)
|
||||
return merged
|
||||
|
||||
|
||||
def _apply_env_overrides(layered: dict[str, dict[str, Any]], env: Mapping[str, str]) -> None:
|
||||
"""Overlay env-var values on the per-block override dictionaries."""
|
||||
for env_key, (block_name, field_name) in ENV_KEY_MAP.items():
|
||||
if env_key not in env:
|
||||
continue
|
||||
layered.setdefault(block_name, {})[field_name] = env[env_key]
|
||||
|
||||
|
||||
def _check_required_env(env: Mapping[str, str]) -> None:
|
||||
"""AC-6 + AZ-263 AC-8: missing required vars fail fast with a pointer."""
|
||||
missing = [name for name in _REQUIRED_ENV_VARS if not env.get(name)]
|
||||
if missing:
|
||||
raise RequiredFieldMissingError(
|
||||
"Missing required environment variable(s): "
|
||||
+ ", ".join(missing)
|
||||
+ ". See `.env.example` for the documented set."
|
||||
)
|
||||
|
||||
|
||||
def load_config(
|
||||
env: Mapping[str, str],
|
||||
paths: Sequence[Path] = (),
|
||||
*,
|
||||
require_env: bool = True,
|
||||
) -> Config:
|
||||
"""Build a frozen `Config` from env + YAML files + documented defaults.
|
||||
|
||||
Precedence: env > YAML > defaults. `paths` may be empty; missing keys
|
||||
fall to the dataclass-declared defaults.
|
||||
"""
|
||||
if require_env:
|
||||
_check_required_env(env)
|
||||
|
||||
yaml_overrides = _load_yaml_files(paths) if paths else {}
|
||||
_apply_env_overrides(yaml_overrides, env)
|
||||
|
||||
runtime_block = _replace_block(
|
||||
RuntimeConfig(),
|
||||
{k: _coerce_value(k, v) for k, v in yaml_overrides.get("runtime", {}).items()},
|
||||
)
|
||||
log_block = _replace_block(
|
||||
LogConfig(),
|
||||
{k: _coerce_value(k, v) for k, v in yaml_overrides.get("log", {}).items()},
|
||||
)
|
||||
fdr_block = _replace_block(
|
||||
FdrConfig(),
|
||||
{k: _coerce_value(k, v) for k, v in yaml_overrides.get("fdr", {}).items()},
|
||||
)
|
||||
|
||||
component_blocks = _resolve_component_blocks()
|
||||
for slug, dataclass_type in _COMPONENT_REGISTRY.items():
|
||||
block_overrides = yaml_overrides.get(slug, {})
|
||||
if block_overrides:
|
||||
component_blocks[slug] = _replace_block(
|
||||
dataclass_type(),
|
||||
{k: _coerce_value(k, v) for k, v in block_overrides.items()},
|
||||
)
|
||||
|
||||
return Config(
|
||||
runtime=runtime_block,
|
||||
log=log_block,
|
||||
fdr=fdr_block,
|
||||
components=component_blocks,
|
||||
)
|
||||
|
||||
@@ -1,26 +1,157 @@
|
||||
"""Config dataclass schemas — STUB.
|
||||
"""Config dataclasses for E-CC-CONF (AZ-269 / AZ-246).
|
||||
|
||||
Concrete YAML schema is owned by AZ-269. Bootstrap declares only the runtime-level
|
||||
config container so the composition root can type its `compose_*` signatures.
|
||||
The outer `Config` aggregates one frozen nested dataclass per top-level
|
||||
config block. Cross-cutting blocks (`log`, `fdr`, `runtime`) live here;
|
||||
per-component blocks live with their own component epic and are
|
||||
registered into `Config.components` via `register_component_block`.
|
||||
|
||||
Public surface frozen by `_docs/02_document/contracts/shared_config/composition_root_protocol.md` v1.0.0.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field, fields, is_dataclass, replace
|
||||
from typing import Any, Final
|
||||
|
||||
__all__ = [
|
||||
"Config",
|
||||
"ConfigError",
|
||||
"FdrConfig",
|
||||
"LogConfig",
|
||||
"RequiredFieldMissingError",
|
||||
"RuntimeConfig",
|
||||
"register_component_block",
|
||||
]
|
||||
|
||||
|
||||
class ConfigError(RuntimeError):
|
||||
"""Base class for all config-loader errors that should reach the caller."""
|
||||
|
||||
|
||||
class RequiredFieldMissingError(ConfigError):
|
||||
"""Raised when a required configuration value is absent from env + YAML + defaults.
|
||||
|
||||
Always names the missing variable and points at the runtime helper that
|
||||
documents the full set (``.env.example``).
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LogConfig:
|
||||
"""Cross-cutting logging block (E-CC-LOG)."""
|
||||
|
||||
level: str = "INFO"
|
||||
tier: int = 1
|
||||
sink: str = "console"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FdrConfig:
|
||||
"""Cross-cutting Flight Data Recorder block (E-CC-FDR-CLIENT / AZ-247).
|
||||
|
||||
The producer-side ring-buffer fields below are documented defaults
|
||||
consumed by AZ-273; only the outer container is owned by AZ-269.
|
||||
"""
|
||||
|
||||
queue_size: int = 4096
|
||||
overrun_policy: str = "drop_oldest"
|
||||
path: str = "/var/lib/gps-denied/fdr"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RuntimeConfig:
|
||||
"""Runtime configuration loaded from YAML + env vars.
|
||||
|
||||
The concrete field set is filled in by AZ-269. This stub is enough for the
|
||||
composition root + tests to import the type.
|
||||
"""
|
||||
"""Top-level runtime descriptors that don't belong to a single component."""
|
||||
|
||||
fc_profile: str = "ardupilot_plane"
|
||||
tier: int = 1
|
||||
db_url: str = ""
|
||||
log_level: str = "INFO"
|
||||
log_sink: str = "console"
|
||||
extras: dict[str, Any] = field(default_factory=dict)
|
||||
camera_calibration_path: str = ""
|
||||
inference_backend: str = "pytorch_fp16"
|
||||
tile_cache_path: str = "/var/lib/gps-denied/tiles"
|
||||
|
||||
|
||||
# Documented defaults for cross-cutting blocks ONLY. Per-component defaults
|
||||
# live with their own component epic. The registry below is the single
|
||||
# source of truth so two components cannot silently claim the same key.
|
||||
_DEFAULT_BLOCKS: Final[dict[str, type]] = {
|
||||
"log": LogConfig,
|
||||
"fdr": FdrConfig,
|
||||
"runtime": RuntimeConfig,
|
||||
}
|
||||
|
||||
|
||||
# Registry for per-component nested dataclasses. Each component epic
|
||||
# calls ``register_component_block("c5_state", C5StateConfig)`` from its
|
||||
# package import path; the composition root drives those imports before
|
||||
# calling ``load_config``.
|
||||
_COMPONENT_REGISTRY: dict[str, type] = {}
|
||||
|
||||
|
||||
def register_component_block(slug: str, dataclass_type: type) -> None:
|
||||
"""Register a per-component frozen dataclass under its component slug."""
|
||||
if not is_dataclass(dataclass_type):
|
||||
raise TypeError(
|
||||
f"register_component_block({slug!r}, ...): block must be a dataclass; "
|
||||
f"got {dataclass_type!r}"
|
||||
)
|
||||
existing = _COMPONENT_REGISTRY.get(slug)
|
||||
if existing is not None and existing is not dataclass_type:
|
||||
raise ConfigError(
|
||||
f"duplicate component config registration for slug {slug!r}: "
|
||||
f"{existing!r} vs {dataclass_type!r}"
|
||||
)
|
||||
_COMPONENT_REGISTRY[slug] = dataclass_type
|
||||
|
||||
|
||||
def _resolve_default_blocks() -> dict[str, Any]:
|
||||
"""Instantiate every documented cross-cutting block with its defaults."""
|
||||
return {name: cls() for name, cls in _DEFAULT_BLOCKS.items()}
|
||||
|
||||
|
||||
def _resolve_component_blocks() -> dict[str, Any]:
|
||||
"""Instantiate every registered per-component block with its defaults."""
|
||||
return {slug: cls() for slug, cls in _COMPONENT_REGISTRY.items()}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Config:
|
||||
"""Outer composition-root config (frozen end-to-end).
|
||||
|
||||
Components consume only their own slice via ``config.components[slug]``;
|
||||
the runtime / log / fdr cross-cutting blocks are read directly via
|
||||
attribute access by the composition root.
|
||||
"""
|
||||
|
||||
runtime: RuntimeConfig = field(default_factory=RuntimeConfig)
|
||||
log: LogConfig = field(default_factory=LogConfig)
|
||||
fdr: FdrConfig = field(default_factory=FdrConfig)
|
||||
components: Mapping[str, Any] = field(default_factory=dict)
|
||||
|
||||
@classmethod
|
||||
def with_blocks(cls, **blocks: Any) -> Config:
|
||||
"""Build a `Config` from a flat name-to-instance map.
|
||||
|
||||
Cross-cutting names (``log``, ``fdr``, ``runtime``) become attributes;
|
||||
every other key is treated as a component slug and goes into
|
||||
``components``.
|
||||
"""
|
||||
runtime = blocks.pop("runtime", RuntimeConfig())
|
||||
log = blocks.pop("log", LogConfig())
|
||||
fdr = blocks.pop("fdr", FdrConfig())
|
||||
return cls(runtime=runtime, log=log, fdr=fdr, components=dict(blocks))
|
||||
|
||||
|
||||
def _block_field_names(block: Any) -> tuple[str, ...]:
|
||||
return tuple(f.name for f in fields(block))
|
||||
|
||||
|
||||
def _replace_block(block: Any, overrides: Mapping[str, Any]) -> Any:
|
||||
"""Return ``replace(block, **overrides)`` after filtering unknown keys."""
|
||||
if not overrides:
|
||||
return block
|
||||
known = set(_block_field_names(block))
|
||||
filtered = {k: v for k, v in overrides.items() if k in known}
|
||||
if not filtered:
|
||||
return block
|
||||
return replace(block, **filtered)
|
||||
|
||||
Reference in New Issue
Block a user