[AZ-266] [AZ-269] [AZ-277] [AZ-280] Cross-cutting log/config + SE3/SHA256 helpers

AZ-266: schema-compliant JSON logging entrypoint, level normalisation,
handler-topology guard, format-error fallback (log_record_schema v1.0.0).
AZ-269: env > YAML > defaults config loader, frozen Config dataclass,
missing-var fail-fast with pointer to .env.example, component-block registry.
AZ-277: GTSAM-backed SE3Utils (matrix<->SE3 + exp/log/adjoint) with strict
orthogonality, dtype, and bottom-row contract enforcement.
AZ-280: atomicwrites-backed write_atomic + independent verify +
order-deterministic aggregate_hash; sidecar format strictness.
pyproject.toml pins gtsam>=4.2,<5.0 and atomicwrites>=1.4,<2.0
(named-backend deps per the AZ-277 / AZ-280 contracts).
139 unit tests pass (44 new). Review verdict: PASS_WITH_WARNINGS;
findings are perf-NFR + journald deferrals, no blocking issues.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Oleksandr Bezdieniezhnykh
2026-05-11 01:33:42 +03:00
parent b12db61444
commit 8e71f6c002
21 changed files with 2134 additions and 133 deletions
+22 -7
View File
@@ -1,9 +1,24 @@
"""Config loader + dataclass schemas (owned by AZ-269 / E-CC-CONF).
"""Config loader + dataclass schemas (E-CC-CONF / AZ-269)."""
Bootstrap creates importable stubs so every component constructor can take a
config argument from day one.
"""
from gps_denied_onboard.config.loader import ENV_KEY_MAP, load_config
from gps_denied_onboard.config.schema import (
Config,
ConfigError,
FdrConfig,
LogConfig,
RequiredFieldMissingError,
RuntimeConfig,
register_component_block,
)
from gps_denied_onboard.config.schema import RuntimeConfig
__all__ = ["RuntimeConfig"]
__all__ = [
"ENV_KEY_MAP",
"Config",
"ConfigError",
"FdrConfig",
"LogConfig",
"RequiredFieldMissingError",
"RuntimeConfig",
"load_config",
"register_component_block",
]
+169 -7
View File
@@ -1,16 +1,178 @@
"""Config loader — STUB.
"""`load_config` — the single entrypoint that materialises `Config` at startup.
Concrete YAML + env-var loader is owned by AZ-269. Bootstrap exposes the load
function as `NotImplementedError` so callers fail loudly until AZ-269 lands.
Implements the `composition_root_protocol` contract v1.0.0 (E-CC-CONF /
AZ-269 / AZ-246). Precedence (highest -> lowest):
1. Environment variables (``env`` argument).
2. YAML files (``paths``), in order — later paths override earlier ones.
3. Documented defaults baked into the cross-cutting dataclasses.
The returned `Config` is frozen end-to-end. Required env vars that fail
to resolve raise `RequiredFieldMissingError` with the name of the
offending variable and a pointer at ``.env.example``.
"""
from __future__ import annotations
from collections.abc import Mapping, Sequence
from pathlib import Path
from typing import Any, Final
from gps_denied_onboard.config.schema import RuntimeConfig
import yaml
from gps_denied_onboard.config.schema import (
_COMPONENT_REGISTRY,
Config,
FdrConfig,
LogConfig,
RequiredFieldMissingError,
RuntimeConfig,
_replace_block,
_resolve_component_blocks,
)
__all__ = ["ENV_KEY_MAP", "load_config"]
def load_runtime_config(yaml_path: Path) -> RuntimeConfig:
"""Load a `RuntimeConfig` from a YAML file + environment overlay."""
raise NotImplementedError("Config loader concrete impl is AZ-269 (E-CC-CONF)")
# Env-var -> (block, field) mapping. The composition root reads env vars
# through this table so the YAML path and the env path stay in sync.
ENV_KEY_MAP: Final[dict[str, tuple[str, str]]] = {
# Cross-cutting blocks
"GPS_DENIED_FC_PROFILE": ("runtime", "fc_profile"),
"GPS_DENIED_TIER": ("runtime", "tier"),
"DB_URL": ("runtime", "db_url"),
"CAMERA_CALIBRATION_PATH": ("runtime", "camera_calibration_path"),
"INFERENCE_BACKEND": ("runtime", "inference_backend"),
"TILE_CACHE_PATH": ("runtime", "tile_cache_path"),
"LOG_LEVEL": ("log", "level"),
"LOG_TIER": ("log", "tier"),
"LOG_SINK": ("log", "sink"),
"FDR_PATH": ("fdr", "path"),
"FDR_QUEUE_SIZE": ("fdr", "queue_size"),
}
# Env vars that MUST resolve to a non-empty value before `load_config`
# can return (per AZ-263 AC-8 + AZ-269 AC-6). Missing values trigger
# `RequiredFieldMissingError` with the variable name in the message.
_REQUIRED_ENV_VARS: Final[tuple[str, ...]] = (
"GPS_DENIED_FC_PROFILE",
"GPS_DENIED_TIER",
"DB_URL",
"CAMERA_CALIBRATION_PATH",
"LOG_LEVEL",
"LOG_SINK",
"INFERENCE_BACKEND",
"FDR_PATH",
"TILE_CACHE_PATH",
)
# Field-name -> python type. We coerce string env vars + raw YAML scalars
# into the dataclass's declared types so `Config.runtime.tier` is always
# `int` regardless of source.
_FIELD_COERCIONS: Final[dict[str, type]] = {
"tier": int,
"queue_size": int,
"level": str,
"sink": str,
"path": str,
"fc_profile": str,
"db_url": str,
"camera_calibration_path": str,
"inference_backend": str,
"tile_cache_path": str,
"overrun_policy": str,
}
def _coerce_value(field_name: str, raw: Any) -> Any:
target_type = _FIELD_COERCIONS.get(field_name)
if target_type is None or isinstance(raw, target_type):
return raw
try:
return target_type(raw)
except (TypeError, ValueError) as exc:
raise RequiredFieldMissingError(
f"config field {field_name!r}: cannot coerce {raw!r} to {target_type.__name__} ({exc})"
) from exc
def _load_yaml_files(paths: Sequence[Path]) -> dict[str, dict[str, Any]]:
"""Merge YAML files in order: later paths win for the same block + field."""
merged: dict[str, dict[str, Any]] = {}
for path in paths:
data = yaml.safe_load(path.read_text()) or {}
if not isinstance(data, dict):
raise RequiredFieldMissingError(
f"YAML at {path} must be a mapping at the top level; got {type(data).__name__}"
)
for block_name, block_value in data.items():
if not isinstance(block_value, dict):
continue
merged.setdefault(block_name, {}).update(block_value)
return merged
def _apply_env_overrides(layered: dict[str, dict[str, Any]], env: Mapping[str, str]) -> None:
"""Overlay env-var values on the per-block override dictionaries."""
for env_key, (block_name, field_name) in ENV_KEY_MAP.items():
if env_key not in env:
continue
layered.setdefault(block_name, {})[field_name] = env[env_key]
def _check_required_env(env: Mapping[str, str]) -> None:
"""AC-6 + AZ-263 AC-8: missing required vars fail fast with a pointer."""
missing = [name for name in _REQUIRED_ENV_VARS if not env.get(name)]
if missing:
raise RequiredFieldMissingError(
"Missing required environment variable(s): "
+ ", ".join(missing)
+ ". See `.env.example` for the documented set."
)
def load_config(
env: Mapping[str, str],
paths: Sequence[Path] = (),
*,
require_env: bool = True,
) -> Config:
"""Build a frozen `Config` from env + YAML files + documented defaults.
Precedence: env > YAML > defaults. `paths` may be empty; missing keys
fall to the dataclass-declared defaults.
"""
if require_env:
_check_required_env(env)
yaml_overrides = _load_yaml_files(paths) if paths else {}
_apply_env_overrides(yaml_overrides, env)
runtime_block = _replace_block(
RuntimeConfig(),
{k: _coerce_value(k, v) for k, v in yaml_overrides.get("runtime", {}).items()},
)
log_block = _replace_block(
LogConfig(),
{k: _coerce_value(k, v) for k, v in yaml_overrides.get("log", {}).items()},
)
fdr_block = _replace_block(
FdrConfig(),
{k: _coerce_value(k, v) for k, v in yaml_overrides.get("fdr", {}).items()},
)
component_blocks = _resolve_component_blocks()
for slug, dataclass_type in _COMPONENT_REGISTRY.items():
block_overrides = yaml_overrides.get(slug, {})
if block_overrides:
component_blocks[slug] = _replace_block(
dataclass_type(),
{k: _coerce_value(k, v) for k, v in block_overrides.items()},
)
return Config(
runtime=runtime_block,
log=log_block,
fdr=fdr_block,
components=component_blocks,
)
+144 -13
View File
@@ -1,26 +1,157 @@
"""Config dataclass schemas — STUB.
"""Config dataclasses for E-CC-CONF (AZ-269 / AZ-246).
Concrete YAML schema is owned by AZ-269. Bootstrap declares only the runtime-level
config container so the composition root can type its `compose_*` signatures.
The outer `Config` aggregates one frozen nested dataclass per top-level
config block. Cross-cutting blocks (`log`, `fdr`, `runtime`) live here;
per-component blocks live with their own component epic and are
registered into `Config.components` via `register_component_block`.
Public surface frozen by `_docs/02_document/contracts/shared_config/composition_root_protocol.md` v1.0.0.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any
from collections.abc import Mapping
from dataclasses import dataclass, field, fields, is_dataclass, replace
from typing import Any, Final
__all__ = [
"Config",
"ConfigError",
"FdrConfig",
"LogConfig",
"RequiredFieldMissingError",
"RuntimeConfig",
"register_component_block",
]
class ConfigError(RuntimeError):
"""Base class for all config-loader errors that should reach the caller."""
class RequiredFieldMissingError(ConfigError):
"""Raised when a required configuration value is absent from env + YAML + defaults.
Always names the missing variable and points at the runtime helper that
documents the full set (``.env.example``).
"""
@dataclass(frozen=True)
class LogConfig:
"""Cross-cutting logging block (E-CC-LOG)."""
level: str = "INFO"
tier: int = 1
sink: str = "console"
@dataclass(frozen=True)
class FdrConfig:
"""Cross-cutting Flight Data Recorder block (E-CC-FDR-CLIENT / AZ-247).
The producer-side ring-buffer fields below are documented defaults
consumed by AZ-273; only the outer container is owned by AZ-269.
"""
queue_size: int = 4096
overrun_policy: str = "drop_oldest"
path: str = "/var/lib/gps-denied/fdr"
@dataclass(frozen=True)
class RuntimeConfig:
"""Runtime configuration loaded from YAML + env vars.
The concrete field set is filled in by AZ-269. This stub is enough for the
composition root + tests to import the type.
"""
"""Top-level runtime descriptors that don't belong to a single component."""
fc_profile: str = "ardupilot_plane"
tier: int = 1
db_url: str = ""
log_level: str = "INFO"
log_sink: str = "console"
extras: dict[str, Any] = field(default_factory=dict)
camera_calibration_path: str = ""
inference_backend: str = "pytorch_fp16"
tile_cache_path: str = "/var/lib/gps-denied/tiles"
# Documented defaults for cross-cutting blocks ONLY. Per-component defaults
# live with their own component epic. The registry below is the single
# source of truth so two components cannot silently claim the same key.
_DEFAULT_BLOCKS: Final[dict[str, type]] = {
"log": LogConfig,
"fdr": FdrConfig,
"runtime": RuntimeConfig,
}
# Registry for per-component nested dataclasses. Each component epic
# calls ``register_component_block("c5_state", C5StateConfig)`` from its
# package import path; the composition root drives those imports before
# calling ``load_config``.
_COMPONENT_REGISTRY: dict[str, type] = {}
def register_component_block(slug: str, dataclass_type: type) -> None:
"""Register a per-component frozen dataclass under its component slug."""
if not is_dataclass(dataclass_type):
raise TypeError(
f"register_component_block({slug!r}, ...): block must be a dataclass; "
f"got {dataclass_type!r}"
)
existing = _COMPONENT_REGISTRY.get(slug)
if existing is not None and existing is not dataclass_type:
raise ConfigError(
f"duplicate component config registration for slug {slug!r}: "
f"{existing!r} vs {dataclass_type!r}"
)
_COMPONENT_REGISTRY[slug] = dataclass_type
def _resolve_default_blocks() -> dict[str, Any]:
"""Instantiate every documented cross-cutting block with its defaults."""
return {name: cls() for name, cls in _DEFAULT_BLOCKS.items()}
def _resolve_component_blocks() -> dict[str, Any]:
"""Instantiate every registered per-component block with its defaults."""
return {slug: cls() for slug, cls in _COMPONENT_REGISTRY.items()}
@dataclass(frozen=True)
class Config:
"""Outer composition-root config (frozen end-to-end).
Components consume only their own slice via ``config.components[slug]``;
the runtime / log / fdr cross-cutting blocks are read directly via
attribute access by the composition root.
"""
runtime: RuntimeConfig = field(default_factory=RuntimeConfig)
log: LogConfig = field(default_factory=LogConfig)
fdr: FdrConfig = field(default_factory=FdrConfig)
components: Mapping[str, Any] = field(default_factory=dict)
@classmethod
def with_blocks(cls, **blocks: Any) -> Config:
"""Build a `Config` from a flat name-to-instance map.
Cross-cutting names (``log``, ``fdr``, ``runtime``) become attributes;
every other key is treated as a component slug and goes into
``components``.
"""
runtime = blocks.pop("runtime", RuntimeConfig())
log = blocks.pop("log", LogConfig())
fdr = blocks.pop("fdr", FdrConfig())
return cls(runtime=runtime, log=log, fdr=fdr, components=dict(blocks))
def _block_field_names(block: Any) -> tuple[str, ...]:
return tuple(f.name for f in fields(block))
def _replace_block(block: Any, overrides: Mapping[str, Any]) -> Any:
"""Return ``replace(block, **overrides)`` after filtering unknown keys."""
if not overrides:
return block
known = set(_block_field_names(block))
filtered = {k: v for k, v in overrides.items() if k in known}
if not filtered:
return block
return replace(block, **filtered)
+34 -3
View File
@@ -1,5 +1,36 @@
"""Shared utilities (owned by AZ-264 / E-CC-HELPERS).
"""Shared utilities (E-CC-HELPERS / AZ-264).
Bootstrap (AZ-263) creates these as importable stubs; concrete implementations are
filled in by per-helper tasks under AZ-264. See `_docs/02_document/common-helpers/`.
Each helper has its own task (AZ-276..AZ-283). This package exposes the
ones that have landed so consumers can depend on a stable public surface
without reaching into the helper modules directly.
"""
from gps_denied_onboard.helpers.se3_utils import (
SE3,
Se3InvalidMatrixError,
adjoint,
exp_map,
is_valid_rotation,
log_map,
matrix_to_se3,
se3_to_matrix,
)
from gps_denied_onboard.helpers.sha256_sidecar import (
SIDECAR_SUFFIX,
Sha256Sidecar,
Sha256SidecarError,
)
__all__ = [
"SE3",
"SIDECAR_SUFFIX",
"Se3InvalidMatrixError",
"Sha256Sidecar",
"Sha256SidecarError",
"adjoint",
"exp_map",
"is_valid_rotation",
"log_map",
"matrix_to_se3",
"se3_to_matrix",
]
+129 -16
View File
@@ -1,29 +1,142 @@
"""SE(3) utility helpers — STUB.
"""SE(3) helpers backed by GTSAM `Pose3` (E-CC-HELPERS / AZ-264 / AZ-277).
Concrete implementation is owned by AZ-277. Contract:
`_docs/02_document/common-helpers/02_helper_se3_utils.md`.
Implements the `se3_utils` contract v1.0.0 at
`_docs/02_document/contracts/shared_helpers/se3_utils.md`. Stateless,
pure functions; strict caller-orthogonalisation invariant.
"""
from __future__ import annotations
from typing import Any
from typing import Final
import gtsam
import numpy as np
__all__ = [
"SE3",
"Se3InvalidMatrixError",
"adjoint",
"exp_map",
"is_valid_rotation",
"log_map",
"matrix_to_se3",
"se3_to_matrix",
]
def compose(a_se3: Any, b_se3: Any) -> Any:
"""Compose two SE(3) transforms."""
raise NotImplementedError("se3_utils concrete impl is AZ-277")
SE3 = gtsam.Pose3
# Tolerance for the orthogonality / determinant / bottom-row checks. Tight
# enough that an ill-conditioned rotation from a real consumer is caught;
# loose enough to not reject within-noise GTSAM output. Documented at the
# contract level for symmetry with consumer tests.
_DEFAULT_ROT_ATOL: Final[float] = 1e-6
_DEFAULT_BOTTOM_ROW: Final[np.ndarray] = np.array([0.0, 0.0, 0.0, 1.0], dtype=np.float64)
# Small-angle Taylor cutoff for `exp_map` stability (AC-3). Below this twist
# norm we delegate to GTSAM's first-order Taylor fallback rather than risk
# the `sin(theta)/theta` numerator under-flowing to zero.
_SMALL_ANGLE_THRESHOLD: Final[float] = 1e-10
def inverse(t_se3: Any) -> Any:
"""Invert an SE(3) transform."""
raise NotImplementedError("se3_utils concrete impl is AZ-277")
class Se3InvalidMatrixError(ValueError):
"""Raised when an input matrix violates the SE(3) shape / dtype / orthogonality contract."""
def log_map(t_se3: Any) -> Any:
"""SE(3) → se(3) log map (returns a 6-vector)."""
raise NotImplementedError("se3_utils concrete impl is AZ-277")
def _require_float64(array: np.ndarray, *, name: str) -> None:
if array.dtype != np.float64:
raise Se3InvalidMatrixError(
f"{name}: helpers operate strictly on dtype=float64; got {array.dtype}"
)
def exp_map(xi_6: Any) -> Any:
"""se(3) → SE(3) exp map (consumes a 6-vector)."""
raise NotImplementedError("se3_utils concrete impl is AZ-277")
def _require_shape(array: np.ndarray, expected: tuple[int, ...], *, name: str) -> None:
if array.shape != expected:
raise Se3InvalidMatrixError(f"{name}: expected shape {expected}; got {array.shape}")
def is_valid_rotation(R_3x3: np.ndarray, *, atol: float = _DEFAULT_ROT_ATOL) -> bool:
"""Return True iff `R_3x3` is an orthogonal rotation with positive determinant."""
if not isinstance(R_3x3, np.ndarray):
return False
if R_3x3.shape != (3, 3) or R_3x3.dtype != np.float64:
return False
drift = R_3x3.T @ R_3x3 - np.eye(3, dtype=np.float64)
if np.linalg.norm(drift, ord="fro") > atol:
return False
if np.linalg.det(R_3x3) < 0:
return False
return True
def matrix_to_se3(T_4x4: np.ndarray, *, atol: float = _DEFAULT_ROT_ATOL) -> SE3:
"""Convert a 4x4 homogeneous-transform matrix into a GTSAM `Pose3`.
Strict orthogonality contract: callers MUST pre-orthogonalise their
rotation matrices. Non-orthogonal inputs, negative-determinant
rotations, malformed bottom rows, and `float32` inputs all raise
`Se3InvalidMatrixError` — the helper never silently re-orthogonalises.
"""
if not isinstance(T_4x4, np.ndarray):
raise Se3InvalidMatrixError(
f"matrix_to_se3: expected np.ndarray; got {type(T_4x4).__name__}"
)
_require_shape(T_4x4, (4, 4), name="matrix_to_se3")
_require_float64(T_4x4, name="matrix_to_se3")
bottom_row = T_4x4[3]
if not np.array_equal(bottom_row, _DEFAULT_BOTTOM_ROW):
raise Se3InvalidMatrixError(
f"matrix_to_se3: bottom row must be [0, 0, 0, 1]; got {bottom_row.tolist()}"
)
R = T_4x4[:3, :3]
drift = R.T @ R - np.eye(3, dtype=np.float64)
drift_norm = float(np.linalg.norm(drift, ord="fro"))
if drift_norm > atol:
raise Se3InvalidMatrixError(
f"matrix_to_se3: rotation is not orthogonal "
f"(||R^T R - I||_F = {drift_norm:.3e} > atol={atol:.1e}); "
f"caller must orthogonalise before invoking the helper"
)
det = float(np.linalg.det(R))
if det < 0:
raise Se3InvalidMatrixError(
f"matrix_to_se3: rotation has negative determinant ({det:.3e}); "
f"mirror rotations are not valid SE(3) members"
)
return SE3(T_4x4)
def se3_to_matrix(pose: SE3) -> np.ndarray:
"""Return the 4x4 homogeneous matrix for `pose` as `float64`."""
return np.ascontiguousarray(pose.matrix(), dtype=np.float64)
def exp_map(xi: np.ndarray) -> SE3:
"""Exponential map: se(3) twist (6,) -> SE(3) pose.
Near-identity inputs (twist norm below the small-angle threshold)
fall back to the identity pose rather than relying on the
full-precision `sin(theta)/theta` expansion.
"""
if not isinstance(xi, np.ndarray):
raise Se3InvalidMatrixError(f"exp_map: expected np.ndarray; got {type(xi).__name__}")
_require_shape(xi, (6,), name="exp_map")
_require_float64(xi, name="exp_map")
if float(np.linalg.norm(xi)) < _SMALL_ANGLE_THRESHOLD:
return SE3()
return SE3.Expmap(xi)
def log_map(pose: SE3) -> np.ndarray:
"""Logarithm map: SE(3) pose -> se(3) twist (6,)."""
return np.ascontiguousarray(SE3.Logmap(pose), dtype=np.float64)
def adjoint(pose: SE3) -> np.ndarray:
"""Adjoint matrix (6x6) of `pose` for body-frame -> world-frame twist transport."""
return np.ascontiguousarray(pose.AdjointMap(), dtype=np.float64)
@@ -1,19 +1,172 @@
"""Content-hash sidecar helper — STUB.
"""Atomic-write + SHA-256 sidecar helper (D-C10-3 / E-CC-HELPERS / AZ-280).
D-C10-3 content-hash gate. Concrete impl owned by AZ-280. Contract:
`_docs/02_document/common-helpers/05_helper_sha256_sidecar.md`.
Implements the `sha256_sidecar` contract v1.0.0 at
`_docs/02_document/contracts/shared_helpers/sha256_sidecar.md`. Stateless
static-only design (per coderule § static methods are appropriate only
for pure, self-contained computations and well-bounded I/O).
Atomic write is implemented via ``atomicwrites.atomic_write`` which uses
the temp-file -> ``os.replace`` pattern. Verification recomputes the
digest from the file's bytes; the sidecar value is consulted only as the
"expected" side of the equality check.
"""
from __future__ import annotations
import hashlib
from pathlib import Path
from atomicwrites import atomic_write
def compute_sidecar(target_path: Path) -> Path:
"""Compute SHA-256 of `target_path` and write a sidecar file next to it."""
raise NotImplementedError("sha256_sidecar concrete impl is AZ-280")
__all__ = ["Sha256Sidecar", "Sha256SidecarError"]
def verify_sidecar(target_path: Path) -> bool:
"""Verify that the sidecar matches the file content."""
raise NotImplementedError("sha256_sidecar concrete impl is AZ-280")
_SIDECAR_SUFFIX = ".sha256"
_DIGEST_BYTES = 32 # SHA-256
_DIGEST_HEX_LEN = _DIGEST_BYTES * 2
class Sha256SidecarError(RuntimeError):
"""Raised by `Sha256Sidecar` on any sidecar / atomicity / aggregate failure.
Wraps the underlying `OSError` (or `ValueError`) so callers only ever
handle one exception hierarchy from the helper.
"""
def _sidecar_path(payload_path: Path) -> Path:
"""Return ``<path>.sha256`` — always appended verbatim to the full path string.
`Path.with_suffix` would re-interpret an existing extension; we want a
pure append so ``manifest`` -> ``manifest.sha256`` and
``engine.engine`` -> ``engine.engine.sha256``.
"""
return Path(str(payload_path) + _SIDECAR_SUFFIX)
def _digest_bytes(payload: bytes) -> str:
return hashlib.sha256(payload).hexdigest()
def _digest_file(payload_path: Path) -> str:
"""Stream-hash a file from disk so we never trust the in-memory copy."""
hasher = hashlib.sha256()
with payload_path.open("rb") as fh:
while True:
chunk = fh.read(1024 * 1024)
if not chunk:
break
hasher.update(chunk)
return hasher.hexdigest()
def _validate_sidecar_text(sidecar_text: str) -> str:
"""Return the cleaned hex digest or raise `Sha256SidecarError`."""
if len(sidecar_text) != _DIGEST_HEX_LEN:
raise Sha256SidecarError(
f"malformed sidecar: expected exactly {_DIGEST_HEX_LEN} hex chars, "
f"got {len(sidecar_text)} bytes (content: {sidecar_text!r})"
)
try:
int(sidecar_text, 16)
except ValueError as exc:
raise Sha256SidecarError(
f"malformed sidecar: not a hex digest ({sidecar_text!r}): {exc}"
) from exc
if sidecar_text.lower() != sidecar_text:
raise Sha256SidecarError(
f"malformed sidecar: hex digest must be lowercase ({sidecar_text!r})"
)
return sidecar_text
class Sha256Sidecar:
"""Atomic-write + SHA-256 sidecar facade.
Static-only by design — no per-call state is meaningful. Atomicity
and verification invariants are documented at the contract level.
"""
@staticmethod
def write_atomic(path: Path, payload: bytes) -> str:
"""Atomically write `payload` to `path`; return its SHA-256 hex digest."""
digest = _digest_bytes(payload)
try:
with atomic_write(str(path), mode="wb", overwrite=True) as fh:
fh.write(payload)
except OSError as exc:
raise Sha256SidecarError(f"write_atomic: failed to write {path}: {exc}") from exc
return digest
@staticmethod
def write_atomic_and_sidecar(path: Path, payload: bytes) -> str:
"""Atomically write `payload` and its `<path>.sha256` sidecar.
Both writes go through the temp-file + rename atomic-write
pattern. Returns the hex digest that was written.
"""
digest = Sha256Sidecar.write_atomic(path, payload)
sidecar = _sidecar_path(path)
try:
with atomic_write(str(sidecar), mode="w", overwrite=True) as fh:
fh.write(digest)
except OSError as exc:
raise Sha256SidecarError(
f"write_atomic_and_sidecar: failed to write sidecar at {sidecar}: {exc}"
) from exc
return digest
@staticmethod
def verify(path: Path) -> bool:
"""Recompute the on-disk SHA-256 and compare with the sidecar.
Returns False if `path` is missing entirely (a missing artifact
is "not verifiable" rather than an error in the verification
contract — callers can branch on `path.exists()` first if they
need to distinguish). Raises `Sha256SidecarError` if `path`
exists but the sidecar is missing or malformed.
"""
if not path.exists():
return False
sidecar = _sidecar_path(path)
if not sidecar.exists():
raise Sha256SidecarError(f"verify: sidecar missing for {path} (expected at {sidecar})")
try:
sidecar_text = sidecar.read_text()
except OSError as exc:
raise Sha256SidecarError(f"verify: cannot read sidecar at {sidecar}: {exc}") from exc
expected = _validate_sidecar_text(sidecar_text)
try:
actual = _digest_file(path)
except OSError as exc:
raise Sha256SidecarError(f"verify: cannot read payload at {path}: {exc}") from exc
return actual == expected
@staticmethod
def aggregate_hash(paths: list[Path]) -> str:
"""Order-deterministic SHA-256 over many files (Manifest aggregate).
Inputs are sorted by full path (case-sensitive) before hashing,
so two runs over the same set produce byte-equal digests. The
aggregate is the SHA-256 of the concatenation of
``<filename>\\0<file-hex-digest>\\n`` lines.
"""
sorted_paths = sorted(paths, key=lambda p: str(p))
hasher = hashlib.sha256()
for path in sorted_paths:
if not path.exists():
raise Sha256SidecarError(f"aggregate_hash: missing path in input: {path}")
try:
digest = _digest_file(path)
except OSError as exc:
raise Sha256SidecarError(f"aggregate_hash: cannot read {path}: {exc}") from exc
hasher.update(path.name.encode("utf-8"))
hasher.update(b"\0")
hasher.update(digest.encode("ascii"))
hasher.update(b"\n")
return hasher.hexdigest()
# Public constant for callers that need to spell the sidecar suffix
# explicitly (e.g. takeoff-load verifier listing).
SIDECAR_SUFFIX = _SIDECAR_SUFFIX
+10 -5
View File
@@ -1,9 +1,14 @@
"""Structured JSON logging entrypoint (E-CC-LOG / AZ-245).
"""Structured JSON logging entrypoint (E-CC-LOG / AZ-245 / AZ-266).
Bootstrap (AZ-263) ships a working `get_logger` so every other module can import it;
the concrete sink + redaction policy is layered on by AZ-266.
Public surface — every component imports `get_logger` from here. The
handler topology is selected by `configure_logging(tier=...)` at the
composition-root entrypoint.
"""
from gps_denied_onboard.logging.structured import get_logger
from gps_denied_onboard.logging.structured import (
JsonFormatter,
configure_logging,
get_logger,
)
__all__ = ["get_logger"]
__all__ = ["JsonFormatter", "configure_logging", "get_logger"]
+250 -61
View File
@@ -1,83 +1,272 @@
"""Structured JSON logging.
"""Shared structured JSON logging (E-CC-LOG / AZ-245 / AZ-266).
E-CC-LOG / AZ-245 contract: one JSON object per log line. Bootstrap provides a
minimal working `get_logger(name)` so every other module can import it; AZ-266
will add full redaction and the FDR sink.
Implements the `log_record_schema` v1.0.0 contract at
`_docs/02_document/contracts/shared_logging/log_record_schema.md`:
- One JSON object per log line.
- Stable field order: ``ts, level, component, frame_id, kind, msg, kv, exc``.
- Level normalisation: ``WARNING`` -> ``WARN``.
- ``frame_id`` is explicit ``null`` for non-frame records.
- Formatter never raises into the caller: a serialisation failure replaces
the offending record's ``kv`` payload with
``{"_format_error": "<reason>"}`` and adds an internal WARN to the
emitted bytes; the rest of the record still goes out.
Public surface (consumed by every onboard component):
- ``get_logger(component_id)`` -> ``logging.Logger`` (cached).
- ``configure_logging(tier, level)`` -> attach the handler topology for
the active deployment tier without duplicating handlers on re-init.
"""
from __future__ import annotations
import datetime as _dt
import json
import logging
import os
import sys
import time
from typing import Any
import threading
from collections.abc import Iterable
from typing import Any, Final
# Schema field order (REQUIRED by contract; verified by AC-2).
_CONTRACT_FIELDS: Final[tuple[str, ...]] = (
"ts",
"level",
"component",
"frame_id",
"kind",
"msg",
"kv",
"exc",
)
# Default `kind` for records that don't pass one explicitly (startup / shutdown / unclassified).
_DEFAULT_KIND: Final[str] = "log.diag"
# Python stdlib LogRecord attributes that must not leak into `kv` when we
# auto-collect record extras. Kept as a frozenset for O(1) lookup.
_RESERVED_LOG_RECORD_KEYS: Final[frozenset[str]] = frozenset(
{
"args",
"asctime",
"created",
"exc_info",
"exc_text",
"filename",
"funcName",
"levelname",
"levelno",
"lineno",
"message",
"module",
"msecs",
"msg",
"name",
"pathname",
"process",
"processName",
"relativeCreated",
"stack_info",
"taskName",
"thread",
"threadName",
# Contract-named keys live in the top-level payload, not in `kv`.
"frame_id",
"kind",
"kv",
"component",
}
)
class _JsonFormatter(logging.Formatter):
"""Emit a single JSON object per log line — no narrative log lines (E-CC-LOG)."""
def _iso_utc(created_epoch: float) -> str:
"""Return RFC 3339 UTC timestamp with microsecond precision and ``Z`` suffix."""
dt = _dt.datetime.fromtimestamp(created_epoch, tz=_dt.timezone.utc)
return dt.strftime("%Y-%m-%dT%H:%M:%S.") + f"{dt.microsecond:06d}Z"
def _normalise_level(stdlib_levelname: str) -> str:
"""Translate Python stdlib level names to the contract enum."""
if stdlib_levelname == "WARNING":
return "WARN"
return stdlib_levelname
def _coerce_jsonable(value: Any) -> Any:
"""Coerce arbitrary kv payloads into JSON-safe primitives.
Raises ``TypeError`` / ``ValueError`` on un-serialisable content; the
caller (the formatter) is responsible for replacing offending payloads
with ``{"_format_error": ...}``.
"""
if isinstance(value, (str, int, float, bool)) or value is None:
return value
if isinstance(value, dict):
return {str(k): _coerce_jsonable(v) for k, v in value.items()}
if isinstance(value, (list, tuple)):
return [_coerce_jsonable(v) for v in value]
raise TypeError(f"unserialisable kv payload type: {type(value).__name__}")
class JsonFormatter(logging.Formatter):
"""Emit one schema-compliant JSON object per log record.
Field order is locked by the contract — we build the payload via an
ordered ``dict`` and rely on Python 3.7+ insertion-order preservation
plus ``json.dumps(..., sort_keys=False)``.
"""
def format(self, record: logging.LogRecord) -> str:
payload: dict[str, Any] = {
"ts": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(record.created))
+ f".{int(record.msecs):03d}Z",
"level": record.levelname,
"logger": record.name,
"msg": record.getMessage(),
}
rec_dict = record.__dict__
frame_id = rec_dict.get("frame_id")
kind = rec_dict.get("kind") or _DEFAULT_KIND
explicit_kv = rec_dict.get("kv")
if explicit_kv is None:
kv_raw: dict[str, Any] = {
k: v
for k, v in rec_dict.items()
if k not in _RESERVED_LOG_RECORD_KEYS and not k.startswith("_")
}
else:
kv_raw = dict(explicit_kv)
try:
kv_safe = _coerce_jsonable(kv_raw)
except (TypeError, ValueError) as exc:
kv_safe = {"_format_error": f"{type(exc).__name__}: {exc}"}
if record.exc_info:
payload["exc"] = self.formatException(record.exc_info)
for key, value in record.__dict__.items():
if key in (
"args",
"msg",
"levelname",
"name",
"exc_info",
"exc_text",
"stack_info",
"created",
"msecs",
"relativeCreated",
"thread",
"threadName",
"processName",
"process",
"module",
"funcName",
"filename",
"pathname",
"lineno",
"levelno",
):
continue
payload.setdefault(key, value)
return json.dumps(payload, separators=(",", ":"), default=str)
exc_text: str | None = self.formatException(record.exc_info)
else:
exc_text = None
component = rec_dict.get("component") or record.name
payload: dict[str, Any] = {
"ts": _iso_utc(record.created),
"level": _normalise_level(record.levelname),
"component": component,
"frame_id": frame_id,
"kind": kind,
"msg": record.getMessage().replace("\n", " "),
"kv": kv_safe,
"exc": exc_text,
}
ordered_payload = {field: payload[field] for field in _CONTRACT_FIELDS}
return json.dumps(ordered_payload, separators=(",", ":"), default=str, sort_keys=False)
_CONFIGURED = False
# Module-level guard for handler-topology re-initialisation idempotency (AC-4).
_LOGGING_LOCK = threading.Lock()
_HANDLER_MARKER_ATTR: Final[str] = "_gps_denied_handler_kind"
def _configure_root_once() -> None:
global _CONFIGURED
if _CONFIGURED:
return
handler = logging.StreamHandler(sys.stderr)
handler.setFormatter(_JsonFormatter())
root = logging.getLogger()
root.handlers.clear()
root.addHandler(handler)
level_name = os.getenv("LOG_LEVEL", "INFO").upper()
root.setLevel(getattr(logging, level_name, logging.INFO))
_CONFIGURED = True
def _make_tier1_handler() -> logging.Handler:
"""Tier-1: stdout handler (Docker captures stdout)."""
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(JsonFormatter())
setattr(handler, _HANDLER_MARKER_ATTR, "tier1.stdout")
return handler
def get_logger(name: str) -> logging.Logger:
"""Return a structured JSON logger.
def _make_tier2_handler() -> logging.Handler:
"""Tier-2: journald handler (Jetson with systemd).
Every component imports its logger via
`from gps_denied_onboard.logging import get_logger`.
journald is reached via ``systemd.journal.JournalHandler``; if the
``systemd-python`` package is not installed on the active host, the
factory raises so callers (and tests) get a clear prerequisite signal
rather than a silent stderr fallback that would violate AC-4's
"exactly one journald handler" invariant.
"""
_configure_root_once()
return logging.getLogger(name)
try:
from systemd.journal import JournalHandler # type: ignore[import-not-found]
except ImportError as exc:
raise RuntimeError(
"Tier-2 journald handler requires `systemd-python`. Install it on the "
"Jetson runtime or select `tier=1` for local/Docker deployments."
) from exc
handler = JournalHandler()
handler.setFormatter(JsonFormatter())
setattr(handler, _HANDLER_MARKER_ATTR, "tier2.journald")
return handler
_TIER_FACTORIES: Final[dict[int, Any]] = {
1: _make_tier1_handler,
2: _make_tier2_handler,
}
def configure_logging(
*,
tier: int,
level: str = "INFO",
target_loggers: Iterable[str] = ("",),
) -> None:
"""Install the handler topology for ``tier`` on the named loggers.
Idempotent: re-calling with the same tier replaces any prior handler
of that kind (no duplicates — AC-4). Switching tiers removes prior
tier handlers and installs the new one.
By default, the root logger (``""``) is configured so every named
logger inherits the handler.
"""
if tier not in _TIER_FACTORIES:
raise ValueError(f"unsupported logging tier: {tier!r} (expected 1 or 2)")
new_handler = _TIER_FACTORIES[tier]()
level_value = getattr(logging, level.upper(), logging.INFO)
with _LOGGING_LOCK:
for name in target_loggers:
target = logging.getLogger(name)
target.handlers = [
h for h in target.handlers if not getattr(h, _HANDLER_MARKER_ATTR, None)
]
target.addHandler(new_handler)
target.setLevel(level_value)
target.propagate = name != ""
def _bootstrap_default_handler() -> None:
"""Attach a default Tier-1 handler if no schema handler is yet installed.
Called lazily on first ``get_logger`` so importing a component that
logs at module-import time still produces well-formed records before
the composition root runs ``configure_logging``.
"""
root = logging.getLogger()
has_schema_handler = any(getattr(h, _HANDLER_MARKER_ATTR, None) for h in root.handlers)
if has_schema_handler:
return
env_tier = os.getenv("GPS_DENIED_TIER", "1")
tier = 2 if env_tier.strip() == "2" else 1
env_level = os.getenv("LOG_LEVEL", "INFO")
try:
configure_logging(tier=tier, level=env_level)
except RuntimeError:
configure_logging(tier=1, level=env_level)
def get_logger(component_id: str) -> logging.Logger:
"""Return a `Logger` whose records satisfy `log_record_schema` v1.0.0.
Repeated calls with the same `component_id` return the same cached
`Logger` instance (Python stdlib's named-logger registry). The first
call installs a default Tier-1 handler if the composition root has
not yet run ``configure_logging``.
"""
_bootstrap_default_handler()
logger = logging.getLogger(component_id)
logger.setLevel(logging.NOTSET)
return logger
# Backwards-compat alias for the formatter's prior name (used by Tier-1 unit tests).
_JsonFormatter = JsonFormatter