mirror of
https://github.com/azaion/gps-denied-onboard.git
synced 2026-06-22 21:21:14 +00:00
[AZ-305] c6 PostgresFilesystemStore: TileStore + TileMetadataStore impl
Adds the production PostgresFilesystemStore implementing both protocols in a single class. Filesystem-backed JPEG I/O (atomic sidecar write, read-only mmap) + Postgres-backed metadata (spatial bbox, LRU, voting, upload bookkeeping). Wires composition via `from_config` classmethod. Key behaviors: - AC-3 strict reading: INSERT runs first inside an open transaction; duplicate-key collisions raise `TileMetadataError` BEFORE any byte is written, leaving the original file + sidecar byte-identical. Atomic sidecar write happens inside the same transaction; commit closes it. Comp-delete remains as a safety net for the rare commit-after-write failure path. - AC-2 content-hash gate runs before any I/O. - Construction performs an orphan-file reconciliation scan and emits an INFO `c6.store.construct` log with steady-state stats. Adds `c6.write` and `c6.write_failed` FDR record kinds (schema v1.1.0, forward-compatible) and a thin operator CLI at `c6_tile_cache.tools dump` for inspection. Dependencies: adds `psycopg-pool>=3.2,<4.0` for the connection pool used on the F3 read-hot path. Tests: 25 new tests for c6_tile_cache cover AC-1..AC-15 plus MmapTilePixelHandle + helper round-trips. Full Tier-2 unit suite passes (1215 passed, 8 skipped, 1 pre-existing unrelated failure `test_ac8_read_host_tuple_on_jetson` — missing `pynvml` on macOS, Jetson-only). Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -15,10 +15,10 @@ from typing import Final
|
||||
from gps_denied_onboard.config.schema import ConfigError
|
||||
|
||||
__all__ = [
|
||||
"C6TileCacheConfig",
|
||||
"KNOWN_DESCRIPTOR_INDEX_RUNTIMES",
|
||||
"KNOWN_METADATA_RUNTIMES",
|
||||
"KNOWN_TILE_STORE_RUNTIMES",
|
||||
"C6TileCacheConfig",
|
||||
]
|
||||
|
||||
KNOWN_TILE_STORE_RUNTIMES: Final[frozenset[str]] = frozenset({"postgres_filesystem"})
|
||||
@@ -57,6 +57,7 @@ class C6TileCacheConfig:
|
||||
descriptor_index_runtime: str = "faiss_hnsw"
|
||||
root_dir: str = "/var/lib/gps-denied/tiles"
|
||||
postgres_dsn: str = ""
|
||||
postgres_pool_size: int = 4
|
||||
lru_eviction_threshold_bytes: int = 10 * 1024**3
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
@@ -78,6 +79,10 @@ class C6TileCacheConfig:
|
||||
)
|
||||
if not self.root_dir:
|
||||
raise ConfigError("C6TileCacheConfig.root_dir must be non-empty")
|
||||
if self.postgres_pool_size <= 0:
|
||||
raise ConfigError(
|
||||
f"C6TileCacheConfig.postgres_pool_size must be > 0; got {self.postgres_pool_size}"
|
||||
)
|
||||
if self.lru_eviction_threshold_bytes <= 0:
|
||||
raise ConfigError(
|
||||
f"C6TileCacheConfig.lru_eviction_threshold_bytes must be > 0; "
|
||||
|
||||
@@ -164,7 +164,7 @@ class DescriptorIndex(Protocol):
|
||||
"""
|
||||
|
||||
def search_topk(
|
||||
self, query: "np.ndarray", k: int
|
||||
self, query: np.ndarray, k: int
|
||||
) -> list[tuple[TileId, float]]:
|
||||
"""Top-K nearest neighbour search.
|
||||
|
||||
@@ -188,7 +188,7 @@ class DescriptorIndex(Protocol):
|
||||
|
||||
def rebuild_from_descriptors(
|
||||
self,
|
||||
descriptors: "np.ndarray",
|
||||
descriptors: np.ndarray,
|
||||
tile_ids: list[TileId],
|
||||
hnsw_params: HnswParams,
|
||||
) -> None:
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,90 @@
|
||||
"""``c6_tile_cache.tools`` — operator-side CLI for post-flight inspection (AZ-305).
|
||||
|
||||
Usage:
|
||||
|
||||
python -m gps_denied_onboard.components.c6_tile_cache.tools dump \\
|
||||
--zoom 18 --lat 49.94 --lon 36.31 --output tile.jpg
|
||||
|
||||
When ``--output`` is omitted the JPEG bytes are streamed to ``stdout`` so
|
||||
the command composes with shell pipelines (``... | exiftool -``,
|
||||
``... | identify -``, etc).
|
||||
|
||||
No formal contract — this is a thin shell over
|
||||
:meth:`PostgresFilesystemStore.read_tile_pixels`. Config is read from the
|
||||
default :func:`gps_denied_onboard.config.load_config` path so the CLI
|
||||
runs against the same DB / tile root as the companion.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from gps_denied_onboard.components.c6_tile_cache._types import TileId
|
||||
from gps_denied_onboard.components.c6_tile_cache.postgres_filesystem_store import (
|
||||
PostgresFilesystemStore,
|
||||
)
|
||||
from gps_denied_onboard.config import load_config
|
||||
|
||||
|
||||
def _build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="c6_tile_cache.tools",
|
||||
description="Operator-side dump utility for C6 tile cache (AZ-305).",
|
||||
)
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
dump = sub.add_parser(
|
||||
"dump",
|
||||
help="Read a single tile from the cache and write its JPEG body.",
|
||||
)
|
||||
dump.add_argument("--zoom", type=int, required=True, help="Tile zoom level (0..21).")
|
||||
dump.add_argument("--lat", type=float, required=True, help="Tile WGS84 latitude.")
|
||||
dump.add_argument("--lon", type=float, required=True, help="Tile WGS84 longitude.")
|
||||
dump.add_argument(
|
||||
"--output",
|
||||
"-o",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Output file path; defaults to stdout when omitted.",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def _dump_tile(zoom: int, lat: float, lon: float, output: Path | None) -> int:
|
||||
config = load_config()
|
||||
store = PostgresFilesystemStore.from_config(config)
|
||||
tile_id = TileId(zoom_level=zoom, lat=lat, lon=lon)
|
||||
handle = store.read_tile_pixels(tile_id)
|
||||
with handle as view:
|
||||
body = bytes(view)
|
||||
if output is None:
|
||||
sys.stdout.buffer.write(body)
|
||||
sys.stdout.buffer.flush()
|
||||
else:
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_bytes(body)
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
logging.basicConfig(
|
||||
level=os.environ.get("LOG_LEVEL", "INFO"),
|
||||
format="%(asctime)s %(levelname)s %(name)s %(message)s",
|
||||
stream=sys.stderr,
|
||||
)
|
||||
args = _build_parser().parse_args(argv)
|
||||
if args.command == "dump":
|
||||
return _dump_tile(args.zoom, args.lat, args.lon, args.output)
|
||||
# argparse already enforces `required=True` on the subparser dest, so
|
||||
# this branch is unreachable in practice; kept defensive to satisfy
|
||||
# type checkers and to give a clear error if a new subcommand is added
|
||||
# without wiring it through.
|
||||
raise SystemExit(f"unknown command: {args.command}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -115,6 +115,18 @@ KNOWN_PAYLOAD_KEYS: Final[dict[str, frozenset[str]]] = {
|
||||
"measured_at_ns",
|
||||
}
|
||||
),
|
||||
# AZ-305 / E-C6: emitted by PostgresFilesystemStore on every successful
|
||||
# write_tile. `tile_id` is the canonical UUIDv5 derived from
|
||||
# (zoom_level, tile_x, tile_y, source, flight_id). `source` is the
|
||||
# `TileSource` enum value. `disk_bytes` is the JPEG payload length.
|
||||
# `content_sha256` is the lowercase hex digest of the JPEG body.
|
||||
"c6.write": frozenset({"tile_id", "source", "disk_bytes", "content_sha256"}),
|
||||
# AZ-305 / E-C6: emitted on every failed write_tile path. `reason`
|
||||
# is a short machine-readable tag (content_hash_mismatch, freshness_reject,
|
||||
# metadata_error, fs_error); `error_class` is the exception class name;
|
||||
# `message` is the rewrapped exception's str (truncated to keep the
|
||||
# record inline).
|
||||
"c6.write_failed": frozenset({"tile_id", "source", "reason", "error_class", "message"}),
|
||||
}
|
||||
|
||||
KNOWN_KINDS: Final[frozenset[str]] = frozenset(KNOWN_PAYLOAD_KEYS.keys())
|
||||
|
||||
@@ -52,7 +52,7 @@ def _is_build_flag_on(flag_name: str) -> bool:
|
||||
return raw.strip().lower() in {"on", "1", "true", "yes"}
|
||||
|
||||
|
||||
def _c6_config(config: "Config") -> "C6TileCacheConfig":
|
||||
def _c6_config(config: Config) -> C6TileCacheConfig:
|
||||
"""Pull the registered C6 config block.
|
||||
|
||||
``c6_tile_cache.__init__`` registers it on import; if the package
|
||||
@@ -62,18 +62,21 @@ def _c6_config(config: "Config") -> "C6TileCacheConfig":
|
||||
return config.components["c6_tile_cache"]
|
||||
|
||||
|
||||
def build_tile_store(config: "Config") -> "TileStore":
|
||||
def build_tile_store(config: Config) -> TileStore:
|
||||
"""Construct the :class:`TileStore` impl selected by config.
|
||||
|
||||
Today only ``"postgres_filesystem"`` is wired; the runtime label
|
||||
is validated at config-load time so unknown labels never reach
|
||||
here. Concrete impl is produced by AZ-305.
|
||||
here. Concrete impl produced by AZ-305 — the constructor is
|
||||
invoked via ``PostgresFilesystemStore.from_config(config)`` which
|
||||
wires the ``ConnectionPool`` / ``FdrClient`` / logger / static
|
||||
helper dependencies from the config block.
|
||||
"""
|
||||
block = _c6_config(config)
|
||||
runtime = block.store_runtime
|
||||
if runtime == "postgres_filesystem":
|
||||
try:
|
||||
from gps_denied_onboard.components.c6_tile_cache.postgres_filesystem_store import ( # noqa: PLC0415
|
||||
from gps_denied_onboard.components.c6_tile_cache.postgres_filesystem_store import (
|
||||
PostgresFilesystemStore,
|
||||
)
|
||||
except ModuleNotFoundError as exc:
|
||||
@@ -83,13 +86,13 @@ def build_tile_store(config: "Config") -> "TileStore":
|
||||
"'c6_tile_cache.postgres_filesystem_store' has not been "
|
||||
"built into this binary yet (AZ-305 pending)."
|
||||
) from exc
|
||||
return PostgresFilesystemStore(config)
|
||||
return PostgresFilesystemStore.from_config(config)
|
||||
raise RuntimeNotAvailableError(
|
||||
f"TileStore runtime {runtime!r} is not buildable in this binary."
|
||||
)
|
||||
|
||||
|
||||
def build_tile_metadata_store(config: "Config") -> "TileMetadataStore":
|
||||
def build_tile_metadata_store(config: Config) -> TileMetadataStore:
|
||||
"""Construct the :class:`TileMetadataStore` impl selected by config.
|
||||
|
||||
Today the same ``PostgresFilesystemStore`` class implements both
|
||||
@@ -102,7 +105,7 @@ def build_tile_metadata_store(config: "Config") -> "TileMetadataStore":
|
||||
runtime = block.metadata_runtime
|
||||
if runtime == "postgres_filesystem":
|
||||
try:
|
||||
from gps_denied_onboard.components.c6_tile_cache.postgres_filesystem_store import ( # noqa: PLC0415
|
||||
from gps_denied_onboard.components.c6_tile_cache.postgres_filesystem_store import (
|
||||
PostgresFilesystemStore,
|
||||
)
|
||||
except ModuleNotFoundError as exc:
|
||||
@@ -112,13 +115,13 @@ def build_tile_metadata_store(config: "Config") -> "TileMetadataStore":
|
||||
"'c6_tile_cache.postgres_filesystem_store' has not been "
|
||||
"built into this binary yet (AZ-305 pending)."
|
||||
) from exc
|
||||
return PostgresFilesystemStore(config)
|
||||
return PostgresFilesystemStore.from_config(config)
|
||||
raise RuntimeNotAvailableError(
|
||||
f"TileMetadataStore runtime {runtime!r} is not buildable in this binary."
|
||||
)
|
||||
|
||||
|
||||
def build_descriptor_index(config: "Config") -> "DescriptorIndex":
|
||||
def build_descriptor_index(config: Config) -> DescriptorIndex:
|
||||
"""Construct the :class:`DescriptorIndex` impl selected by config.
|
||||
|
||||
Gated by ``BUILD_FAISS_INDEX``: if the flag is OFF, the concrete
|
||||
@@ -134,7 +137,7 @@ def build_descriptor_index(config: "Config") -> "DescriptorIndex":
|
||||
"BUILD_FAISS_INDEX=ON in this binary; the flag is OFF."
|
||||
)
|
||||
try:
|
||||
from gps_denied_onboard.components.c6_tile_cache.faiss_descriptor_index import ( # noqa: PLC0415
|
||||
from gps_denied_onboard.components.c6_tile_cache.faiss_descriptor_index import (
|
||||
FaissDescriptorIndex,
|
||||
)
|
||||
except ModuleNotFoundError as exc:
|
||||
|
||||
Reference in New Issue
Block a user