mirror of
https://github.com/azaion/gps-denied-onboard.git
synced 2026-06-22 16:11:13 +00:00
[AZ-305] c6 PostgresFilesystemStore: TileStore + TileMetadataStore impl
Adds the production PostgresFilesystemStore implementing both protocols in a single class. Filesystem-backed JPEG I/O (atomic sidecar write, read-only mmap) + Postgres-backed metadata (spatial bbox, LRU, voting, upload bookkeeping). Wires composition via `from_config` classmethod. Key behaviors: - AC-3 strict reading: INSERT runs first inside an open transaction; duplicate-key collisions raise `TileMetadataError` BEFORE any byte is written, leaving the original file + sidecar byte-identical. Atomic sidecar write happens inside the same transaction; commit closes it. Comp-delete remains as a safety net for the rare commit-after-write failure path. - AC-2 content-hash gate runs before any I/O. - Construction performs an orphan-file reconciliation scan and emits an INFO `c6.store.construct` log with steady-state stats. Adds `c6.write` and `c6.write_failed` FDR record kinds (schema v1.1.0, forward-compatible) and a thin operator CLI at `c6_tile_cache.tools dump` for inspection. Dependencies: adds `psycopg-pool>=3.2,<4.0` for the connection pool used on the F3 read-hot path. Tests: 25 new tests for c6_tile_cache cover AC-1..AC-15 plus MmapTilePixelHandle + helper round-trips. Full Tier-2 unit suite passes (1215 passed, 8 skipped, 1 pre-existing unrelated failure `test_ac8_read_host_tuple_on_jetson` — missing `pynvml` on macOS, Jetson-only). Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -15,10 +15,10 @@ from typing import Final
|
||||
from gps_denied_onboard.config.schema import ConfigError
|
||||
|
||||
__all__ = [
|
||||
"C6TileCacheConfig",
|
||||
"KNOWN_DESCRIPTOR_INDEX_RUNTIMES",
|
||||
"KNOWN_METADATA_RUNTIMES",
|
||||
"KNOWN_TILE_STORE_RUNTIMES",
|
||||
"C6TileCacheConfig",
|
||||
]
|
||||
|
||||
KNOWN_TILE_STORE_RUNTIMES: Final[frozenset[str]] = frozenset({"postgres_filesystem"})
|
||||
@@ -57,6 +57,7 @@ class C6TileCacheConfig:
|
||||
descriptor_index_runtime: str = "faiss_hnsw"
|
||||
root_dir: str = "/var/lib/gps-denied/tiles"
|
||||
postgres_dsn: str = ""
|
||||
postgres_pool_size: int = 4
|
||||
lru_eviction_threshold_bytes: int = 10 * 1024**3
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
@@ -78,6 +79,10 @@ class C6TileCacheConfig:
|
||||
)
|
||||
if not self.root_dir:
|
||||
raise ConfigError("C6TileCacheConfig.root_dir must be non-empty")
|
||||
if self.postgres_pool_size <= 0:
|
||||
raise ConfigError(
|
||||
f"C6TileCacheConfig.postgres_pool_size must be > 0; got {self.postgres_pool_size}"
|
||||
)
|
||||
if self.lru_eviction_threshold_bytes <= 0:
|
||||
raise ConfigError(
|
||||
f"C6TileCacheConfig.lru_eviction_threshold_bytes must be > 0; "
|
||||
|
||||
@@ -164,7 +164,7 @@ class DescriptorIndex(Protocol):
|
||||
"""
|
||||
|
||||
def search_topk(
|
||||
self, query: "np.ndarray", k: int
|
||||
self, query: np.ndarray, k: int
|
||||
) -> list[tuple[TileId, float]]:
|
||||
"""Top-K nearest neighbour search.
|
||||
|
||||
@@ -188,7 +188,7 @@ class DescriptorIndex(Protocol):
|
||||
|
||||
def rebuild_from_descriptors(
|
||||
self,
|
||||
descriptors: "np.ndarray",
|
||||
descriptors: np.ndarray,
|
||||
tile_ids: list[TileId],
|
||||
hnsw_params: HnswParams,
|
||||
) -> None:
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,90 @@
|
||||
"""``c6_tile_cache.tools`` — operator-side CLI for post-flight inspection (AZ-305).
|
||||
|
||||
Usage:
|
||||
|
||||
python -m gps_denied_onboard.components.c6_tile_cache.tools dump \\
|
||||
--zoom 18 --lat 49.94 --lon 36.31 --output tile.jpg
|
||||
|
||||
When ``--output`` is omitted the JPEG bytes are streamed to ``stdout`` so
|
||||
the command composes with shell pipelines (``... | exiftool -``,
|
||||
``... | identify -``, etc).
|
||||
|
||||
No formal contract — this is a thin shell over
|
||||
:meth:`PostgresFilesystemStore.read_tile_pixels`. Config is read from the
|
||||
default :func:`gps_denied_onboard.config.load_config` path so the CLI
|
||||
runs against the same DB / tile root as the companion.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from gps_denied_onboard.components.c6_tile_cache._types import TileId
|
||||
from gps_denied_onboard.components.c6_tile_cache.postgres_filesystem_store import (
|
||||
PostgresFilesystemStore,
|
||||
)
|
||||
from gps_denied_onboard.config import load_config
|
||||
|
||||
|
||||
def _build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="c6_tile_cache.tools",
|
||||
description="Operator-side dump utility for C6 tile cache (AZ-305).",
|
||||
)
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
dump = sub.add_parser(
|
||||
"dump",
|
||||
help="Read a single tile from the cache and write its JPEG body.",
|
||||
)
|
||||
dump.add_argument("--zoom", type=int, required=True, help="Tile zoom level (0..21).")
|
||||
dump.add_argument("--lat", type=float, required=True, help="Tile WGS84 latitude.")
|
||||
dump.add_argument("--lon", type=float, required=True, help="Tile WGS84 longitude.")
|
||||
dump.add_argument(
|
||||
"--output",
|
||||
"-o",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Output file path; defaults to stdout when omitted.",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def _dump_tile(zoom: int, lat: float, lon: float, output: Path | None) -> int:
|
||||
config = load_config()
|
||||
store = PostgresFilesystemStore.from_config(config)
|
||||
tile_id = TileId(zoom_level=zoom, lat=lat, lon=lon)
|
||||
handle = store.read_tile_pixels(tile_id)
|
||||
with handle as view:
|
||||
body = bytes(view)
|
||||
if output is None:
|
||||
sys.stdout.buffer.write(body)
|
||||
sys.stdout.buffer.flush()
|
||||
else:
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_bytes(body)
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
logging.basicConfig(
|
||||
level=os.environ.get("LOG_LEVEL", "INFO"),
|
||||
format="%(asctime)s %(levelname)s %(name)s %(message)s",
|
||||
stream=sys.stderr,
|
||||
)
|
||||
args = _build_parser().parse_args(argv)
|
||||
if args.command == "dump":
|
||||
return _dump_tile(args.zoom, args.lat, args.lon, args.output)
|
||||
# argparse already enforces `required=True` on the subparser dest, so
|
||||
# this branch is unreachable in practice; kept defensive to satisfy
|
||||
# type checkers and to give a clear error if a new subcommand is added
|
||||
# without wiring it through.
|
||||
raise SystemExit(f"unknown command: {args.command}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user