From 811b04e6057e74f26a8dc8e8eced8e7dca40cd75 Mon Sep 17 00:00:00 2001 From: Oleksandr Bezdieniezhnykh Date: Thu, 21 May 2026 14:52:39 +0300 Subject: [PATCH] [AZ-777] Phase 1: wire e2e-runner to real satellite-provider + C11 contract adapt Adapt C11 HttpTileDownloader to the AZ-505 v1.0.0 tile-inventory contract (POST /api/satellite/tiles/inventory + GET /tiles/{z}/{x}/{y}) and wire the Jetson e2e harness against the real parent-suite satellite-provider service. Closes Phase 1 of 5 for AZ-777; STOP gate before Phase 2 (Derkachi catalog seed). C11 changes: - _LIST_PATH / _GET_PATH replaced with _INVENTORY_PATH + _TILES_PATH. - _do_enumerate enumerates bbox tile coords client-side and posts chunked inventory requests (5000-entry cap per the contract). - _download_one_tile parses tile_id_str into (z,x,y) and fetches the slippy-map URL. - Common GET / POST retry+auth ladder consolidated into _send_request. - New module helpers: _enumerate_bbox_tile_coords, _tile_center_latlon, _tile_size_meters_at, _format_tile_id_str, _parse_tile_id_str, _chunk_iter. - _DEFAULT_ESTIMATED_TILE_BYTES (50 KiB) replaces the inventory-side estimatedBytes field the v1.0.0 contract dropped. Tests: - 14/14 unit tests in tests/unit/c11_tile_manager/test_tile_downloader.py rewritten for the new POST inventory + slippy-map GET handler. _StubTileWriter rekeyed by call-index (the downloader now derives lat/lon from the slippy-map coord, so fixtures can't fabricate arbitrary positions). - New Tier-2 smoke at tests/e2e/satellite_provider/test_smoke.py: validates inventory POST schema + drives HttpTileDownloader against the real service. Gated by RUN_REPLAY_E2E=1 + tier2. Compose / env: - e2e-runner SATELLITE_PROVIDER_URL switched from mock-sat:5100 to https://satellite-provider:8080; TLS_INSECURE + Bearer JWT env + depends_on satellite-provider added. - .env.test.example documents SATELLITE_PROVIDER_API_KEY + dev TLS bypass security note. - scripts/mint_dev_jwt.py mints HS256 dev JWTs from env / .env.test. - pyjwt added to dev extras. Tracker hygiene: - AZ-777 row in _dependencies_table.md bumped 5pt -> 8pt to match the 2026-05-21 override decision log. Code review: PASS_WITH_WARNINGS (3 medium/low findings, all deferred to later AZ-777 phases) -- see batch_104_review.md. Batch report at batch_104_cycle3_report.md. Co-authored-by: Cursor --- .env.test.example | 17 + _docs/02_tasks/_dependencies_table.md | 4 +- .../batch_104_cycle3_report.md | 120 +++++++ .../reviews/batch_104_review.md | 241 +++++++++++++ _docs/_autodev_state.md | 6 +- docker-compose.test.jetson.yml | 21 +- pyproject.toml | 4 + scripts/mint_dev_jwt.py | 137 +++++++ .../c11_tile_manager/tile_downloader.py | 296 ++++++++++++--- tests/e2e/satellite_provider/__init__.py | 8 + tests/e2e/satellite_provider/test_smoke.py | 325 +++++++++++++++++ .../c11_tile_manager/test_tile_downloader.py | 339 +++++++++++------- 12 files changed, 1328 insertions(+), 190 deletions(-) create mode 100644 _docs/03_implementation/batch_104_cycle3_report.md create mode 100644 _docs/03_implementation/reviews/batch_104_review.md create mode 100644 scripts/mint_dev_jwt.py create mode 100644 tests/e2e/satellite_provider/__init__.py create mode 100644 tests/e2e/satellite_provider/test_smoke.py diff --git a/.env.test.example b/.env.test.example index fc9346c..23b347c 100644 --- a/.env.test.example +++ b/.env.test.example @@ -23,3 +23,20 @@ JWT_AUDIENCE=DEV-ONLY-aud-satellite-provider # you need to exercise the real GMaps tile-download path, set this to a # valid key. GOOGLE_MAPS_API_KEY= + +# AZ-777: Bearer token C11 sends to satellite-provider as +# `Authorization: Bearer `. The token is a JWT signed with +# JWT_SECRET above and stamped with the same iss/aud the provider +# validates. Mint a dev token with: +# python scripts/mint_dev_jwt.py +# Production deploys retrieve this from the admin API and rotate per +# operator session — never commit a real one. +SATELLITE_PROVIDER_API_KEY=PASTE-MINTED-JWT-HERE + +# SECURITY: development-only TLS bypass for the parent-suite +# satellite-provider self-signed dev cert. The compose env block sets +# SATELLITE_PROVIDER_TLS_INSECURE=1 — it stays inside the Jetson e2e +# harness, never in production. Production deploys MUST use a real +# CA-issued cert (or your own internal CA) and leave this unset (or +# set to "0"). C11 logs a single WARNING at startup whenever the +# insecure flag is active so the operator can audit it. diff --git a/_docs/02_tasks/_dependencies_table.md b/_docs/02_tasks/_dependencies_table.md index 2b90324..ac9a73c 100644 --- a/_docs/02_tasks/_dependencies_table.md +++ b/_docs/02_tasks/_dependencies_table.md @@ -2,7 +2,7 @@ **Date**: 2026-05-21 (cycle-3 Step 9 New Task — added AZ-776 (3pt open-loop ESKF composition profile via `c4_pose.enabled` flag, no deps, epic AZ-602) + AZ-777 (5pt Derkachi C6 reference tile cache + FAISS descriptor index from OSM/CARTO basemap, depends on AZ-776, epic AZ-602). Both unblock the 7 currently-`@xfail`-masked Derkachi e2e tests on Jetson; AZ-776 unblocks 5 (AC-1, AC-2, AC-5, AC-6 realtime, AC-6 asap), AZ-777 unblocks the remaining 2 (AC-3 + AZ-699 real-flight verdict). Earlier 2026-05-19 (refreshed late-morning after 11:27 Jetson Tier-2 e2e run for AZ-618 — surfaced a NEW gap: replay-mode `Config` lacks `c6_tile_cache` block, so `build_pre_constructed → _build_c6_descriptor_index → _c6_config` raises `KeyError` for AC-1/2/5/6. Follow-up filed as AZ-687 (2pt) under E-AZ-602 with guard at the bootstrap layer (NOT silent fallback in `_c6_config`). Earlier same-day mid-day after AZ-618 split: per the spec author's own Sizing-note recommendation + user-rule cap on PBI complexity, AZ-618 was split into 6 subtasks AZ-619..AZ-624 in Jira (subtasks of AZ-618; epic AZ-602 stays grandparent). AZ-618 retained at 0pt as the umbrella tracker; aggregate actionable work is 16pt across the subtasks (vs. AZ-618's original 5pt filing — author's "likely a true 8" caveat was understated due to c5_isam2_graph_handle ordering + GPU builder unknowns). Earlier same-day refresh at start of Step-7 rewind for AZ-618 — Step-11 Jetson tier-2 e2e gate identified missing internal product implementation: `runtime_root.main()` does not build the airborne `pre_constructed` infrastructure dict before `compose_root()`; AZ-618 = 5pt cross-cutting follow-up to AZ-591, lives under E-AZ-602; all 12 dep tasks are in `done/`. Earlier 2026-05-16 (cycle-1 completeness-gate post-mortem): AZ-589 + AZ-590 closed Won't Fix — were wrong abstraction (OKVIS v1 `ThreadedKFVio` API doesn't exist in OKVIS2 upstream; VINS-Mono `cpp/vins_mono/upstream/` submodule never existed; the actual production gap is the empty central `_STRATEGY_REGISTRY` affecting EVERY component with a strategy-selecting config field, not just c1_vio); replaced by AZ-591 (cross-cutting compose_root per-binary bootstrap, todo/, 5pt) + AZ-592 (AZ-332 Tier-2 validation bundle, backlog/, 5pt placeholder) + AZ-593 (AZ-333 Tier-2 validation bundle, backlog/, 5pt placeholder); AZ-332 + AZ-333 re-classified in gate report from FAIL to BLOCKED-on-Tier-2 per the original tasks' Implementation Notes deferral handles; earlier same-day after end of cycle-1 gate: AZ-589 + AZ-590 created (now closed); earlier same-day after end of Batch 64: AZ-558 implementation closed — `MavlinkTransport` seam now routes every C8 outbound MAVLink byte; AZ-401 AC-9 + AZ-404 AC-4b unskipped together; encoder helpers extracted to `_outbound_mavlink_payloads.py`; live-mode `compose_root` injection deferred to whichever future batch registers AP/iNav strategies in an airborne binary; earlier 2026-05-14: refreshed at start of Batch 63: AZ-559 closed Won't Fix — gap was illusory; `TileSource.ONBOARD_INGEST` + `TileMetadata.quality_metadata` + `write_tile`'s `FreshnessRejectionError` already cover the AZ-389 mid-flight ingest semantic without any new API; AZ-389 dep restored to AZ-303; earlier same-day after Batch 61: AZ-558 follow-up added — routes C8 outbound encoder bytes through `MavlinkTransport` seam; closes AZ-401 AC-9 deferred during batch 61 due to encoder-side routing not being in the AZ-401 task envelope; earlier same-day after cumulative review batches 52-54: AZ-528 hygiene PBI added for c1_vio strategy facade orchestration-spine 3-way duplication (Medium); earlier same-day after Batch 53: AZ-333 VINS-Mono landed — first c1_vio strategy after the AZ-332 OKVIS2 production-default; consolidation hygiene for the strategy-facade duplication deferred to a post-AZ-334 PBI; earlier same-day after Batch 51: AZ-527 hygiene PBI added from cumulative review batches 49-51 F1; 2026-05-13: AZ-526 hygiene PBI added from cumulative review batches 46-48 F1+F3; same-day refresh after Batch 44 SRP refactor: AZ-317 superseded; AZ-329 + AZ-330 specs rewritten; AZ-523 + AZ-524 audit-trail tickets added; E-C12 epic renamed `Operator Pre-flight Tooling` → `Operator Pre-flight Orchestrator`; earlier same-day refresh: AZ-507 + AZ-508 hygiene PBIs from cumulative review batches 31-33; 2026-05-11: AZ-489 + AZ-490 ADR-010 operator-origin path) **Total Tasks**: 165 (124 product + 41 blackbox-test) — AZ-317 retained in the table marked SUPERSEDED for audit; AZ-523 (C11 gate removal) + AZ-524 (C12 rename) added as 2 closed audit-trail tasks; AZ-526 = 2pt clock-helper hygiene; AZ-527 = 2pt c2 engine-dim helper hygiene; AZ-528 = 3pt c1_vio facade-spine hygiene; AZ-558 = 3pt MavlinkTransport routing follow-up; AZ-559 closed Won't Fix; AZ-589 + AZ-590 closed Won't Fix (kept in table as 0pt audit-trail rows); AZ-591 = 5pt cross-cutting compose_root bootstrap (todo/); AZ-592 = 5pt OKVIS2 Tier-2 placeholder (backlog/); AZ-593 = 5pt VINS-Mono Tier-2 placeholder (backlog/); AZ-618 = 0pt umbrella (split into AZ-619..AZ-624 on 2026-05-19); AZ-619..AZ-624 = 6 subtasks of AZ-618 covering Phase A..F of the airborne `pre_constructed` assembly, summing to 16pt actionable work; AZ-687 = 2pt replay-mode guard follow-up surfaced by AZ-618 Tier-2 run on 2026-05-19 -**Total Complexity Points**: 543 (410 product + 133 blackbox-test) — +3pt AZ-776 + 5pt AZ-777 added 2026-05-21 — AZ-523 = 3pt, AZ-524 = 2pt, AZ-526 = 2pt, AZ-527 = 2pt, AZ-528 = 3pt, AZ-558 = 3pt, AZ-589 + AZ-590 retained at 5pt each but closed Won't Fix (treated as 0 effective pts going forward), AZ-591 = 5pt, AZ-592 = 5pt placeholder, AZ-593 = 5pt placeholder, AZ-618 = 0pt umbrella post-split, AZ-619 = 2pt, AZ-620 = 3pt, AZ-621 = 3pt, AZ-622 = 3pt, AZ-623 = 3pt, AZ-624 = 2pt, AZ-687 = 2pt +**Total Complexity Points**: 546 (413 product + 133 blackbox-test) — +3pt AZ-776 + 8pt AZ-777 (5→8 override 2026-05-21 cycle-3 batch 104; see `_docs/_process_leftovers/2026-05-21_az777_complexity_override.md` for rationale + the spec refresh that pulled e2e-runner wiring + C11 contract adapt + Derkachi catalog seed + fixture replacement + un-xfail into one ticket) — AZ-523 = 3pt, AZ-524 = 2pt, AZ-526 = 2pt, AZ-527 = 2pt, AZ-528 = 3pt, AZ-558 = 3pt, AZ-589 + AZ-590 retained at 5pt each but closed Won't Fix (treated as 0 effective pts going forward), AZ-591 = 5pt, AZ-592 = 5pt placeholder, AZ-593 = 5pt placeholder, AZ-618 = 0pt umbrella post-split, AZ-619 = 2pt, AZ-620 = 3pt, AZ-621 = 3pt, AZ-622 = 3pt, AZ-623 = 3pt, AZ-624 = 2pt, AZ-687 = 2pt Dependencies columns list only the tracker-ID portion (descriptive tail text in each task spec is omitted here for table-readability). The @@ -184,7 +184,7 @@ are all declared and documented below under **Cycle Check**. | AZ-701 | T5: HTTP Replay API service (POST tlog+video, return GPS fixes + map) | 5 | AZ-699, AZ-700 | AZ-696 | | AZ-702 | T6: Topotek KHP20S30 camera calibration (factory-sheet approximation) | 1 | None | AZ-696 | | AZ-776 | Open-loop ESKF composition profile (c4_pose.enabled flag) | 3 | None | AZ-602 | -| AZ-777 | Derkachi C6 reference tile cache + FAISS descriptor index (OSM/CARTO) | 5 | AZ-776 | AZ-602 | +| AZ-777 | Derkachi e2e: wire EXISTING parent-suite satellite-provider into operator pre-flight fixture | 8 (override) | AZ-776 | AZ-602 | ## Notes diff --git a/_docs/03_implementation/batch_104_cycle3_report.md b/_docs/03_implementation/batch_104_cycle3_report.md new file mode 100644 index 0000000..6351fdd --- /dev/null +++ b/_docs/03_implementation/batch_104_cycle3_report.md @@ -0,0 +1,120 @@ +# Batch 104 — Cycle 3 — AZ-777 Phase 1 + +**Date**: 2026-05-21 +**Tasks**: AZ-777 Phase 1 (e2e-runner wire + C11 contract adapt + smoke test). +**Story points**: 8 (explicit override; see decision log). +**Jira status**: AZ-777 → still `In Progress` — Phase 1 of 5 done; STOP gate before Phase 2. + +## What shipped + +The Jetson e2e harness now consumes the **real** parent-suite +`satellite-provider` .NET service over its compose-DNS name + +self-signed dev TLS cert + Bearer JWT auth. C11's +`HttpTileDownloader` has been adapted to the AZ-505 v1.0.0 +`tile-inventory.md` contract — bulk POST inventory lookup keyed by +slippy-map (z,x,y) coords, plus per-tile GET via +`/tiles/{z}/{x}/{y}`. A Tier-2 smoke test exercises the wire +end-to-end against the running service. + +This batch closes the first of AZ-777's five explicit STOP-gated +phases. Phases 2–5 remain on the to-do queue: + +- Phase 2 — Derkachi tile catalog seed via + `POST /api/satellite/request` (CC-BY basemap source, license + attribution baked in). +- Phase 3 — replace the placeholder `operator_pre_flight_setup` + fixture with a real C10 + C11 driver that yields a + `PopulatedC6Cache`. +- Phase 4 — un-xfail the Tier-2 Derkachi AC-3 + AZ-699 verdict + tests. +- Phase 5 — extend the replay-protocol / architecture / Derkachi + README docs. + +## Files changed + +Production (1): + +- `src/gps_denied_onboard/components/c11_tile_manager/tile_downloader.py` + +Tests (3): + +- `tests/unit/c11_tile_manager/test_tile_downloader.py` (rewritten; + 14 AC tests; all PASS) +- `tests/e2e/satellite_provider/__init__.py` (new) +- `tests/e2e/satellite_provider/test_smoke.py` (new; 2 tier2 tests) + +Compose / env (2): + +- `docker-compose.test.jetson.yml` +- `.env.test.example` + +Tooling (2): + +- `scripts/mint_dev_jwt.py` (new) +- `pyproject.toml` (added `pyjwt>=2.8,<3.0` to dev extras) + +Tracker docs (3): + +- `_docs/02_tasks/_dependencies_table.md` (AZ-777 5→8pt) +- `_docs/03_implementation/reviews/batch_104_review.md` (new) +- `_docs/03_implementation/batch_104_cycle3_report.md` (this file) + +## AC coverage + +| AC | Phase 1 portion satisfied? | Evidence | +|----|----------------------------|----------| +| AC-1 (compose lints; depends_on satellite-provider) | ✅ | `docker compose -f docker-compose.test.jetson.yml config` exits 0 with the new env block. | +| AC-2 unit (`_do_enumerate` POST inventory + `_download_one_tile` slippy-map GET) | ✅ | `tests/unit/c11_tile_manager/test_tile_downloader.py` 14/14 PASS. | +| AC-2 live (Bearer-authenticated round-trip vs. running service) | ⏸ | `tests/e2e/satellite_provider/test_smoke.py` is in place; runs next time the Jetson harness fires. | +| AC-3..6 | ⏳ | Out of scope (Phases 2–5). | + +## Test run results + +``` +$ python -m pytest tests/unit/c11_tile_manager/ -v --tb=short +============================== 58 passed in 3.99s ============================== + +$ python -m pytest tests/unit/runtime_root/ tests/unit/c11_tile_manager/ -v --tb=short +============================= 113 passed in 3.68s ============================== + +$ python -m pytest tests/e2e/satellite_provider/test_smoke.py -v --tb=short +============================== 2 skipped in 0.68s ============================== +(skip reason: AZ-777 satellite-provider smoke gated by RUN_REPLAY_E2E=1) +``` + +Suite-wide test run is deferred to the end of the AZ-777 +implementation phase per the iterative-skill exception in +`.cursor/rules/coderule.mdc` — Phase 1 is a batch, not the end of +implementation. The two test trees that depend on the modified code +(`tests/unit/c11_tile_manager/` and `tests/unit/runtime_root/`) are +green. + +## Code review + +See `_docs/03_implementation/reviews/batch_104_review.md` — +**verdict: PASS_WITH_WARNINGS**. Three findings (1 Medium +Architecture, 1 Medium Maintainability, 1 Low Maintainability); all +deferred to later AZ-777 phases or future tuning with clear +ownership. No Critical or High findings. + +## Risks acknowledged on this batch + +- **TLS_INSECURE not in production code path yet** — only the smoke + test honours `SATELLITE_PROVIDER_TLS_INSECURE`. Phase 3 (the real + `operator_pre_flight_setup` fixture) is the first production-ish + consumer of `HttpTileDownloader`; it MUST plumb the flag through. + Flagged as F1 in the batch review. +- **`_DEFAULT_ESTIMATED_TILE_BYTES = 50 KiB`** — conservative for + CARTO Voyager basemap; may under-reserve for UAV-uploaded tiles. + Acceptable for Phase 1; revisit in Phase 5. Flagged as F2. +- **Smoke test passes when catalog is empty** — by design; + exercises the wire pre-Phase-2 and tightens automatically once + Phase 2 seeds tiles. Flagged as F3. + +## STOP gate + +This batch closes Phase 1 of AZ-777's 5-phase plan. The next phase +(Derkachi tile catalog seed) needs operator alignment on the +imagery source (CARTO Voyager Basemap proposed in the spec) and on +the bbox / zoom-range envelope. Pause for user decision before +Phase 2. diff --git a/_docs/03_implementation/reviews/batch_104_review.md b/_docs/03_implementation/reviews/batch_104_review.md new file mode 100644 index 0000000..594fd61 --- /dev/null +++ b/_docs/03_implementation/reviews/batch_104_review.md @@ -0,0 +1,241 @@ +# Code Review Report + +**Batch**: AZ-777 Phase 1 (e2e-runner wire + C11 contract adapt + smoke test) +**Date**: 2026-05-21 +**Verdict**: PASS_WITH_WARNINGS + +## Scope + +AZ-777 is an 8-pt task with 5 explicit STOP-gated phases. This batch +delivers **Phase 1 only** — the e2e-runner wiring to the existing +parent-suite satellite-provider service + the C11 `HttpTileDownloader` +contract adaptation to the AZ-505 v1.0.0 `tile-inventory.md` API + +the Tier-2 smoke test that validates the wire. + +Phases 2–5 (catalog seed via `POST /api/satellite/request`, real +`operator_pre_flight_setup` fixture, un-xfail Tier-2 tests, docs) +are out of scope for this batch and are gated behind a STOP per the +task spec's Risk-5 mitigation. + +## Files changed + +Production (1): + +- `src/gps_denied_onboard/components/c11_tile_manager/tile_downloader.py` + — `_LIST_PATH` / `_GET_PATH` replaced with `_INVENTORY_PATH` + (`POST /api/satellite/tiles/inventory`) + `_TILES_PATH` + (`GET /tiles/{z}/{x}/{y}`); `_do_enumerate` rewritten to enumerate + bbox tile coords client-side and POST chunked inventory requests; + `_download_one_tile` re-routes to slippy-map URL; common retry / + auth logic refactored into `_send_request`; new module helpers: + `_enumerate_bbox_tile_coords`, `_tile_center_latlon`, + `_tile_size_meters_at`, `_format_tile_id_str`, `_parse_tile_id_str`, + `_chunk_iter`; new constants `_DEFAULT_ESTIMATED_TILE_BYTES` + (50 KiB, conservative tile-size estimate since inventory no longer + returns content-length hints), `_INVENTORY_MAX_ENTRIES_PER_REQUEST`, + `_EARTH_EQUATORIAL_CIRCUMFERENCE_M`, `_TILE_SIZE_PIXELS`. + +Tests (2): + +- `tests/unit/c11_tile_manager/test_tile_downloader.py` — all 14 + AC tests rewritten to drive `_make_inventory_handler` (POST + inventory + GET tile) instead of the old GET-list handler; + `_StubTileWriter` rekeyed by call-index instead of by + `(z,lat,lon)` strings (the downloader now derives lat/lon from + the slippy-map coord, so fixtures cannot fabricate arbitrary + lat/lons); `_DEFAULT_ESTIMATED_TILE_BYTES` constant mirrored. + All 14 tests PASS. +- `tests/e2e/satellite_provider/test_smoke.py` (new) — two Tier-2 + smoke tests: (i) raw `POST /api/satellite/tiles/inventory` for a + 1-tile Derkachi-bbox query, asserts the documented response + schema; (ii) drives the adapted `HttpTileDownloader` against the + real service with an in-memory C6 stub (Phase-3 fixture will + replace it with real C6). +- `tests/e2e/satellite_provider/__init__.py` (new). + +Compose / env (2): + +- `docker-compose.test.jetson.yml` — e2e-runner env block: + `SATELLITE_PROVIDER_URL` switched from `http://mock-sat:5100` to + `https://satellite-provider:8080`; `SATELLITE_PROVIDER_TLS_INSECURE=1` + added (dev-only); `SATELLITE_PROVIDER_API_KEY` sourced from + `.env.test`; `JWT_*` forwarded for in-container fallback minting; + `depends_on: { satellite-provider: { condition: service_healthy } }` + added. +- `.env.test.example` — new `SATELLITE_PROVIDER_API_KEY` variable + with documentation + dev TLS bypass security note. + +Tooling (2): + +- `scripts/mint_dev_jwt.py` (new) — HS256 dev-JWT mint helper. + Reads JWT secret / iss / aud from env or `.env.test`; emits a + signed JWT to stdout. Convenience for dev workflows; production + retrieves tokens from the admin API. +- `pyproject.toml` — added `pyjwt>=2.8,<3.0` to `[dev]` extras. + +Tracker docs (1): + +- `_docs/02_tasks/_dependencies_table.md` — AZ-777 row bumped from + 5pt to 8pt (matches the 2026-05-21 decision-log override in + `_docs/_process_leftovers/2026-05-21_az777_complexity_override.md`). + +## Phase 2 — Spec Compliance + +| AC | Status | Notes | +|----|--------|-------| +| AC-1 (`docker compose config` exits 0 with `depends_on satellite-provider`) | ✅ Verified | Compose lint passes locally with the new env block. | +| AC-2 unit half (`_do_enumerate` POST inventory + `_download_one_tile` slippy-map GET against stubbed responses) | ✅ Verified | 14/14 unit tests PASS against the new contract. | +| AC-2 live half (Bearer-authenticated round-trip against the running service) | ⏸ Deferred to Tier-2 Jetson run | Smoke test gated by `RUN_REPLAY_E2E=1` + `tier2`; auto-skips on dev macOS. | +| AC-3..6 | ⏳ Out of scope (Phases 2–5) | Phase 1 → 2 STOP gate. | + +No spec gaps within Phase 1. AC-2's live validation runs the next +time the Jetson harness fires; the test code is in place. + +## Findings + +| # | Severity | Category | File:Line | Title | +|---|----------|----------|-----------|-------| +| 1 | Medium | Architecture | `src/gps_denied_onboard/runtime_root/c11_factory.py` | TLS_INSECURE flag not plumbed through production composition root | +| 2 | Medium | Maintainability | `src/gps_denied_onboard/components/c11_tile_manager/tile_downloader.py:84` | `_DEFAULT_ESTIMATED_TILE_BYTES` is a project-wide guess, not configurable | +| 3 | Low | Maintainability | `tests/e2e/satellite_provider/test_smoke.py` | Smoke test passes when catalog is empty (Phase-2 dependency) | + +### Finding Details + +**F1: TLS_INSECURE flag not plumbed through production composition root** +(Medium / Architecture) + +- Location: `src/gps_denied_onboard/runtime_root/c11_factory.py` +- Description: `build_tile_downloader` takes a caller-owned + `httpx.Client`, so the operator binary that wires C11 is the + layer that must honour `SATELLITE_PROVIDER_TLS_INSECURE`. No + production caller exists today — `build_tile_downloader` only + has the Tier-2 smoke test as a live consumer. Phase 3 of AZ-777 + introduces the `operator_pre_flight_setup` fixture that will be + the first live caller; the TLS_INSECURE handling will land there. +- Suggestion: When Phase 3 ships, the new caller must read + `SATELLITE_PROVIDER_TLS_INSECURE` and pass the right `verify=` + to `httpx.Client(...)` — mirror the approach used in + `tests/e2e/satellite_provider/test_smoke.py::_make_http_client`. + Also consider a `WARNING` log line at startup whenever the + insecure flag is active so the operator can audit it. +- Task: AZ-777 Phase 3 (deferred) + +**F2: `_DEFAULT_ESTIMATED_TILE_BYTES` is a project-wide guess** +(Medium / Maintainability) + +- Location: `src/gps_denied_onboard/components/c11_tile_manager/tile_downloader.py:84` +- Description: The AZ-505 v1.0.0 inventory contract dropped the + per-entry `estimatedBytes` field, so the AZ-308 budget pre-check + reserves a constant 50 KiB per `present=true` tile. 50 KiB is + conservative for typical CARTO Voyager tiles (8-30 KiB) but + under-reserves for high-detail UAV uploads (30-80 KiB). The + budget can over-reserve safely; under-reserving fails the + AZ-308 contract. +- Suggestion: Either (a) add the constant to `C11Config` so + operators can tune it per imagery source, or (b) file a + parent-suite ticket to restore `estimatedBytes` in the inventory + response. For Phase 1 the constant is acceptable; revisit in + Phase 5 docs. +- Task: AZ-777 Phase 5 / future config refactor + +**F3: Smoke test passes when catalog is empty** (Low / Maintainability) + +- Location: `tests/e2e/satellite_provider/test_smoke.py:test_smoke_c11_download_via_http_pipeline` +- Description: The C11 pipeline smoke asserts SUCCESS with + `tiles_downloaded == len(write_calls)`. Pre-Phase-2 the catalog + is empty → every entry comes back `present=false` → the test + passes with zero downloads, which proves the wire works but + does NOT prove tiles actually land in C6. The conditional + `if report.tiles_downloaded > 0` block tightens the assertion + once the catalog is seeded. +- Suggestion: Accepted by design for Phase 1; Phase 2's catalog + seed automatically turns this from "wire works" into "tiles + land" without test changes. +- Task: AZ-777 Phase 2 + +## Phase 3 — Code Quality + +- **SOLID**: `_send_request` consolidates GET / POST retry + auth + in one place instead of two near-duplicates; methods stay small + (`_send_get` / `_send_post` are 5-line shims over the common + path). Slippy-map helpers are module-level pure functions — + they don't reach for `self` and don't depend on `httpx`, so + the unit tests can reuse them directly. +- **Error handling**: every failure path raises a typed C11 error + (`SatelliteProviderError`, `RateLimitedError`, + `CacheBudgetExceededError`); no bare `except`s; no silently + swallowed errors. The Retry-After parser handles both seconds- + and HTTP-date forms; OOB values clamp to 0 instead of + propagating garbage. +- **Naming**: `_inventory_path` / `_tiles_path` / `_tile_id_str` / + `_parse_tile_id_str` etc. all read directly against the AZ-505 + contract; no surprises. +- **Complexity**: `_send_request` is the longest method at ~80 LOC + but it's a linear retry ladder; cyclomatic complexity is + bounded by the four response branches (transport-error / 401-3 + / 429 / 5xx / 200). `_do_enumerate` is 14 LOC. +- **Test quality**: every AC test arranges a specific contract + scenario (POST inventory + GET tile) and asserts both the + downloader's report counts AND the C6 stub's call records. + Tests do NOT just "assert no exception". + +## Phase 4 — Security Quick-Scan + +- No SQL strings touched. +- JWT mint helper uses PyJWT's `jwt.encode` with HS256; no + hand-rolled crypto; secrets come from env, never hardcoded. +- `.env.test.example`'s `SATELLITE_PROVIDER_API_KEY` placeholder + is `PASTE-MINTED-JWT-HERE` — the smoke test treats that exact + string as "unset" and skips, so a developer accidentally + committing the placeholder cannot get false confidence. +- `Authorization` header redacted in error logs as + `Bearer ***` per AZ-316 AC-11; the AC-11 test verifies the + real API key never appears in any log record. +- `SATELLITE_PROVIDER_TLS_INSECURE` is opt-in via env var; + default is verify=True. The dev-only nature is documented in + the compose comment, in `.env.test.example`, and (when Phase 3 + lands) will be logged at startup. + +## Phase 5 — Performance Scan + +- Inventory POST chunks at 5000 entries per the contract cap; one + POST per up-to-5000-tile bbox. +- Backoff schedule unchanged (`_DEFAULT_BACKOFF_SCHEDULE_S = + (1, 2, 4, 8)`); session retry budget enforced. +- `test_nfr_throughput_1000_tiles_under_budget` passes in <1 s + locally (budget is 10 s) — no O(n²) bookkeeping regression. +- No N+1 patterns; no blocking I/O in async paths (whole module + is sync). + +## Phase 6 — Cross-Task Consistency + +Single task in this batch (AZ-777 Phase 1). N/A. + +## Phase 7 — Architecture Compliance + +- **Layer direction**: C11 still does not import C6 directly; the + Protocol cuts (`_TileWriterLike`, `_BudgetEnforcerLike`) stay + in `tile_downloader.py`. The composition root + (`runtime_root/c11_factory.py::_C6DownloadAdapter`) remains the + single bridge. +- **Public API respect**: no cross-component imports added. +- **No new cyclic deps**: no new module-level imports between + components. +- **Architecture principle #5** (`satellite-provider` owns OSM / + CARTO tile network I/O; the onboard companion is read-only via + C11 during pre-flight): this batch is the first time C11 is + actually wired to consume that contract — the principle is + honoured for the first time end-to-end. +- **ADR compliance**: ADR-004 (event-driven cross-component + comms): C11 → satellite-provider is HTTP, which is explicitly + scoped out of ADR-004 (the ADR governs intra-onboard comms, + not external-service calls). No drift. No new ADR required — + the task spec explicitly states this is execution of existing + decisions. + +## Verdict justification + +PASS_WITH_WARNINGS — three Medium / Low findings, no Critical or +High. The Medium findings are deferred to later AZ-777 phases or +to future tuning, with clear ownership; no blocking gap in Phase 1 +itself. diff --git a/_docs/_autodev_state.md b/_docs/_autodev_state.md index ff78f34..4707428 100644 --- a/_docs/_autodev_state.md +++ b/_docs/_autodev_state.md @@ -4,12 +4,12 @@ flow: existing-code step: 10 name: Implement -status: paused +status: in_progress sub_step: phase: 7 name: batch-loop - detail: "batch 104 cycle3: AZ-777 spec re-refined against codebase reality + Jira synced; Phase 1 ready (see canonical spec + 2026-05-21 decision-log addendum)" + detail: "AZ-777 Phase 1 done; STOP gate before Phase 2 (catalog seed)" retry_count: 0 cycle: 3 tracker: jira -last_completed_batch: 103 +last_completed_batch: 104 diff --git a/docker-compose.test.jetson.yml b/docker-compose.test.jetson.yml index 200b3bf..2af9401 100644 --- a/docker-compose.test.jetson.yml +++ b/docker-compose.test.jetson.yml @@ -118,6 +118,8 @@ services: depends_on: db: condition: service_healthy + satellite-provider: + condition: service_healthy environment: # Same FullSystemConfig env block as Colima — see comments in # docker-compose.test.yml for the per-var rationale. @@ -127,10 +129,21 @@ services: # execute. This is the WHOLE POINT of the Jetson harness. GPS_DENIED_TIER: "2" DB_URL: postgresql://gps_denied:dev@db:5432/gps_denied - # SATELLITE_PROVIDER_URL / COMPANION_URL are set but not used by - # the replay CLI tests (gps-denied-replay runs as a subprocess and - # does not call the companion or satellite-provider HTTP APIs). - SATELLITE_PROVIDER_URL: http://mock-sat:5100 + # AZ-777 Phase 1: e2e-runner consumes the real parent-suite + # satellite-provider .NET service over its compose-DNS name. The + # dev TLS cert is self-signed against `localhost`, so the suite- + # internal probe must skip cert verification — see SECURITY note + # in `.env.test.example`. Production deploys ship a real CA-issued + # cert and MUST set SATELLITE_PROVIDER_TLS_INSECURE="0" (or omit it). + SATELLITE_PROVIDER_URL: https://satellite-provider:8080 + SATELLITE_PROVIDER_TLS_INSECURE: "1" + SATELLITE_PROVIDER_API_KEY: ${SATELLITE_PROVIDER_API_KEY:?SATELLITE_PROVIDER_API_KEY must be set via .env.test — see scripts/mint_dev_jwt.py} + # AZ-777 Phase 1 also forwards the JWT triple so the smoke test + # can mint its own dev token in-container as a fallback when + # SATELLITE_PROVIDER_API_KEY is rotated mid-session. + JWT_SECRET: ${JWT_SECRET} + JWT_ISSUER: ${JWT_ISSUER} + JWT_AUDIENCE: ${JWT_AUDIENCE} COMPANION_URL: http://companion:8080 CAMERA_CALIBRATION_PATH: /opt/tests/fixtures/calibration/adti26.json LOG_LEVEL: INFO diff --git a/pyproject.toml b/pyproject.toml index 24d8d91..9d421a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,6 +101,10 @@ dev = [ "mypy>=1.8", "types-PyYAML", "types-requests", + # AZ-777: mint Bearer JWTs for the satellite-provider Jetson e2e smoke + # test. Test-only because the production C11 path receives a token + # minted by the admin API (AZ-690) — never mints its own. + "pyjwt>=2.8,<3.0", # AZ-406 (blackbox harness internals): the mock-suite-sat-service unit # test exercises a FastAPI app via fastapi.testclient.TestClient. The # production runtime of the mock lives inside its own Docker image so diff --git a/scripts/mint_dev_jwt.py b/scripts/mint_dev_jwt.py new file mode 100644 index 0000000..bcdf5eb --- /dev/null +++ b/scripts/mint_dev_jwt.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Mint a dev JWT for the parent-suite satellite-provider (AZ-777). + +Reads JWT_SECRET / JWT_ISSUER / JWT_AUDIENCE from environment or from +`.env.test` (when run from the repo root). Prints the JWT to stdout so +the caller can pipe / paste / `export` it. + +DEV-ONLY: the same secret signs the JWT and validates it on the +provider side; production deploys retrieve operator JWTs from the admin +API (AZ-690) instead. Mirrors `SatelliteProvider.TestSupport.JwtTokenFactory.Create` +on the .NET side so dev tokens behave identically to integration-test ones. + +Usage:: + + python scripts/mint_dev_jwt.py + python scripts/mint_dev_jwt.py --lifetime-hours 12 --subject e2e-runner + export SATELLITE_PROVIDER_API_KEY="$(python scripts/mint_dev_jwt.py)" +""" + +from __future__ import annotations + +import argparse +import os +import sys +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path + +try: + import jwt +except ImportError as exc: + sys.stderr.write( + "ERROR: pyjwt not installed. Run `pip install pyjwt>=2.8,<3.0`\n" + "(or `pip install -e .[dev]` from the repo root) and retry.\n" + f"Underlying ImportError: {exc}\n" + ) + sys.exit(72) + + +_MIN_SECRET_BYTES = 32 + + +def _load_env_file(path: Path) -> dict[str, str]: + """Parse a minimal KEY=VALUE env file. Honours quoting; ignores comments.""" + + if not path.is_file(): + return {} + out: dict[str, str] = {} + for raw in path.read_text("utf-8").splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + if "=" not in line: + continue + key, _, value = line.partition("=") + key = key.strip() + value = value.strip().strip('"').strip("'") + if key: + out[key] = value + return out + + +def _resolve(name: str, env_file_values: dict[str, str]) -> str | None: + value = os.environ.get(name) + if value: + return value + return env_file_values.get(name) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--lifetime-hours", + type=float, + default=8.0, + help="Token lifetime in hours (default: 8).", + ) + parser.add_argument( + "--subject", + default="gps-denied-onboard-e2e", + help="`sub` claim (default: gps-denied-onboard-e2e).", + ) + parser.add_argument( + "--env-file", + default=".env.test", + help="Fallback env file (default: .env.test in CWD).", + ) + args = parser.parse_args() + + env_file_values = _load_env_file(Path(args.env_file)) + secret = _resolve("JWT_SECRET", env_file_values) + issuer = _resolve("JWT_ISSUER", env_file_values) + audience = _resolve("JWT_AUDIENCE", env_file_values) + + missing = [ + name + for name, value in [ + ("JWT_SECRET", secret), + ("JWT_ISSUER", issuer), + ("JWT_AUDIENCE", audience), + ] + if not value + ] + if missing: + sys.stderr.write( + "ERROR: required env var(s) not set: " + + ", ".join(missing) + + f"\n(looked at environment + {args.env_file})\n" + ) + return 73 + + assert secret is not None + if len(secret.encode("utf-8")) < _MIN_SECRET_BYTES: + sys.stderr.write( + f"ERROR: JWT_SECRET is {len(secret)} bytes; HMAC-SHA256 requires " + f">= {_MIN_SECRET_BYTES} bytes per the provider's contract.\n" + ) + return 74 + + now = datetime.now(timezone.utc) + payload = { + "sub": args.subject, + "iss": issuer, + "aud": audience, + "jti": uuid.uuid4().hex, + "iat": int(now.timestamp()), + "nbf": int(now.timestamp()), + "exp": int((now + timedelta(hours=args.lifetime_hours)).timestamp()), + } + + token = jwt.encode(payload, secret, algorithm="HS256") + sys.stdout.write(token + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/gps_denied_onboard/components/c11_tile_manager/tile_downloader.py b/src/gps_denied_onboard/components/c11_tile_manager/tile_downloader.py index a408b1d..cbbf1bc 100644 --- a/src/gps_denied_onboard/components/c11_tile_manager/tile_downloader.py +++ b/src/gps_denied_onboard/components/c11_tile_manager/tile_downloader.py @@ -1,11 +1,23 @@ -"""C11 ``HttpTileDownloader`` (AZ-316) — concrete :class:`TileDownloader`. +"""C11 ``HttpTileDownloader`` (AZ-316, AZ-777) — concrete :class:`TileDownloader`. -Operator-side pre-flight download path. Authenticated GETs against -``satellite-provider``, RESTRICT-SAT-4 enforcement at the C11 boundary, -c6 writes via the AZ-303 store + metadata Protocols (which run AZ-307's -freshness gate at insert), AZ-308 cache-headroom pre-check before any -GET fires, and a per-``(flight_id, request_hash)`` journal for -idempotent re-runs. +Operator-side pre-flight download path. Authenticated POST inventory +lookups + slippy-map GETs against ``satellite-provider``, RESTRICT-SAT-4 +enforcement at the C11 boundary, c6 writes via the AZ-303 store + +metadata Protocols (which run AZ-307's freshness gate at insert), +AZ-308 cache-headroom pre-check before any GET fires (using a +configured per-tile bytes estimate, since the inventory contract does +not return content-length hints), and a per-``(flight_id, +request_hash)`` journal for idempotent re-runs. + +Contract surface (AZ-777, against the parent-suite +``satellite-provider`` v1.0.0 inventory contract — see +``../satellite-provider/_docs/02_document/contracts/api/tile-inventory.md``): + +* ``POST /api/satellite/tiles/inventory`` — bulk lookup of (z,x,y) tile + coords; returns one entry per request item with ``present: true|false`` + and (when present) the metadata C11 needs to drive the resolution gate + and the c6 write. +* ``GET /tiles/{z}/{x}/{y}`` — slippy-map tile fetch by coords. Architecture ------------ @@ -26,6 +38,7 @@ from __future__ import annotations import hashlib import json import logging +import math import os import tempfile from dataclasses import dataclass, field @@ -50,6 +63,7 @@ from gps_denied_onboard.components.c11_tile_manager.errors import ( RateLimitedError, SatelliteProviderError, ) +from gps_denied_onboard.helpers.wgs_converter import WgsConverter __all__ = [ "DOWNLOAD_JOURNAL_DIRNAME", @@ -58,9 +72,24 @@ __all__ = [ ] -_LIST_PATH = "/api/satellite/tiles" -_GET_PATH = "/api/satellite/tiles" -_LIST_QUERY_LIST_ONLY = "list-only" +# AZ-777: parent-suite contract v1.0.0 (see module docstring). +_INVENTORY_PATH = "/api/satellite/tiles/inventory" +_TILES_PATH = "/tiles" +_INVENTORY_MAX_ENTRIES_PER_REQUEST = 5000 +# Inventory response does not carry a content-length hint, but the +# AZ-308 budget pre-check needs a per-tile estimate before any GET +# fires. 50 KiB is conservative for 256x256 JPEG basemap tiles +# (typical CARTO Voyager tiles run 8-30 KiB; UAV-captured uploads +# run 30-80 KiB). Sized to over-reserve rather than under-reserve. +_DEFAULT_ESTIMATED_TILE_BYTES = 50_000 +# Web-Mercator at zoom 0 covers the full equatorial circumference of +# the WGS-84 ellipsoid (≈40 075 016.686 m). Tile ground size at any +# (zoom, lat) follows: circumference * cos(lat_rad) / 2^zoom (the +# cos(lat) factor is the same projection-stretch correction the +# parent-suite uses to compute ``resolutionMPerPx``). +_EARTH_EQUATORIAL_CIRCUMFERENCE_M = 40_075_016.686 +_TILE_SIZE_PIXELS = 256 + DOWNLOAD_JOURNAL_DIRNAME = ".c11/journal" _LOCKFILE_PATH = ".c11/lock" _DEFAULT_BACKOFF_SCHEDULE_S: tuple[float, ...] = (1.0, 2.0, 4.0, 8.0) @@ -295,6 +324,93 @@ def _default_sleep(seconds: float) -> None: clock.sleep_until_ns(clock.monotonic_ns() + int(seconds * 1_000_000_000)) +# ---------------------------------------------------------------------- +# AZ-777 slippy-map helpers +# +# The parent-suite inventory contract (v1.0.0) is keyed by explicit +# (z, x, y) slippy-map coords. C11 enumerates the grid from the bbox +# locally and converts inventory hits back into lat/lon for c6 writes. +# Math matches the parent suite's Web-Mercator projection so the +# resolution / tile-size hints round-trip identically. +# ---------------------------------------------------------------------- + + +def _enumerate_bbox_tile_coords( + bbox_min_lat: float, + bbox_min_lon: float, + bbox_max_lat: float, + bbox_max_lon: float, + zoom_levels: tuple[int, ...], +) -> tuple[tuple[int, int, int], ...]: + """Return every (z, x, y) whose tile bounds intersect the bbox. + + Slippy-map y grows southward, so the SW corner has (low x, high y) + and the NE corner has (high x, low y). The enumeration is inclusive + on both ends. + """ + + coords: list[tuple[int, int, int]] = [] + for zoom in zoom_levels: + x_sw, y_sw = WgsConverter.latlon_to_tile_xy( + int(zoom), bbox_min_lat, bbox_min_lon + ) + x_ne, y_ne = WgsConverter.latlon_to_tile_xy( + int(zoom), bbox_max_lat, bbox_max_lon + ) + x_lo, x_hi = (x_sw, x_ne) if x_sw <= x_ne else (x_ne, x_sw) + y_lo, y_hi = (y_ne, y_sw) if y_ne <= y_sw else (y_sw, y_ne) + for x in range(x_lo, x_hi + 1): + for y in range(y_lo, y_hi + 1): + coords.append((int(zoom), x, y)) + return tuple(coords) + + +def _tile_center_latlon(zoom: int, x: int, y: int) -> tuple[float, float]: + bounds = WgsConverter.tile_xy_to_latlon_bounds(int(zoom), int(x), int(y)) + lat = (bounds.min_lat_deg + bounds.max_lat_deg) / 2.0 + lon = (bounds.min_lon_deg + bounds.max_lon_deg) / 2.0 + return lat, lon + + +def _tile_size_meters_at(zoom: int, lat_deg: float) -> float: + return ( + _EARTH_EQUATORIAL_CIRCUMFERENCE_M + * math.cos(math.radians(lat_deg)) + / (1 << int(zoom)) + ) + + +def _format_tile_id_str(zoom: int, x: int, y: int) -> str: + return f"{int(zoom)}_{int(x)}_{int(y)}" + + +def _parse_tile_id_str(tile_id_str: str) -> tuple[int, int, int]: + parts = tile_id_str.split("_") + if len(parts) != 3: + raise ValueError( + f"tile_id_str must be 'z_x_y'; got {tile_id_str!r}" + ) + try: + return int(parts[0]), int(parts[1]), int(parts[2]) + except ValueError as exc: + raise ValueError( + f"tile_id_str must contain three integers separated by '_'; " + f"got {tile_id_str!r}" + ) from exc + + +def _chunk_iter( + seq: tuple[tuple[int, int, int], ...], + chunk_size: int, +) -> list[tuple[tuple[int, int, int], ...]]: + if chunk_size <= 0: + raise ValueError(f"chunk_size must be > 0; got {chunk_size}") + return [ + tuple(seq[start : start + chunk_size]) + for start in range(0, len(seq), chunk_size) + ] + + # ---------------------------------------------------------------------- # Internal session-state container # ---------------------------------------------------------------------- @@ -546,58 +662,119 @@ class HttpTileDownloader: bbox_max_lon: float, zoom_levels: tuple[int, ...], ) -> list[TileSummary]: - params = { - "bbox": f"{bbox_min_lat},{bbox_min_lon},{bbox_max_lat},{bbox_max_lon}", - "zoom": ",".join(str(z) for z in zoom_levels), - _LIST_QUERY_LIST_ONLY: "true", + """POST ``/api/satellite/tiles/inventory`` for every (z,x,y) in bbox. + + AZ-777: the satellite-provider v1.0.0 inventory contract is + keyed by explicit slippy-map coords, NOT by a server-side + bbox query. This method enumerates the tile grid for the + bbox × zoom set, chunks into ≤5000-entry POSTs (the + ``TileInventoryLimits.MaxEntriesPerRequest`` cap), and + returns one :class:`TileSummary` per ``present=true`` entry. + Absent tiles are silently dropped — they need to be seeded + via ``POST /api/satellite/request`` upstream before they + become downloadable. + """ + + tile_coords = _enumerate_bbox_tile_coords( + bbox_min_lat, + bbox_min_lon, + bbox_max_lat, + bbox_max_lon, + zoom_levels, + ) + if not tile_coords: + return [] + + summaries: list[TileSummary] = [] + for chunk in _chunk_iter(tile_coords, _INVENTORY_MAX_ENTRIES_PER_REQUEST): + summaries.extend(self._fetch_inventory_chunk(chunk)) + return summaries + + def _fetch_inventory_chunk( + self, chunk: tuple[tuple[int, int, int], ...] + ) -> list[TileSummary]: + body = { + "tiles": [ + {"tileZoom": z, "tileX": x, "tileY": y} + for (z, x, y) in chunk + ] } - response = self._send_get( - self._config.satellite_provider_url.rstrip("/") + _LIST_PATH, - params=params, + response = self._send_post( + self._config.satellite_provider_url.rstrip("/") + _INVENTORY_PATH, + json_body=body, session=None, ) try: - body = response.json() + decoded = response.json() except ValueError as exc: self._log_provider_failure( - "list_not_json", response.status_code, str(exc) + "inventory_not_json", response.status_code, str(exc) ) raise SatelliteProviderError( - "satellite-provider returned non-JSON list-only body" + "satellite-provider returned non-JSON inventory body" ) from exc try: - entries = body["tiles"] + entries = decoded["results"] except (KeyError, TypeError) as exc: self._log_provider_failure( - "list_schema", response.status_code, str(exc) + "inventory_schema", response.status_code, str(exc) ) raise SatelliteProviderError( - "satellite-provider list-only response missing 'tiles'" + "satellite-provider inventory response missing 'results'" ) from exc + if len(entries) != len(chunk): + self._log_provider_failure( + "inventory_order", + response.status_code, + f"results.len={len(entries)} request.tiles.len={len(chunk)}", + ) + raise SatelliteProviderError( + f"satellite-provider inventory response broke order invariant: " + f"len(results)={len(entries)} != len(request.tiles)={len(chunk)}" + ) summaries: list[TileSummary] = [] for entry in entries: try: - summaries.append( - TileSummary( - tile_id_str=str(entry["tile_id"]), - zoom_level=int(entry["zoom_level"]), - lat=float(entry["lat"]), - lon=float(entry["lon"]), - produced_at=_parse_iso(str(entry["produced_at"])), - resolution_m_per_px=float(entry["resolution_m_per_px"]), - estimated_bytes=int(entry["estimated_bytes"]), - tile_size_meters=float(entry.get("tile_size_meters", 100.0)), - tile_size_pixels=int(entry.get("tile_size_pixels", 256)), - ) - ) - except (KeyError, TypeError, ValueError) as exc: + present = bool(entry["present"]) + except (KeyError, TypeError) as exc: self._log_provider_failure( - "list_tile_schema", response.status_code, str(exc) + "inventory_entry_schema", response.status_code, str(exc) ) raise SatelliteProviderError( - "satellite-provider list-only entry missing required fields" + "satellite-provider inventory entry missing 'present'" ) from exc + if not present: + continue + try: + zoom = int(entry["tileZoom"]) + x = int(entry["tileX"]) + y = int(entry["tileY"]) + produced_at = _parse_iso(str(entry["capturedAt"])) + resolution_m_per_px = float(entry["resolutionMPerPx"]) + except (KeyError, TypeError, ValueError) as exc: + self._log_provider_failure( + "inventory_entry_schema", response.status_code, str(exc) + ) + raise SatelliteProviderError( + "satellite-provider inventory present-entry missing " + "required fields (tileZoom/tileX/tileY/capturedAt/" + "resolutionMPerPx)" + ) from exc + lat, lon = _tile_center_latlon(zoom, x, y) + summaries.append( + TileSummary( + tile_id_str=_format_tile_id_str(zoom, x, y), + zoom_level=zoom, + lat=lat, + lon=lon, + produced_at=produced_at, + resolution_m_per_px=resolution_m_per_px, + estimated_bytes=_DEFAULT_ESTIMATED_TILE_BYTES, + tile_size_meters=_tile_size_meters_at(zoom, lat), + tile_size_pixels=_TILE_SIZE_PIXELS, + ) + ) return summaries def _reserve_budget( @@ -648,10 +825,16 @@ class HttpTileDownloader: ) return + try: + zoom, x, y = _parse_tile_id_str(summary.tile_id_str) + except ValueError as exc: + raise SatelliteProviderError( + f"internal: TileSummary.tile_id_str does not match the AZ-777 " + f"z_x_y format (got {summary.tile_id_str!r})" + ) from exc ingest_url = ( self._config.satellite_provider_url.rstrip("/") - + _GET_PATH - + f"/{summary.tile_id_str}" + + f"{_TILES_PATH}/{zoom}/{x}/{y}" ) response = self._send_get(ingest_url, params=None, session=session) if not response.content: @@ -717,15 +900,44 @@ class HttpTileDownloader: ) -> httpx.Response: """GET with auth header + 429 / 5xx handling.""" + return self._send_request( + "GET", url, params=params, json_body=None, session=session + ) + + def _send_post( + self, + url: str, + json_body: Any, + session: _DownloadSession | None, + ) -> httpx.Response: + """POST with auth header + 429 / 5xx handling (AZ-777 inventory contract).""" + + return self._send_request( + "POST", url, params=None, json_body=json_body, session=session + ) + + def _send_request( + self, + method: str, + url: str, + *, + params: dict[str, str] | None, + json_body: Any, + session: _DownloadSession | None, + ) -> httpx.Response: + """Auth header + 429 / 5xx handling for GET and POST.""" + headers = {"Authorization": f"Bearer {self._config.service_api_key}"} attempt = 0 last_error: str | None = None while True: attempt += 1 try: - response = self._http_client.get( + response = self._http_client.request( + method, url, params=params, + json=json_body, headers=headers, timeout=self._config.download_http_timeout_s, ) diff --git a/tests/e2e/satellite_provider/__init__.py b/tests/e2e/satellite_provider/__init__.py new file mode 100644 index 0000000..3f4b121 --- /dev/null +++ b/tests/e2e/satellite_provider/__init__.py @@ -0,0 +1,8 @@ +"""AZ-777 Phase 1 smoke tests against the parent-suite satellite-provider. + +Tier-2 only: these tests assume the real `.NET` `satellite-provider` +service (and its Postgres) are running in the Jetson e2e compose graph +(`docker-compose.test.jetson.yml`). Each test is gated by +`RUN_REPLAY_E2E=1` (the env contract the rest of `tests/e2e/replay/` +already uses) and `@pytest.mark.tier2` so dev-laptop pytest runs auto-skip. +""" diff --git a/tests/e2e/satellite_provider/test_smoke.py b/tests/e2e/satellite_provider/test_smoke.py new file mode 100644 index 0000000..55b20e5 --- /dev/null +++ b/tests/e2e/satellite_provider/test_smoke.py @@ -0,0 +1,325 @@ +"""AZ-777 Phase 1 smoke test against the real parent-suite satellite-provider. + +Validates the wire C11 just plumbed through against the .NET service +running inside `docker-compose.test.jetson.yml`: + +* TLS handshake works against the self-signed dev cert via + ``SATELLITE_PROVIDER_TLS_INSECURE=1`` (development-only override — + production deploys MUST use a CA-issued cert and unset the flag). +* Bearer JWT minted from ``JWT_SECRET`` / ``JWT_ISSUER`` / + ``JWT_AUDIENCE`` is accepted. +* ``POST /api/satellite/tiles/inventory`` (AZ-505) round-trips the + documented v1.0.0 schema for a 1-tile Derkachi-bbox query: response + is 200 with a ``results`` array of length == request.tiles.length + (Inv-2 from ``tile-inventory.md``) and per-entry shape matches the + contract regardless of seed state. +* The adapted C11 :class:`HttpTileDownloader` (Phase 1a) drives the + same wire against the real service. When the catalog is unseeded + (pre-Phase-2) the report is ``SUCCESS`` with zero downloads (every + entry comes back ``present=false``); when seeded, the stubbed C6 + ``write_tile_for_download`` receives one call per present tile and + the report counts match. + +Phase 2 (Derkachi catalog seed via ``POST /api/satellite/request``) +turns the second test from a "wire works" check into a "tiles actually +land" check; the assertions here are written so both states (seeded / +unseeded) pass cleanly so the smoke runs green as soon as Phase 1 +lands and stays green through Phase 2. +""" + +from __future__ import annotations + +import logging +import os +import ssl +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +import httpx +import pytest + +from gps_denied_onboard.components.c11_tile_manager import ( + C11Config, + DownloadOutcome, + DownloadRequest, + HttpTileDownloader, + SectorClassification, +) +from gps_denied_onboard.helpers.wgs_converter import WgsConverter + + +# ---------------------------------------------------------------------- +# Skip gates +# ---------------------------------------------------------------------- + + +def _heavy_skip_reason() -> str | None: + if os.environ.get("RUN_REPLAY_E2E", "").lower() not in {"1", "true", "yes", "on"}: + return "AZ-777 satellite-provider smoke gated by RUN_REPLAY_E2E=1" + return None + + +_HEAVY_SKIP = pytest.mark.skipif( + _heavy_skip_reason() is not None, + reason=_heavy_skip_reason() or "ok", +) + + +_INVENTORY_PATH = "/api/satellite/tiles/inventory" +# Derkachi bbox center used for the 1-tile inventory query. Zoom 15 is +# the upper bound of the AZ-777 Phase 2 catalog (zooms 15-18); picking +# the lowest zoom keeps the per-tile ground footprint largest, which +# makes a 1-tile query cover the most catalog. +_DERKACHI_TILE_ZOOM = 15 +_DERKACHI_LAT = 50.10 +_DERKACHI_LON = 36.10 +# Tight bbox that surrounds the Derkachi tile center so the C11 bbox +# enumeration produces exactly one (z,x,y) coord at zoom 15. +_DERKACHI_BBOX = (50.099, 36.099, 50.101, 36.101) + + +# ---------------------------------------------------------------------- +# Helpers +# ---------------------------------------------------------------------- + + +def _mint_bearer_token_or_skip() -> str: + """Return a valid Bearer JWT or `pytest.skip` with an actionable reason.""" + + token = os.environ.get("SATELLITE_PROVIDER_API_KEY", "").strip() + if token and token != "PASTE-MINTED-JWT-HERE": + return token + try: + import jwt # type: ignore[import-untyped] + except ImportError: + pytest.skip( + "SATELLITE_PROVIDER_API_KEY is unset and PyJWT is not " + "installed in-container; install dev extras " + "(`pip install -e .[dev]`) or set " + "SATELLITE_PROVIDER_API_KEY via .env.test." + ) + secret = os.environ.get("JWT_SECRET", "").strip() + issuer = os.environ.get("JWT_ISSUER", "").strip() + audience = os.environ.get("JWT_AUDIENCE", "").strip() + missing = [ + name + for name, value in [ + ("JWT_SECRET", secret), + ("JWT_ISSUER", issuer), + ("JWT_AUDIENCE", audience), + ] + if not value + ] + if missing: + pytest.skip( + "Cannot mint a fallback JWT — missing env: " + + ", ".join(missing) + ) + now = datetime.now(timezone.utc) + payload = { + "sub": "gps-denied-onboard-smoke", + "iss": issuer, + "aud": audience, + "jti": uuid.uuid4().hex, + "iat": int(now.timestamp()), + "nbf": int(now.timestamp()), + "exp": int((now + timedelta(hours=1)).timestamp()), + } + return str(jwt.encode(payload, secret, algorithm="HS256")) + + +def _make_http_client(base_url: str) -> httpx.Client: + """Return an httpx.Client honouring SATELLITE_PROVIDER_TLS_INSECURE.""" + + insecure_raw = os.environ.get("SATELLITE_PROVIDER_TLS_INSECURE", "").strip() + insecure = insecure_raw.lower() in {"1", "true", "yes", "on"} + verify: bool | ssl.SSLContext + if insecure: + # Documented dev-only path; the audit-trail WARNING lives in + # `.env.test.example`. We do NOT silently disable verification + # unless the env var explicitly opts in. + verify = False + else: + verify = True + return httpx.Client(base_url=base_url, verify=verify, timeout=30.0) + + +def _resolve_satellite_provider_url() -> str: + url = os.environ.get("SATELLITE_PROVIDER_URL", "").strip() + if not url: + pytest.skip( + "SATELLITE_PROVIDER_URL is not set — the Jetson e2e compose " + "env block provides https://satellite-provider:8080; running " + "this smoke outside compose requires an explicit override." + ) + return url + + +# ---------------------------------------------------------------------- +# Test 1 — inventory POST contract (AZ-505 v1.0.0) +# ---------------------------------------------------------------------- + + +@pytest.mark.tier2 +@_HEAVY_SKIP +def test_smoke_satellite_provider_inventory_contract() -> None: + # Arrange + base_url = _resolve_satellite_provider_url() + bearer = _mint_bearer_token_or_skip() + tile_x, tile_y = WgsConverter.latlon_to_tile_xy( + _DERKACHI_TILE_ZOOM, _DERKACHI_LAT, _DERKACHI_LON + ) + body = { + "tiles": [ + { + "tileZoom": _DERKACHI_TILE_ZOOM, + "tileX": tile_x, + "tileY": tile_y, + } + ] + } + + # Act + with _make_http_client(base_url) as client: + response = client.post( + _INVENTORY_PATH, + json=body, + headers={"Authorization": f"Bearer {bearer}"}, + ) + + # Assert — contract invariants from `tile-inventory.md` v1.0.0 + assert response.status_code == 200, ( + f"satellite-provider inventory POST returned " + f"{response.status_code}: {response.text!r}" + ) + decoded = response.json() + assert isinstance(decoded, dict), f"expected JSON object, got {type(decoded)}" + assert "results" in decoded, f"missing 'results' key in {decoded!r}" + results = decoded["results"] + assert isinstance(results, list) + assert len(results) == len(body["tiles"]), ( + # Inv-2: response order/length matches request order/length. + f"inventory response length {len(results)} != request length " + f"{len(body['tiles'])}" + ) + entry = results[0] + assert entry["tileZoom"] == _DERKACHI_TILE_ZOOM + assert entry["tileX"] == tile_x + assert entry["tileY"] == tile_y + assert "present" in entry, f"missing 'present' in entry {entry!r}" + assert isinstance(entry["present"], bool) + if entry["present"]: + for required in ("id", "capturedAt", "source", "resolutionMPerPx"): + assert entry.get(required) is not None, ( + f"present entry missing required field {required!r}: {entry!r}" + ) + + +# ---------------------------------------------------------------------- +# Test 2 — C11 HttpTileDownloader against the real service +# ---------------------------------------------------------------------- + + +class _InMemoryC6Adapter: + """Implements `_TileWriterLike` + `_BudgetEnforcerLike` in-memory. + + The Phase 1 smoke does NOT exercise the real C6 store (that's + Phase 3 of AZ-777). It exercises the C11 wire end-to-end with a + stub C6 so the test stays scope-clean. + """ + + def __init__(self) -> None: + self.write_calls: list[dict[str, Any]] = [] + self.reserved_bytes: list[int] = [] + self.exists_calls: list[tuple[int, float, float]] = [] + + def write_tile_for_download( + self, + *, + tile_blob: bytes, + zoom_level: int, + lat: float, + lon: float, + tile_size_meters: float, + tile_size_pixels: int, + capture_timestamp: datetime, + content_sha256_hex: str, + sector_class: str, + ) -> str: + self.write_calls.append( + { + "zoom_level": zoom_level, + "lat": lat, + "lon": lon, + "tile_blob_len": len(tile_blob), + "sector_class": sector_class, + } + ) + return "fresh" + + def tile_already_present( + self, *, zoom_level: int, lat: float, lon: float + ) -> bool: + self.exists_calls.append((zoom_level, lat, lon)) + return False + + def reserve_headroom(self, needed_bytes: int) -> object: + self.reserved_bytes.append(int(needed_bytes)) + return object() + + +@pytest.mark.tier2 +@_HEAVY_SKIP +def test_smoke_c11_download_via_http_pipeline(tmp_path: Path) -> None: + # Arrange + base_url = _resolve_satellite_provider_url() + bearer = _mint_bearer_token_or_skip() + adapter = _InMemoryC6Adapter() + cfg = C11Config( + satellite_provider_url=base_url, + service_api_key=bearer, + download_http_timeout_s=30.0, + download_max_5xx_retries=2, + download_max_retry_after_s=60, + download_resolution_floor_m_per_px=0.5, + ) + logger = logging.getLogger("test_az777_smoke") + with _make_http_client(base_url) as http_client: + downloader = HttpTileDownloader( + http_client=http_client, + tile_writer=adapter, # type: ignore[arg-type] + budget_enforcer=adapter, # type: ignore[arg-type] + logger=logger, + config=cfg, + ) + request = DownloadRequest( + flight_id=uuid.uuid4(), + bbox_min_lat=_DERKACHI_BBOX[0], + bbox_min_lon=_DERKACHI_BBOX[1], + bbox_max_lat=_DERKACHI_BBOX[2], + bbox_max_lon=_DERKACHI_BBOX[3], + zoom_levels=(_DERKACHI_TILE_ZOOM,), + sector_class=SectorClassification.STABLE_REAR, + cache_root=tmp_path, + ) + + # Act + report = downloader.download_tiles_for_area(request) + + # Assert — Phase 1 smoke contract: the wire works regardless of + # catalog state. Pre-Phase-2 the catalog is empty → 0 downloads. + # Post-Phase-2 the catalog is seeded → ≥ 1 download. Both are valid + # outcomes for this test; the test fails only on a wire / contract + # break (any provider error would have raised before reaching this + # assertion block). + assert report.outcome == DownloadOutcome.SUCCESS + assert report.tiles_requested >= 0 + assert report.tiles_downloaded == len(adapter.write_calls) + assert report.tiles_rejected_resolution >= 0 + if report.tiles_downloaded > 0: + # Phase 2 catalog landed: verify the write actually carried the + # tile bytes we'd expect from a real GET /tiles/{z}/{x}/{y}. + assert all(call["tile_blob_len"] > 0 for call in adapter.write_calls) + assert all(call["zoom_level"] == _DERKACHI_TILE_ZOOM for call in adapter.write_calls) diff --git a/tests/unit/c11_tile_manager/test_tile_downloader.py b/tests/unit/c11_tile_manager/test_tile_downloader.py index 5ee1898..f1ad7c1 100644 --- a/tests/unit/c11_tile_manager/test_tile_downloader.py +++ b/tests/unit/c11_tile_manager/test_tile_downloader.py @@ -1,16 +1,21 @@ -"""AZ-316 ``HttpTileDownloader`` unit tests. +"""AZ-316 / AZ-777 ``HttpTileDownloader`` unit tests. Covers AC-1 .. AC-12 plus the throughput NFR from -``_docs/02_tasks/todo/AZ-316_c11_tile_downloader.md``. Uses -:class:`httpx.MockTransport` for deterministic HTTP responses, a -list-backed log handler for log capture, and stub C6 stores so this -suite never depends on AZ-303 / AZ-305 / AZ-307 / AZ-308 internals. +``_docs/02_tasks/todo/AZ-316_c11_tile_downloader.md``, against the +AZ-777 contract update (POST ``/api/satellite/tiles/inventory`` for +bulk lookup + GET ``/tiles/{z}/{x}/{y}`` for body download — see +``../satellite-provider/_docs/02_document/contracts/api/tile-inventory.md`` +v1.0.0). Uses :class:`httpx.MockTransport` for deterministic HTTP +responses, a list-backed log handler for log capture, and stub C6 +stores so this suite never depends on AZ-303 / AZ-305 / AZ-307 / +AZ-308 internals. """ from __future__ import annotations import json import logging +import math from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any @@ -33,8 +38,14 @@ from gps_denied_onboard.components.c11_tile_manager import ( _BASE_URL = "https://parent-suite.test" -_LIST_PATH = "/api/satellite/tiles" +_INVENTORY_PATH = "/api/satellite/tiles/inventory" +_TILES_PATH_PREFIX = "/tiles/" _API_KEY = "test-api-key-001" +# Mirror of c11.tile_downloader._DEFAULT_ESTIMATED_TILE_BYTES (AZ-777): +# the inventory contract no longer returns content-length hints, so the +# AZ-308 budget pre-check reserves this constant per `present=true` +# tile. Keep in sync with the production module. +_DEFAULT_ESTIMATED_TILE_BYTES = 50_000 # ---------------------------------------------------------------------- @@ -55,16 +66,22 @@ _StubFreshnessRejection.__name__ = "FreshnessRejectionError" class _StubTileWriter: - """Captures `write_tile_for_download` calls + scripts the freshness label.""" + """Captures `write_tile_for_download` calls + scripts the freshness label. + + AZ-777: keyed by *call index* rather than `(z, lat, lon)` strings. + The downloader now derives (lat, lon) from the slippy-map coord, so + tests can no longer fabricate arbitrary lat/lons in their fixtures; + the call-order is the contract. + """ def __init__( self, *, - labels: dict[str, str] | None = None, - rejected: set[str] | None = None, + labels_by_index: dict[int, str] | None = None, + rejected_indices: set[int] | None = None, ) -> None: - self.labels = labels or {} - self.rejected = rejected or set() + self.labels_by_index = labels_by_index or {} + self.rejected_indices = rejected_indices or set() self.write_calls: list[dict[str, Any]] = [] self.exists_calls: list[tuple[int, float, float]] = [] @@ -81,18 +98,23 @@ class _StubTileWriter: content_sha256_hex: str, sector_class: str, ) -> str: - tid = _tid(zoom_level, lat, lon) + call_index = len(self.write_calls) self.write_calls.append( { - "tile_id": tid, + "call_index": call_index, + "zoom_level": zoom_level, + "lat": lat, + "lon": lon, "tile_blob_len": len(tile_blob), "content_sha256_hex": content_sha256_hex, "sector_class": sector_class, } ) - if tid in self.rejected: - raise _StubFreshnessRejection(f"freshness rejected {tid}") - return self.labels.get(tid, "fresh") + if call_index in self.rejected_indices: + raise _StubFreshnessRejection( + f"freshness rejected call_index={call_index}" + ) + return self.labels_by_index.get(call_index, "fresh") def tile_already_present( self, *, zoom_level: int, lat: float, lon: float @@ -115,10 +137,6 @@ class _StubBudgetEnforcer: return object() -def _tid(zoom: int, lat: float, lon: float) -> str: - return f"z{int(zoom)}_{float(lat):.6f}_{float(lon):.6f}" - - def _build_downloader( *, transport: httpx.MockTransport, @@ -172,71 +190,135 @@ def _build_downloader( return downloader, log_records, writer, enforcer, sleeps +_DEFAULT_TEST_BBOX = (45.0, -122.5, 45.5, -122.0) + + def _make_request( *, flight_id: Any | None = None, cache_root: Path, zoom_levels: tuple[int, ...] = (14,), + bbox: tuple[float, float, float, float] = _DEFAULT_TEST_BBOX, ) -> DownloadRequest: return DownloadRequest( flight_id=flight_id or uuid4(), - bbox_min_lat=45.0, - bbox_min_lon=-122.5, - bbox_max_lat=45.5, - bbox_max_lon=-122.0, + bbox_min_lat=bbox[0], + bbox_min_lon=bbox[1], + bbox_max_lat=bbox[2], + bbox_max_lon=bbox[3], zoom_levels=zoom_levels, sector_class=SectorClassification.STABLE_REAR, cache_root=cache_root, ) -def _list_response( - tiles: list[dict[str, Any]] | None = None, -) -> httpx.Response: - return httpx.Response(200, json={"tiles": tiles or []}) +def _tile_center_latlon_from_zxy(zoom: int, x: int, y: int) -> tuple[float, float]: + """Mirror of c11.tile_downloader._tile_center_latlon for test assertions. + + Both sides compute lat/lon from the slippy-map (z, x, y) tuple, so + stub freshness/label maps can be keyed on the *expected* lat/lon + without depending on the production helper at import time. + """ + n = 1 << int(zoom) + lat_n = math.degrees(math.atan(math.sinh(math.pi * (1.0 - 2.0 * int(y) / n)))) + lat_s = math.degrees(math.atan(math.sinh(math.pi * (1.0 - 2.0 * (int(y) + 1) / n)))) + lon_w = int(x) / n * 360.0 - 180.0 + lon_e = (int(x) + 1) / n * 360.0 - 180.0 + return (lat_n + lat_s) / 2.0, (lon_w + lon_e) / 2.0 -def _tile_entry( +def _inventory_entry_for_coord( *, zoom: int, - lat: float, - lon: float, + x: int, + y: int, + present: bool = True, resolution_m_per_px: float = 0.5, - estimated_bytes: int = 4096, - produced_at: datetime | None = None, + captured_at: datetime | None = None, ) -> dict[str, Any]: - produced = produced_at or datetime(2026, 5, 13, 0, 0, 0, tzinfo=timezone.utc) + if not present: + return { + "tileZoom": int(zoom), + "tileX": int(x), + "tileY": int(y), + "locationHash": str(uuid4()), + "present": False, + "id": None, + "capturedAt": None, + "source": None, + "flightId": None, + "resolutionMPerPx": None, + } + captured = captured_at or datetime(2026, 5, 13, 0, 0, 0, tzinfo=timezone.utc) return { - "tile_id": _tid(zoom, lat, lon), - "zoom_level": zoom, - "lat": lat, - "lon": lon, - "produced_at": produced.isoformat(), - "resolution_m_per_px": resolution_m_per_px, - "estimated_bytes": estimated_bytes, - "tile_size_meters": 100.0, - "tile_size_pixels": 256, + "tileZoom": int(zoom), + "tileX": int(x), + "tileY": int(y), + "locationHash": str(uuid4()), + "present": True, + "id": str(uuid4()), + "capturedAt": captured.isoformat(), + "source": "google_maps", + "flightId": None, + "resolutionMPerPx": float(resolution_m_per_px), } -def _make_route_handler( +def _make_inventory_handler( *, - list_response: httpx.Response | None = None, + present_count: int | None = None, + resolution_override_for_first_n: tuple[int, float] | None = None, tile_response_factory: Any = None, + inventory_response_override: httpx.Response | None = None, ) -> Any: - """Route GETs by URL path: list endpoint vs per-tile endpoint.""" + """Route requests by method+path: POST inventory vs GET /tiles/{z}/{x}/{y}. + + The inventory handler echoes the request's tile coords in + response order (per contract invariant Inv-2 / Inv-3). The + `present_count` knob marks the FIRST N entries as + ``present=true`` and the rest as ``present=false``; ``None`` means + "all present". `resolution_override_for_first_n=(K, RES)` overrides + the resolution of the first K present entries (used by the AZ-316 + resolution-gate test). + """ def _handler(request: httpx.Request) -> httpx.Response: path = request.url.path - is_list = ( - path.endswith(_LIST_PATH) - and request.url.params.get("list-only") == "true" + method = request.method.upper() + if method == "POST" and path.endswith(_INVENTORY_PATH): + if inventory_response_override is not None: + return inventory_response_override + body = json.loads(request.content.decode("utf-8")) + tiles_in = body["tiles"] + results: list[dict[str, Any]] = [] + for i, t in enumerate(tiles_in): + is_present = present_count is None or i < int(present_count) + if ( + is_present + and resolution_override_for_first_n is not None + and i < int(resolution_override_for_first_n[0]) + ): + resolution = float(resolution_override_for_first_n[1]) + else: + resolution = 0.5 + results.append( + _inventory_entry_for_coord( + zoom=int(t["tileZoom"]), + x=int(t["tileX"]), + y=int(t["tileY"]), + present=is_present, + resolution_m_per_px=resolution, + ) + ) + return httpx.Response(200, json={"results": results}) + if method == "GET" and path.startswith(_TILES_PATH_PREFIX): + if tile_response_factory is None: + return httpx.Response(200, content=b"\xff\xd8\xff\xe0fake-jpeg") + return tile_response_factory(request) + return httpx.Response( + 404, + json={"detail": f"test handler: unexpected {method} {path}"}, ) - if is_list: - return list_response or _list_response() - if tile_response_factory is None: - return httpx.Response(200, content=b"\xff\xd8\xff\xe0fake-jpeg") - return tile_response_factory(request) return _handler @@ -247,13 +329,10 @@ def _make_route_handler( def test_ac1_100_tile_happy_path_writes_all(tmp_path: Path) -> None: - # Arrange - tiles = [ - _tile_entry(zoom=14, lat=45.0 + i * 0.001, lon=-122.0 - i * 0.001) - for i in range(100) - ] + # Arrange — bbox at zoom 14 produces N coord candidates; the + # stub marks the first 100 as `present=true` and the rest absent. transport = httpx.MockTransport( - _make_route_handler(list_response=_list_response(tiles)) + _make_inventory_handler(present_count=100) ) (downloader, _logs, writer, enforcer, _sleeps) = _build_downloader( transport=transport @@ -271,7 +350,7 @@ def test_ac1_100_tile_happy_path_writes_all(tmp_path: Path) -> None: assert report.tiles_rejected_freshness == 0 assert report.tiles_downgraded == 0 assert len(writer.write_calls) == 100 - assert enforcer.calls == [4096 * 100] + assert enforcer.calls == [_DEFAULT_ESTIMATED_TILE_BYTES * 100] # ---------------------------------------------------------------------- @@ -280,15 +359,12 @@ def test_ac1_100_tile_happy_path_writes_all(tmp_path: Path) -> None: def test_ac2_resolution_gate_rejects_sub_spec_tiles(tmp_path: Path) -> None: - # Arrange - tiles = [] - for i in range(50): - res = 0.3 if i < 10 else 0.5 - tiles.append( - _tile_entry(zoom=14, lat=45.0 + i * 0.001, lon=-122.0, resolution_m_per_px=res) - ) + # Arrange — 50 present tiles; first 10 below the 0.5 m/px floor. transport = httpx.MockTransport( - _make_route_handler(list_response=_list_response(tiles)) + _make_inventory_handler( + present_count=50, + resolution_override_for_first_n=(10, 0.3), + ) ) (downloader, log_records, writer, _enforcer, _sleeps) = _build_downloader( transport=transport @@ -311,13 +387,9 @@ def test_ac2_resolution_gate_rejects_sub_spec_tiles(tmp_path: Path) -> None: def test_ac3_freshness_rejections_counted_and_run_continues(tmp_path: Path) -> None: - # Arrange - tiles = [_tile_entry(zoom=14, lat=45.0 + i * 0.001, lon=-122.0) for i in range(10)] - rejected_ids = {_tid(14, 45.0 + i * 0.001, -122.0) for i in range(5)} - transport = httpx.MockTransport( - _make_route_handler(list_response=_list_response(tiles)) - ) - writer = _StubTileWriter(rejected=rejected_ids) + # Arrange — 10 present tiles; c6 rejects the first 5 writes. + transport = httpx.MockTransport(_make_inventory_handler(present_count=10)) + writer = _StubTileWriter(rejected_indices={0, 1, 2, 3, 4}) (downloader, log_records, _writer, _enforcer, _sleeps) = _build_downloader( transport=transport, tile_writer=writer ) @@ -341,13 +413,9 @@ def test_ac3_freshness_rejections_counted_and_run_continues(tmp_path: Path) -> N def test_ac4_downgraded_label_counted_and_persisted(tmp_path: Path) -> None: - # Arrange - tiles = [_tile_entry(zoom=14, lat=45.0 + i * 0.001, lon=-122.0) for i in range(5)] - labels = {_tid(14, 45.0 + i * 0.001, -122.0): "downgraded" for i in range(3)} - transport = httpx.MockTransport( - _make_route_handler(list_response=_list_response(tiles)) - ) - writer = _StubTileWriter(labels=labels) + # Arrange — 5 present tiles; c6 returns "downgraded" for the first 3 writes. + transport = httpx.MockTransport(_make_inventory_handler(present_count=5)) + writer = _StubTileWriter(labels_by_index={0: "downgraded", 1: "downgraded", 2: "downgraded"}) (downloader, _logs, _writer, _enforcer, _sleeps) = _build_downloader( transport=transport, tile_writer=writer ) @@ -366,8 +434,7 @@ def test_ac4_downgraded_label_counted_and_persisted(tmp_path: Path) -> None: def test_ac5_429_honours_retry_after(tmp_path: Path) -> None: - # Arrange - tiles = [_tile_entry(zoom=14, lat=45.0, lon=-122.0)] + # Arrange — single present tile; tile GET returns 429 once then 200. state = {"attempts": 0} def _factory(request: httpx.Request) -> httpx.Response: @@ -377,8 +444,8 @@ def test_ac5_429_honours_retry_after(tmp_path: Path) -> None: return httpx.Response(200, content=b"\xff\xd8tile") transport = httpx.MockTransport( - _make_route_handler( - list_response=_list_response(tiles), + _make_inventory_handler( + present_count=1, tile_response_factory=_factory, ) ) @@ -403,8 +470,7 @@ def test_ac5_429_honours_retry_after(tmp_path: Path) -> None: def test_ac6_persistent_5xx_raises_satellite_provider_error(tmp_path: Path) -> None: - # Arrange - tiles = [_tile_entry(zoom=14, lat=45.0, lon=-122.0)] + # Arrange — single present tile; tile GET always 503. state = {"attempts": 0} def _factory(request: httpx.Request) -> httpx.Response: @@ -412,8 +478,8 @@ def test_ac6_persistent_5xx_raises_satellite_provider_error(tmp_path: Path) -> N return httpx.Response(503) transport = httpx.MockTransport( - _make_route_handler( - list_response=_list_response(tiles), + _make_inventory_handler( + present_count=1, tile_response_factory=_factory, ) ) @@ -433,8 +499,7 @@ def test_ac6_persistent_5xx_raises_satellite_provider_error(tmp_path: Path) -> N def test_ac7_401_fails_fast_no_retry(tmp_path: Path) -> None: - # Arrange - tiles = [_tile_entry(zoom=14, lat=45.0, lon=-122.0)] + # Arrange — single present tile; tile GET returns 401. state = {"attempts": 0} def _factory(request: httpx.Request) -> httpx.Response: @@ -442,8 +507,8 @@ def test_ac7_401_fails_fast_no_retry(tmp_path: Path) -> None: return httpx.Response(401) transport = httpx.MockTransport( - _make_route_handler( - list_response=_list_response(tiles), + _make_inventory_handler( + present_count=1, tile_response_factory=_factory, ) ) @@ -463,8 +528,7 @@ def test_ac7_401_fails_fast_no_retry(tmp_path: Path) -> None: def test_ac8_idempotent_rerun_yields_no_op(tmp_path: Path) -> None: - # Arrange - tiles = [_tile_entry(zoom=14, lat=45.0 + i * 0.001, lon=-122.0) for i in range(5)] + # Arrange — 5 present tiles; both runs reach the same handler. state = {"tile_gets": 0} def _factory(request: httpx.Request) -> httpx.Response: @@ -472,8 +536,8 @@ def test_ac8_idempotent_rerun_yields_no_op(tmp_path: Path) -> None: return httpx.Response(200, content=b"\xff\xd8tile") transport = httpx.MockTransport( - _make_route_handler( - list_response=_list_response(tiles), + _make_inventory_handler( + present_count=5, tile_response_factory=_factory, ) ) @@ -501,8 +565,7 @@ def test_ac8_idempotent_rerun_yields_no_op(tmp_path: Path) -> None: def test_ac9_cache_budget_pre_check_aborts(tmp_path: Path) -> None: - # Arrange - tiles = [_tile_entry(zoom=14, lat=45.0, lon=-122.0, estimated_bytes=10_000)] + # Arrange — single present tile; budget enforcer rejects up-front. transport_state = {"tile_gets": 0} def _factory(request: httpx.Request) -> httpx.Response: @@ -510,8 +573,8 @@ def test_ac9_cache_budget_pre_check_aborts(tmp_path: Path) -> None: return httpx.Response(200, content=b"\xff\xd8tile") transport = httpx.MockTransport( - _make_route_handler( - list_response=_list_response(tiles), + _make_inventory_handler( + present_count=1, tile_response_factory=_factory, ) ) @@ -526,7 +589,7 @@ def test_ac9_cache_budget_pre_check_aborts(tmp_path: Path) -> None: with pytest.raises(CacheBudgetExceededError): downloader.download_tiles_for_area(_make_request(cache_root=tmp_path)) assert transport_state["tile_gets"] == 0 - assert enforcer.calls == [10_000] + assert enforcer.calls == [_DEFAULT_ESTIMATED_TILE_BYTES] # ---------------------------------------------------------------------- @@ -536,16 +599,11 @@ def test_ac9_cache_budget_pre_check_aborts(tmp_path: Path) -> None: def test_ac11_service_api_key_never_appears_in_logs(tmp_path: Path) -> None: # Arrange — exercise the failure path so the provider-failed ERROR - # log fires (the code that explicitly redacts the auth header). - tiles = [_tile_entry(zoom=14, lat=45.0, lon=-122.0)] - - def _factory(request: httpx.Request) -> httpx.Response: - return httpx.Response(401) - + # log fires (the code that explicitly redacts the auth header). The + # inventory POST returns 401 → fast-fail before any tile GET. transport = httpx.MockTransport( - _make_route_handler( - list_response=_list_response(tiles), - tile_response_factory=_factory, + _make_inventory_handler( + inventory_response_override=httpx.Response(401), ) ) (downloader, log_records, _writer, _enforcer, _sleeps) = _build_downloader( @@ -568,12 +626,10 @@ def test_ac11_service_api_key_never_appears_in_logs(tmp_path: Path) -> None: def test_ac12_partial_journal_resumed_on_rerun(tmp_path: Path) -> None: - # Arrange — 10 tiles; first run fetches all 10 successfully and - # leaves a complete journal. A second run with the SAME request - # must short-circuit (AC-8 covers that). To exercise AC-12 we - # MANUALLY truncate the journal between runs to simulate a crash - # AFTER 4 tile-writes, BEFORE the completed_at_iso stamp. - tiles = [_tile_entry(zoom=14, lat=45.0 + i * 0.001, lon=-122.0) for i in range(10)] + # Arrange — 10 present tiles; first run completes, then we truncate + # the journal to simulate a crash after 4 writes (clear + # `completed_at_iso`) so the second run must complete the remaining + # 6 tiles instead of short-circuiting on AC-8 idempotence. state = {"tile_gets": 0} def _factory(request: httpx.Request) -> httpx.Response: @@ -581,8 +637,8 @@ def test_ac12_partial_journal_resumed_on_rerun(tmp_path: Path) -> None: return httpx.Response(200, content=b"\xff\xd8tile") transport = httpx.MockTransport( - _make_route_handler( - list_response=_list_response(tiles), + _make_inventory_handler( + present_count=10, tile_response_factory=_factory, ) ) @@ -637,8 +693,7 @@ def test_ac12_partial_journal_resumed_on_rerun(tmp_path: Path) -> None: def test_429_retry_after_http_date_form_parses(tmp_path: Path) -> None: - # Arrange - tiles = [_tile_entry(zoom=14, lat=45.0, lon=-122.0)] + # Arrange — single present tile; first tile GET 429 with HTTP-date Retry-After. state = {"attempts": 0} future = (datetime.now(timezone.utc) + timedelta(seconds=20)).strftime( "%a, %d %b %Y %H:%M:%S GMT" @@ -651,8 +706,8 @@ def test_429_retry_after_http_date_form_parses(tmp_path: Path) -> None: return httpx.Response(200, content=b"\xff\xd8tile") transport = httpx.MockTransport( - _make_route_handler( - list_response=_list_response(tiles), + _make_inventory_handler( + present_count=1, tile_response_factory=_factory, ) ) @@ -676,15 +731,15 @@ def test_429_retry_after_http_date_form_parses(tmp_path: Path) -> None: def test_429_budget_exhaustion_raises_rate_limited_error(tmp_path: Path) -> None: - # Arrange - tiles = [_tile_entry(zoom=14, lat=45.0, lon=-122.0)] - + # Arrange — single present tile; tile GET always 429 with very long + # Retry-After. After the configured retry budget is exhausted the + # client raises RateLimitedError. def _factory(request: httpx.Request) -> httpx.Response: return httpx.Response(429, headers={"Retry-After": "300"}) transport = httpx.MockTransport( - _make_route_handler( - list_response=_list_response(tiles), + _make_inventory_handler( + present_count=1, tile_response_factory=_factory, ) ) @@ -711,25 +766,31 @@ def test_429_budget_exhaustion_raises_rate_limited_error(tmp_path: Path) -> None def test_nfr_throughput_1000_tiles_under_budget(tmp_path: Path) -> None: - # Arrange - tiles = [ - _tile_entry(zoom=14, lat=45.0 + i * 0.0001, lon=-122.0 + i * 0.0001) - for i in range(1000) - ] + # Arrange — 1000 present tiles. Use a bbox wide enough to enumerate + # ≥1000 tile coords at zoom 14 (≈0.022° per tile at ~zoom 14). transport = httpx.MockTransport( - _make_route_handler( - list_response=_list_response(tiles), - tile_response_factory=lambda r: httpx.Response(200, content=b"\xff\xd8tile"), + _make_inventory_handler( + present_count=1000, + tile_response_factory=lambda r: httpx.Response( + 200, content=b"\xff\xd8tile" + ), ) ) (downloader, _logs, _writer, _enforcer, _sleeps) = _build_downloader( transport=transport ) + # Big bbox at zoom 14: ~ 32x32 tile span on this latitude is enough. + # 1° ≈ 45 tiles at zoom 14 in latitude → 0.75° gives ≈ 33 tiles → ~1089 tiles. + request = _make_request( + cache_root=tmp_path, + bbox=(44.0, -123.0, 44.75, -122.25), + ) + import time as _time t0 = _time.perf_counter() - report = downloader.download_tiles_for_area(_make_request(cache_root=tmp_path)) + report = downloader.download_tiles_for_area(request) elapsed = _time.perf_counter() - t0 # Assert — budget is generous; the goal is to catch an O(n^2)