"""AZ-316 ``HttpTileDownloader`` unit tests. Covers AC-1 .. AC-12 plus the throughput NFR from ``_docs/02_tasks/todo/AZ-316_c11_tile_downloader.md``. Uses :class:`httpx.MockTransport` for deterministic HTTP responses, a list-backed log handler for log capture, and stub C6 stores so this suite never depends on AZ-303 / AZ-305 / AZ-307 / AZ-308 internals. """ from __future__ import annotations import json import logging from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any from uuid import uuid4 import httpx import pytest from gps_denied_onboard.components.c11_tile_manager import ( C11Config, CacheBudgetExceededError, DownloadOutcome, DownloadRequest, HttpTileDownloader, RateLimitedError, SatelliteProviderError, SectorClassification, request_hash, ) _BASE_URL = "https://parent-suite.test" _LIST_PATH = "/api/satellite/tiles" _API_KEY = "test-api-key-001" # ---------------------------------------------------------------------- # Stubs / fakes # ---------------------------------------------------------------------- class _StubFreshnessRejection(Exception): """Composition-root would surface c6's ``FreshnessRejectionError``; we mirror it locally so the structural check in the downloader matches by class name without importing c6. """ # Re-name the class so the structural check (`__class__.__name__ == # "FreshnessRejectionError"`) inside the downloader matches. _StubFreshnessRejection.__name__ = "FreshnessRejectionError" class _StubTileWriter: """Captures `write_tile_for_download` calls + scripts the freshness label.""" def __init__( self, *, labels: dict[str, str] | None = None, rejected: set[str] | None = None, ) -> None: self.labels = labels or {} self.rejected = rejected or set() self.write_calls: list[dict[str, Any]] = [] self.exists_calls: list[tuple[int, float, float]] = [] def write_tile_for_download( self, *, tile_blob: bytes, zoom_level: int, lat: float, lon: float, tile_size_meters: float, tile_size_pixels: int, capture_timestamp: datetime, content_sha256_hex: str, sector_class: str, ) -> str: tid = _tid(zoom_level, lat, lon) self.write_calls.append( { "tile_id": tid, "tile_blob_len": len(tile_blob), "content_sha256_hex": content_sha256_hex, "sector_class": sector_class, } ) if tid in self.rejected: raise _StubFreshnessRejection(f"freshness rejected {tid}") return self.labels.get(tid, "fresh") def tile_already_present( self, *, zoom_level: int, lat: float, lon: float ) -> bool: self.exists_calls.append((zoom_level, lat, lon)) return False class _StubBudgetEnforcer: """Records `reserve_headroom` calls; raises pre-baked exception when set.""" def __init__(self, raise_on_call: Exception | None = None) -> None: self.calls: list[int] = [] self._raise = raise_on_call def reserve_headroom(self, needed_bytes: int) -> object: self.calls.append(int(needed_bytes)) if self._raise is not None: raise self._raise return object() def _tid(zoom: int, lat: float, lon: float) -> str: return f"z{int(zoom)}_{float(lat):.6f}_{float(lon):.6f}" def _build_downloader( *, transport: httpx.MockTransport, tile_writer: _StubTileWriter | None = None, budget_enforcer: _StubBudgetEnforcer | None = None, config: C11Config | None = None, sleep_recorder: list[float] | None = None, backoff_schedule_s: tuple[float, ...] | None = None, ) -> tuple[ HttpTileDownloader, list[logging.LogRecord], _StubTileWriter, _StubBudgetEnforcer, list[float], ]: log_records: list[logging.LogRecord] = [] class _Handler(logging.Handler): def emit(self, record: logging.LogRecord) -> None: log_records.append(record) logger = logging.getLogger(f"test_az316_{id(log_records)}") logger.handlers.clear() logger.addHandler(_Handler()) logger.setLevel(logging.DEBUG) logger.propagate = False writer = tile_writer or _StubTileWriter() enforcer = budget_enforcer or _StubBudgetEnforcer() sleeps = sleep_recorder if sleep_recorder is not None else [] cfg = config or C11Config( satellite_provider_url=_BASE_URL, service_api_key=_API_KEY, download_http_timeout_s=5.0, download_max_5xx_retries=4, download_max_retry_after_s=600, download_resolution_floor_m_per_px=0.5, ) client = httpx.Client(transport=transport, base_url=_BASE_URL) downloader = HttpTileDownloader( http_client=client, tile_writer=writer, # type: ignore[arg-type] budget_enforcer=enforcer, # type: ignore[arg-type] logger=logger, config=cfg, sleep=lambda s: sleeps.append(s), backoff_schedule_s=backoff_schedule_s, ) return downloader, log_records, writer, enforcer, sleeps def _make_request( *, flight_id: Any | None = None, cache_root: Path, zoom_levels: tuple[int, ...] = (14,), ) -> DownloadRequest: return DownloadRequest( flight_id=flight_id or uuid4(), bbox_min_lat=45.0, bbox_min_lon=-122.5, bbox_max_lat=45.5, bbox_max_lon=-122.0, zoom_levels=zoom_levels, sector_class=SectorClassification.STABLE_REAR, cache_root=cache_root, ) def _list_response( tiles: list[dict[str, Any]] | None = None, ) -> httpx.Response: return httpx.Response(200, json={"tiles": tiles or []}) def _tile_entry( *, zoom: int, lat: float, lon: float, resolution_m_per_px: float = 0.5, estimated_bytes: int = 4096, produced_at: datetime | None = None, ) -> dict[str, Any]: produced = produced_at or datetime(2026, 5, 13, 0, 0, 0, tzinfo=timezone.utc) return { "tile_id": _tid(zoom, lat, lon), "zoom_level": zoom, "lat": lat, "lon": lon, "produced_at": produced.isoformat(), "resolution_m_per_px": resolution_m_per_px, "estimated_bytes": estimated_bytes, "tile_size_meters": 100.0, "tile_size_pixels": 256, } def _make_route_handler( *, list_response: httpx.Response | None = None, tile_response_factory: Any = None, ) -> Any: """Route GETs by URL path: list endpoint vs per-tile endpoint.""" def _handler(request: httpx.Request) -> httpx.Response: path = request.url.path is_list = ( path.endswith(_LIST_PATH) and request.url.params.get("list-only") == "true" ) if is_list: return list_response or _list_response() if tile_response_factory is None: return httpx.Response(200, content=b"\xff\xd8\xff\xe0fake-jpeg") return tile_response_factory(request) return _handler # ---------------------------------------------------------------------- # AC-1: 100-tile happy path # ---------------------------------------------------------------------- def test_ac1_100_tile_happy_path_writes_all(tmp_path: Path) -> None: # Arrange tiles = [ _tile_entry(zoom=14, lat=45.0 + i * 0.001, lon=-122.0 - i * 0.001) for i in range(100) ] transport = httpx.MockTransport( _make_route_handler(list_response=_list_response(tiles)) ) (downloader, _logs, writer, enforcer, _sleeps) = _build_downloader( transport=transport ) request = _make_request(cache_root=tmp_path) # Act report = downloader.download_tiles_for_area(request) # Assert assert report.outcome == DownloadOutcome.SUCCESS assert report.tiles_requested == 100 assert report.tiles_downloaded == 100 assert report.tiles_rejected_resolution == 0 assert report.tiles_rejected_freshness == 0 assert report.tiles_downgraded == 0 assert len(writer.write_calls) == 100 assert enforcer.calls == [4096 * 100] # ---------------------------------------------------------------------- # AC-2: resolution gate rejects sub-spec tiles BEFORE write # ---------------------------------------------------------------------- def test_ac2_resolution_gate_rejects_sub_spec_tiles(tmp_path: Path) -> None: # Arrange tiles = [] for i in range(50): res = 0.3 if i < 10 else 0.5 tiles.append( _tile_entry(zoom=14, lat=45.0 + i * 0.001, lon=-122.0, resolution_m_per_px=res) ) transport = httpx.MockTransport( _make_route_handler(list_response=_list_response(tiles)) ) (downloader, log_records, writer, _enforcer, _sleeps) = _build_downloader( transport=transport ) # Act report = downloader.download_tiles_for_area(_make_request(cache_root=tmp_path)) # Assert assert report.tiles_rejected_resolution == 10 assert report.tiles_downloaded == 40 assert len(writer.write_calls) == 40 res_warnings = [r for r in log_records if getattr(r, "kind", "") == "c11.download.resolution_rejected"] assert len(res_warnings) == 10 # ---------------------------------------------------------------------- # AC-3: c6 freshness rejection counted, not propagated # ---------------------------------------------------------------------- def test_ac3_freshness_rejections_counted_and_run_continues(tmp_path: Path) -> None: # Arrange tiles = [_tile_entry(zoom=14, lat=45.0 + i * 0.001, lon=-122.0) for i in range(10)] rejected_ids = {_tid(14, 45.0 + i * 0.001, -122.0) for i in range(5)} transport = httpx.MockTransport( _make_route_handler(list_response=_list_response(tiles)) ) writer = _StubTileWriter(rejected=rejected_ids) (downloader, log_records, _writer, _enforcer, _sleeps) = _build_downloader( transport=transport, tile_writer=writer ) # Act report = downloader.download_tiles_for_area(_make_request(cache_root=tmp_path)) # Assert assert report.tiles_rejected_freshness == 5 assert report.tiles_downloaded == 5 assert report.outcome == DownloadOutcome.SUCCESS summary_warns = [ r for r in log_records if getattr(r, "kind", "") == "c11.download.freshness_rejected_summary" ] assert len(summary_warns) == 1 # ---------------------------------------------------------------------- # AC-4: stable_rear stale tiles are surfaced as DOWNGRADED # ---------------------------------------------------------------------- def test_ac4_downgraded_label_counted_and_persisted(tmp_path: Path) -> None: # Arrange tiles = [_tile_entry(zoom=14, lat=45.0 + i * 0.001, lon=-122.0) for i in range(5)] labels = {_tid(14, 45.0 + i * 0.001, -122.0): "downgraded" for i in range(3)} transport = httpx.MockTransport( _make_route_handler(list_response=_list_response(tiles)) ) writer = _StubTileWriter(labels=labels) (downloader, _logs, _writer, _enforcer, _sleeps) = _build_downloader( transport=transport, tile_writer=writer ) # Act report = downloader.download_tiles_for_area(_make_request(cache_root=tmp_path)) # Assert assert report.tiles_downgraded == 3 assert report.tiles_downloaded == 5 # ---------------------------------------------------------------------- # AC-5: 429 honours Retry-After # ---------------------------------------------------------------------- def test_ac5_429_honours_retry_after(tmp_path: Path) -> None: # Arrange tiles = [_tile_entry(zoom=14, lat=45.0, lon=-122.0)] state = {"attempts": 0} def _factory(request: httpx.Request) -> httpx.Response: state["attempts"] += 1 if state["attempts"] == 1: return httpx.Response(429, headers={"Retry-After": "30"}) return httpx.Response(200, content=b"\xff\xd8tile") transport = httpx.MockTransport( _make_route_handler( list_response=_list_response(tiles), tile_response_factory=_factory, ) ) sleeps: list[float] = [] (downloader, _logs, _writer, _enforcer, _sleeps) = _build_downloader( transport=transport, sleep_recorder=sleeps ) # Act report = downloader.download_tiles_for_area(_make_request(cache_root=tmp_path)) # Assert assert state["attempts"] == 2 assert sleeps and sleeps[0] >= 30 assert report.retry_count >= 1 assert report.outcome == DownloadOutcome.SUCCESS # ---------------------------------------------------------------------- # AC-6: persistent 5xx aborts with structured error # ---------------------------------------------------------------------- def test_ac6_persistent_5xx_raises_satellite_provider_error(tmp_path: Path) -> None: # Arrange tiles = [_tile_entry(zoom=14, lat=45.0, lon=-122.0)] state = {"attempts": 0} def _factory(request: httpx.Request) -> httpx.Response: state["attempts"] += 1 return httpx.Response(503) transport = httpx.MockTransport( _make_route_handler( list_response=_list_response(tiles), tile_response_factory=_factory, ) ) (downloader, _logs, _writer, _enforcer, _sleeps) = _build_downloader( transport=transport ) # Act / Assert with pytest.raises(SatelliteProviderError): downloader.download_tiles_for_area(_make_request(cache_root=tmp_path)) assert state["attempts"] >= 5 # ---------------------------------------------------------------------- # AC-7: 401 fails fast (no retry) # ---------------------------------------------------------------------- def test_ac7_401_fails_fast_no_retry(tmp_path: Path) -> None: # Arrange tiles = [_tile_entry(zoom=14, lat=45.0, lon=-122.0)] state = {"attempts": 0} def _factory(request: httpx.Request) -> httpx.Response: state["attempts"] += 1 return httpx.Response(401) transport = httpx.MockTransport( _make_route_handler( list_response=_list_response(tiles), tile_response_factory=_factory, ) ) (downloader, log_records, _writer, _enforcer, _sleeps) = _build_downloader( transport=transport ) # Act / Assert with pytest.raises(SatelliteProviderError): downloader.download_tiles_for_area(_make_request(cache_root=tmp_path)) assert state["attempts"] == 1 # ---------------------------------------------------------------------- # AC-8: idempotent re-run after success # ---------------------------------------------------------------------- def test_ac8_idempotent_rerun_yields_no_op(tmp_path: Path) -> None: # Arrange tiles = [_tile_entry(zoom=14, lat=45.0 + i * 0.001, lon=-122.0) for i in range(5)] state = {"tile_gets": 0} def _factory(request: httpx.Request) -> httpx.Response: state["tile_gets"] += 1 return httpx.Response(200, content=b"\xff\xd8tile") transport = httpx.MockTransport( _make_route_handler( list_response=_list_response(tiles), tile_response_factory=_factory, ) ) (downloader, _logs, _writer, _enforcer, _sleeps) = _build_downloader( transport=transport ) request = _make_request(cache_root=tmp_path) # Act — first run first = downloader.download_tiles_for_area(request) first_get_count = state["tile_gets"] # Act — second run (same request, same cache_root → idempotent) second = downloader.download_tiles_for_area(request) # Assert assert first.outcome == DownloadOutcome.SUCCESS assert second.outcome == DownloadOutcome.IDEMPOTENT_NO_OP assert state["tile_gets"] == first_get_count, "second run must NOT fetch" assert second.tiles_downloaded == first.tiles_downloaded # ---------------------------------------------------------------------- # AC-9: cache-budget pre-check aborts before any GET # ---------------------------------------------------------------------- def test_ac9_cache_budget_pre_check_aborts(tmp_path: Path) -> None: # Arrange tiles = [_tile_entry(zoom=14, lat=45.0, lon=-122.0, estimated_bytes=10_000)] transport_state = {"tile_gets": 0} def _factory(request: httpx.Request) -> httpx.Response: transport_state["tile_gets"] += 1 return httpx.Response(200, content=b"\xff\xd8tile") transport = httpx.MockTransport( _make_route_handler( list_response=_list_response(tiles), tile_response_factory=_factory, ) ) enforcer = _StubBudgetEnforcer( raise_on_call=CacheBudgetExceededError("no headroom") ) (downloader, log_records, _writer, _enforcer, _sleeps) = _build_downloader( transport=transport, budget_enforcer=enforcer ) # Act / Assert with pytest.raises(CacheBudgetExceededError): downloader.download_tiles_for_area(_make_request(cache_root=tmp_path)) assert transport_state["tile_gets"] == 0 assert enforcer.calls == [10_000] # ---------------------------------------------------------------------- # AC-11: service API key never logged in plaintext # ---------------------------------------------------------------------- def test_ac11_service_api_key_never_appears_in_logs(tmp_path: Path) -> None: # Arrange — exercise the failure path so the provider-failed ERROR # log fires (the code that explicitly redacts the auth header). tiles = [_tile_entry(zoom=14, lat=45.0, lon=-122.0)] def _factory(request: httpx.Request) -> httpx.Response: return httpx.Response(401) transport = httpx.MockTransport( _make_route_handler( list_response=_list_response(tiles), tile_response_factory=_factory, ) ) (downloader, log_records, _writer, _enforcer, _sleeps) = _build_downloader( transport=transport ) with pytest.raises(SatelliteProviderError): downloader.download_tiles_for_area(_make_request(cache_root=tmp_path)) # Assert flat = " ".join( r.getMessage() + json.dumps(getattr(r, "kv", {})) for r in log_records ) assert _API_KEY not in flat assert "Bearer ***" in flat # ---------------------------------------------------------------------- # AC-12: journal survives mid-batch crash; re-run completes the rest # ---------------------------------------------------------------------- def test_ac12_partial_journal_resumed_on_rerun(tmp_path: Path) -> None: # Arrange — 10 tiles; first run fetches all 10 successfully and # leaves a complete journal. A second run with the SAME request # must short-circuit (AC-8 covers that). To exercise AC-12 we # MANUALLY truncate the journal between runs to simulate a crash # AFTER 4 tile-writes, BEFORE the completed_at_iso stamp. tiles = [_tile_entry(zoom=14, lat=45.0 + i * 0.001, lon=-122.0) for i in range(10)] state = {"tile_gets": 0} def _factory(request: httpx.Request) -> httpx.Response: state["tile_gets"] += 1 return httpx.Response(200, content=b"\xff\xd8tile") transport = httpx.MockTransport( _make_route_handler( list_response=_list_response(tiles), tile_response_factory=_factory, ) ) (downloader, _logs, writer, _enforcer, _sleeps) = _build_downloader( transport=transport ) request = _make_request(cache_root=tmp_path) # First run — completes downloader.download_tiles_for_area(request) after_first = state["tile_gets"] write_calls_after_first = len(writer.write_calls) assert after_first == 10 assert write_calls_after_first == 10 # Simulate crash: rewrite the journal as "completed 4 of 10 tiles, # NOT yet completed" (clear `completed_at_iso`). rh = request_hash( request.flight_id, request.bbox_min_lat, request.bbox_min_lon, request.bbox_max_lat, request.bbox_max_lon, tuple(request.zoom_levels), request.sector_class, _API_KEY, ) journal_path = tmp_path / ".c11/journal" / f"{request.flight_id}__{rh}.json" raw = json.loads(journal_path.read_text("utf-8")) raw["completed_at_iso"] = None raw["tile_ids_completed"] = sorted(raw["tile_ids_completed"])[:4] raw["tile_counts"]["tiles_downloaded"] = 4 journal_path.write_text(json.dumps(raw), encoding="utf-8") # Reset transport counter (writer still records cumulative calls) state["tile_gets"] = 0 writer.write_calls.clear() # Act — second run must fetch only the missing 6 second = downloader.download_tiles_for_area(request) # Assert assert state["tile_gets"] == 6 assert len(writer.write_calls) == 6 assert second.outcome == DownloadOutcome.SUCCESS assert second.tiles_downloaded == 6 # ---------------------------------------------------------------------- # Retry-After HTTP-date form (Risk 1 from the spec) # ---------------------------------------------------------------------- def test_429_retry_after_http_date_form_parses(tmp_path: Path) -> None: # Arrange tiles = [_tile_entry(zoom=14, lat=45.0, lon=-122.0)] state = {"attempts": 0} future = (datetime.now(timezone.utc) + timedelta(seconds=20)).strftime( "%a, %d %b %Y %H:%M:%S GMT" ) def _factory(request: httpx.Request) -> httpx.Response: state["attempts"] += 1 if state["attempts"] == 1: return httpx.Response(429, headers={"Retry-After": future}) return httpx.Response(200, content=b"\xff\xd8tile") transport = httpx.MockTransport( _make_route_handler( list_response=_list_response(tiles), tile_response_factory=_factory, ) ) sleeps: list[float] = [] (downloader, _logs, _writer, _enforcer, _sleeps) = _build_downloader( transport=transport, sleep_recorder=sleeps ) # Act report = downloader.download_tiles_for_area(_make_request(cache_root=tmp_path)) # Assert assert state["attempts"] == 2 assert sleeps and sleeps[0] >= 0 assert report.outcome == DownloadOutcome.SUCCESS # ---------------------------------------------------------------------- # 429 budget exhaustion → RateLimitedError # ---------------------------------------------------------------------- def test_429_budget_exhaustion_raises_rate_limited_error(tmp_path: Path) -> None: # Arrange tiles = [_tile_entry(zoom=14, lat=45.0, lon=-122.0)] def _factory(request: httpx.Request) -> httpx.Response: return httpx.Response(429, headers={"Retry-After": "300"}) transport = httpx.MockTransport( _make_route_handler( list_response=_list_response(tiles), tile_response_factory=_factory, ) ) cfg = C11Config( satellite_provider_url=_BASE_URL, service_api_key=_API_KEY, download_http_timeout_s=5.0, download_max_5xx_retries=4, download_max_retry_after_s=400, download_resolution_floor_m_per_px=0.5, ) (downloader, _logs, _writer, _enforcer, _sleeps) = _build_downloader( transport=transport, config=cfg ) # Act / Assert with pytest.raises(RateLimitedError): downloader.download_tiles_for_area(_make_request(cache_root=tmp_path)) # ---------------------------------------------------------------------- # NFR — bookkeeping throughput on a 1000-tile happy path # ---------------------------------------------------------------------- def test_nfr_throughput_1000_tiles_under_budget(tmp_path: Path) -> None: # Arrange tiles = [ _tile_entry(zoom=14, lat=45.0 + i * 0.0001, lon=-122.0 + i * 0.0001) for i in range(1000) ] transport = httpx.MockTransport( _make_route_handler( list_response=_list_response(tiles), tile_response_factory=lambda r: httpx.Response(200, content=b"\xff\xd8tile"), ) ) (downloader, _logs, _writer, _enforcer, _sleeps) = _build_downloader( transport=transport ) import time as _time t0 = _time.perf_counter() report = downloader.download_tiles_for_area(_make_request(cache_root=tmp_path)) elapsed = _time.perf_counter() - t0 # Assert — budget is generous; the goal is to catch an O(n^2) # bookkeeping regression, not to certify wall-clock throughput. assert report.outcome == DownloadOutcome.SUCCESS assert report.tiles_downloaded == 1000 assert elapsed < 10.0, f"1000-tile bookkeeping took {elapsed:.2f}s"