azaion/gps-denied-onboard
1
0
Fork 0
mirror of https://github.com/azaion/gps-denied-onboard.git synced 2026-06-22 22:11:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

[AZ-777] Phase 1 hotfix (z/x/y) + Phase 2 Derkachi seed + ops

Browse Source 
Phase 1 hotfix:
- C11 HttpTileDownloader adapted to satellite-provider v2.0.0
  z/x/y inventory contract (bulk POST keyed by slippy-map coords).
- Unit tests rewritten to exercise the new inventory schema.
- E2E smoke test updated to match the v2.0.0 wire.

Phase 2 (Derkachi seed + smoke-validated on Jetson):
- tests/fixtures/derkachi_c6/{README,bbox.yaml,seed_region.py}
  drives POST /api/satellite/region against satellite-provider
  with Google Maps as the imagery source. Smoke run produced
  4 regions, 175 tiles, inventory 32/32.
- scripts/mint_dev_jwt.py + run-tests-jetson.sh auto-mint and
  export SATELLITE_PROVIDER_API_KEY using JWT_SECRET / JWT_ISSUER
  / JWT_AUDIENCE env vars (no host port mappings; e2e-runner
  reaches SP via internal docker network only).

Spec amendment: AZ-777 todo spec updated to record the
Google Maps imagery source decision and STOP-gate state.

AZ-777 Phase 3+ work is superseded by Epic AZ-835 (see next
commit).

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Oleksandr Bezdieniezhnykh
2026-05-22 17:39:21 +03:00
parent 811b04e605
commit b15454b9a9
9 changed files with 924 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/gps_denied_onboard/components/c11_tile_manager/tile_downloader.py
+42 -101
View File
@@ -145,9 +145,7 @@ class _TileWriterLike(Protocol):
sector_class: str,
) -> str: ...
def tile_already_present(
self, *, zoom_level: int, lat: float, lon: float
) -> bool: ...
def tile_already_present(self, *, zoom_level: int, lat: float, lon: float) -> bool: ...
@runtime_checkable
@@ -351,12 +349,8 @@ def _enumerate_bbox_tile_coords(
coords: list[tuple[int, int, int]] = []
for zoom in zoom_levels:
x_sw, y_sw = WgsConverter.latlon_to_tile_xy(
int(zoom), bbox_min_lat, bbox_min_lon
)
x_ne, y_ne = WgsConverter.latlon_to_tile_xy(
int(zoom), bbox_max_lat, bbox_max_lon
)
x_sw, y_sw = WgsConverter.latlon_to_tile_xy(int(zoom), bbox_min_lat, bbox_min_lon)
x_ne, y_ne = WgsConverter.latlon_to_tile_xy(int(zoom), bbox_max_lat, bbox_max_lon)
x_lo, x_hi = (x_sw, x_ne) if x_sw <= x_ne else (x_ne, x_sw)
y_lo, y_hi = (y_ne, y_sw) if y_ne <= y_sw else (y_sw, y_ne)
for x in range(x_lo, x_hi + 1):
@@ -373,11 +367,7 @@ def _tile_center_latlon(zoom: int, x: int, y: int) -> tuple[float, float]:
def _tile_size_meters_at(zoom: int, lat_deg: float) -> float:
return (
_EARTH_EQUATORIAL_CIRCUMFERENCE_M
* math.cos(math.radians(lat_deg))
/ (1 << int(zoom))
)
return _EARTH_EQUATORIAL_CIRCUMFERENCE_M * math.cos(math.radians(lat_deg)) / (1 << int(zoom))
def _format_tile_id_str(zoom: int, x: int, y: int) -> str:
@@ -387,15 +377,12 @@ def _format_tile_id_str(zoom: int, x: int, y: int) -> str:
def _parse_tile_id_str(tile_id_str: str) -> tuple[int, int, int]:
parts = tile_id_str.split("_")
if len(parts) != 3:
raise ValueError(
f"tile_id_str must be 'z_x_y'; got {tile_id_str!r}"
)
raise ValueError(f"tile_id_str must be 'z_x_y'; got {tile_id_str!r}")
try:
return int(parts[0]), int(parts[1]), int(parts[2])
except ValueError as exc:
raise ValueError(
f"tile_id_str must contain three integers separated by '_'; "
f"got {tile_id_str!r}"
f"tile_id_str must contain three integers separated by '_'; got {tile_id_str!r}"
) from exc
@@ -405,10 +392,7 @@ def _chunk_iter(
) -> list[tuple[tuple[int, int, int], ...]]:
if chunk_size <= 0:
raise ValueError(f"chunk_size must be > 0; got {chunk_size}")
return [
tuple(seq[start : start + chunk_size])
for start in range(0, len(seq), chunk_size)
]
return [tuple(seq[start : start + chunk_size]) for start in range(0, len(seq), chunk_size)]
# ----------------------------------------------------------------------
@@ -505,8 +489,12 @@ class HttpTileDownloader:
counts = existing.tile_counts
return DownloadBatchReport(
outcome=DownloadOutcome.IDEMPOTENT_NO_OP,
tiles_requested=int(counts.get("tiles_requested", len(existing.tile_ids_completed))),
tiles_downloaded=int(counts.get("tiles_downloaded", len(existing.tile_ids_completed))),
tiles_requested=int(
counts.get("tiles_requested", len(existing.tile_ids_completed))
),
tiles_downloaded=int(
counts.get("tiles_downloaded", len(existing.tile_ids_completed))
),
tiles_rejected_resolution=int(counts.get("tiles_rejected_resolution", 0)),
tiles_rejected_freshness=int(counts.get("tiles_rejected_freshness", 0)),
tiles_downgraded=int(counts.get("tiles_downgraded", 0)),
@@ -667,7 +655,7 @@ class HttpTileDownloader:
AZ-777: the satellite-provider v1.0.0 inventory contract is
keyed by explicit slippy-map coords, NOT by a server-side
bbox query. This method enumerates the tile grid for the
bbox × zoom set, chunks into ≤5000-entry POSTs (the
bbox x zoom set, chunks into ≤5000-entry POSTs (the
``TileInventoryLimits.MaxEntriesPerRequest`` cap), and
returns one :class:`TileSummary` per ``present=true`` entry.
Absent tiles are silently dropped — they need to be seeded
@@ -690,15 +678,8 @@ class HttpTileDownloader:
summaries.extend(self._fetch_inventory_chunk(chunk))
return summaries
def _fetch_inventory_chunk(
self, chunk: tuple[tuple[int, int, int], ...]
) -> list[TileSummary]:
body = {
"tiles": [
{"tileZoom": z, "tileX": x, "tileY": y}
for (z, x, y) in chunk
]
}
def _fetch_inventory_chunk(self, chunk: tuple[tuple[int, int, int], ...]) -> list[TileSummary]:
body = {"tiles": [{"z": z, "x": x, "y": y} for (z, x, y) in chunk]}
response = self._send_post(
self._config.satellite_provider_url.rstrip("/") + _INVENTORY_PATH,
json_body=body,
@@ -707,18 +688,14 @@ class HttpTileDownloader:
try:
decoded = response.json()
except ValueError as exc:
self._log_provider_failure(
"inventory_not_json", response.status_code, str(exc)
)
self._log_provider_failure("inventory_not_json", response.status_code, str(exc))
raise SatelliteProviderError(
"satellite-provider returned non-JSON inventory body"
) from exc
try:
entries = decoded["results"]
except (KeyError, TypeError) as exc:
self._log_provider_failure(
"inventory_schema", response.status_code, str(exc)
)
self._log_provider_failure("inventory_schema", response.status_code, str(exc))
raise SatelliteProviderError(
"satellite-provider inventory response missing 'results'"
) from exc
@@ -738,28 +715,23 @@ class HttpTileDownloader:
try:
present = bool(entry["present"])
except (KeyError, TypeError) as exc:
self._log_provider_failure(
"inventory_entry_schema", response.status_code, str(exc)
)
self._log_provider_failure("inventory_entry_schema", response.status_code, str(exc))
raise SatelliteProviderError(
"satellite-provider inventory entry missing 'present'"
) from exc
if not present:
continue
try:
zoom = int(entry["tileZoom"])
x = int(entry["tileX"])
y = int(entry["tileY"])
zoom = int(entry["z"])
x = int(entry["x"])
y = int(entry["y"])
produced_at = _parse_iso(str(entry["capturedAt"]))
resolution_m_per_px = float(entry["resolutionMPerPx"])
except (KeyError, TypeError, ValueError) as exc:
self._log_provider_failure(
"inventory_entry_schema", response.status_code, str(exc)
)
self._log_provider_failure("inventory_entry_schema", response.status_code, str(exc))
raise SatelliteProviderError(
"satellite-provider inventory present-entry missing "
"required fields (tileZoom/tileX/tileY/capturedAt/"
"resolutionMPerPx)"
"required fields (z/x/y/capturedAt/resolutionMPerPx)"
) from exc
lat, lon = _tile_center_latlon(zoom, x, y)
summaries.append(
@@ -784,9 +756,7 @@ class HttpTileDownloader:
session: _DownloadSession,
) -> None:
remaining_bytes = sum(
int(s.estimated_bytes)
for s in summaries
if s.tile_id_str not in completed_set
int(s.estimated_bytes) for s in summaries if s.tile_id_str not in completed_set
)
if remaining_bytes <= 0:
return
@@ -798,8 +768,7 @@ class HttpTileDownloader:
except Exception as exc:
self._log_budget_failure(remaining_bytes, detail=str(exc))
raise CacheBudgetExceededError(
f"c6 budget enforcer refused {remaining_bytes} bytes "
f"of head-room: {exc}"
f"c6 budget enforcer refused {remaining_bytes} bytes of head-room: {exc}"
) from exc
def _download_one_tile(
@@ -833,17 +802,13 @@ class HttpTileDownloader:
f"z_x_y format (got {summary.tile_id_str!r})"
) from exc
ingest_url = (
self._config.satellite_provider_url.rstrip("/")
+ f"{_TILES_PATH}/{zoom}/{x}/{y}"
self._config.satellite_provider_url.rstrip("/") + f"{_TILES_PATH}/{zoom}/{x}/{y}"
)
response = self._send_get(ingest_url, params=None, session=session)
if not response.content:
self._log_provider_failure(
"empty_body", response.status_code, summary.tile_id_str
)
self._log_provider_failure("empty_body", response.status_code, summary.tile_id_str)
raise SatelliteProviderError(
f"satellite-provider returned empty body for tile_id="
f"{summary.tile_id_str}"
f"satellite-provider returned empty body for tile_id={summary.tile_id_str}"
)
tile_blob = response.content
content_sha256_hex = hashlib.sha256(tile_blob).hexdigest()
@@ -900,9 +865,7 @@ class HttpTileDownloader:
) -> httpx.Response:
"""GET with auth header + 429 / 5xx handling."""
return self._send_request(
"GET", url, params=params, json_body=None, session=session
)
return self._send_request("GET", url, params=params, json_body=None, session=session)
def _send_post(
self,
@@ -912,9 +875,7 @@ class HttpTileDownloader:
) -> httpx.Response:
"""POST with auth header + 429 / 5xx handling (AZ-777 inventory contract)."""
return self._send_request(
"POST", url, params=None, json_body=json_body, session=session
)
return self._send_request("POST", url, params=None, json_body=json_body, session=session)
def _send_request(
self,
@@ -952,18 +913,13 @@ class HttpTileDownloader:
if attempt > self._config.download_max_5xx_retries:
self._log_provider_failure("connection_error", None, last_error)
raise SatelliteProviderError(
f"satellite-provider unreachable after "
f"{attempt - 1} retries: {last_error}"
f"satellite-provider unreachable after {attempt - 1} retries: {last_error}"
) from exc
self._sleep_with_log(
self._backoff_for(attempt - 1), last_error, session
)
self._sleep_with_log(self._backoff_for(attempt - 1), last_error, session)
continue
if response.status_code in (401, 403):
self._log_provider_failure(
"auth_failed", response.status_code, "fail-fast"
)
self._log_provider_failure("auth_failed", response.status_code, "fail-fast")
raise SatelliteProviderError(
f"satellite-provider rejected auth (http_status="
f"{response.status_code}); fail-fast"
@@ -979,12 +935,9 @@ class HttpTileDownloader:
session.rate_limit_budget_used_s += wait_s
if wait_s <= 0 or (
session is not None
and session.rate_limit_budget_used_s
>= self._config.download_max_retry_after_s
and session.rate_limit_budget_used_s >= self._config.download_max_retry_after_s
):
self._log_provider_failure(
"rate_limited", 429, "Retry-After budget exhausted"
)
self._log_provider_failure("rate_limited", 429, "Retry-After budget exhausted")
raise RateLimitedError(
"satellite-provider rate-limited the download; "
f"cumulative Retry-After budget "
@@ -997,22 +950,16 @@ class HttpTileDownloader:
if response.status_code >= 500:
last_error = f"http_status={response.status_code}"
if attempt > self._config.download_max_5xx_retries:
self._log_provider_failure(
"persistent_5xx", response.status_code, last_error
)
self._log_provider_failure("persistent_5xx", response.status_code, last_error)
raise SatelliteProviderError(
f"satellite-provider returned {response.status_code} "
f"after {attempt - 1} retries"
)
self._sleep_with_log(
self._backoff_for(attempt - 1), last_error, session
)
self._sleep_with_log(self._backoff_for(attempt - 1), last_error, session)
continue
if response.status_code != 200:
self._log_provider_failure(
"unexpected_status", response.status_code, "non-200"
)
self._log_provider_failure("unexpected_status", response.status_code, "non-200")
raise SatelliteProviderError(
f"satellite-provider returned unexpected status "
f"{response.status_code} (expected 200)"
@@ -1026,9 +973,7 @@ class HttpTileDownloader:
attempt_idx = len(self._backoff_schedule_s) - 1
return self._backoff_schedule_s[attempt_idx]
def _sleep_with_log(
self, wait_s: float, reason: str, session: _DownloadSession | None
) -> None:
def _sleep_with_log(self, wait_s: float, reason: str, session: _DownloadSession | None) -> None:
if session is not None:
session.retry_count += 1
self._logger.warning(
@@ -1045,9 +990,7 @@ class HttpTileDownloader:
)
self._sleep(wait_s)
def _log_provider_failure(
self, reason: str, http_status: int | None, detail: str
) -> None:
def _log_provider_failure(self, reason: str, http_status: int | None, detail: str) -> None:
self._logger.error(
"Download provider failed",
extra={
@@ -1062,9 +1005,7 @@ class HttpTileDownloader:
},
)
def _log_budget_failure(
self, requested_bytes: int, detail: str | None = None
) -> None:
def _log_budget_failure(self, requested_bytes: int, detail: str | None = None) -> None:
self._logger.error(
"Cache-budget pre-check failed",
extra={