mirror of
https://github.com/azaion/satellite-provider.git
synced 2026-06-21 13:01:14 +00:00
909f69cb3a
Production code:
- POST /api/satellite/tiles/inventory (XOR body, 5000-cap,
most-recent-per-location_hash select, present/absent shaping).
- Kestrel HttpProtocols.Http1AndHttp2 on every listener (AC-5).
- Migration 015 creates tiles_leaflet_path covering index over
(location_hash, captured_at DESC, updated_at DESC, id DESC)
INCLUDE (file_path, source); drops superseded idx_tiles_location_hash.
- TileRepository.GetByTileCoordinatesAsync rewired to filter by
location_hash (Index Only Scan via tiles_leaflet_path).
- TileRepository.GetTilesByLocationHashesAsync added with Npgsql-
direct ANY($1::uuid[]) binding (Dapper IEnumerable expansion is
incompatible with the array form).
- Uuidv5.LocationHashForTile centralises the UUIDv5(TileNamespace,
"{z}/{x}/{y}") formula — single source of truth for the cross-repo
invariant (gps-denied-onboard parity).
Contracts:
- New: contracts/api/tile-inventory.md v1.0.0.
- Bumped: contracts/data-access/tile-storage.md to v2.0.0 (joint
ownership by AZ-503-foundation + AZ-505: schema + covering index +
GetByTileCoordinatesAsync rewrite).
Tests:
- TileInventoryTests covers AC-1, AC-2 (DB-level), AC-4, AC-6.
- Http2MultiplexingTests covers AC-5 (20 concurrent multiplexed GETs
over h2c via SocketsHttpHandler + AppContext Http2Unencrypted switch).
- LeafletPathIndexOnlyTests covers AC-3 (EXPLAIN (ANALYZE, BUFFERS)
asserts Index Only Scan over tiles_leaflet_path with heap_blocks=0).
Docs:
- architecture.md, system-flows.md, data_model.md, module-layout.md,
glossary.md, modules/api_program.md, modules/dataaccess_tile_repository.md,
components/02_data_access/description.md all updated to reference the
v2.0.0 tile-storage contract + new tile-inventory contract + AC-7.
Reports:
- batch_01_cycle6_report.md, batch_01_cycle6_review.md,
implementation_completeness_cycle6_report.md (PASS),
implementation_report_tile_inventory_cycle6.md.
Task spec moved todo/ -> done/.
Co-authored-by: Cursor <cursoragent@cursor.com>
289 lines
14 KiB
C#
289 lines
14 KiB
C#
using System.Diagnostics;
|
|
using Dapper;
|
|
using Microsoft.Extensions.Logging;
|
|
using Npgsql;
|
|
using SatelliteProvider.Common.Configs;
|
|
using SatelliteProvider.Common.Utils;
|
|
using SatelliteProvider.DataAccess.Models;
|
|
|
|
namespace SatelliteProvider.DataAccess.Repositories;
|
|
|
|
public class TileRepository : ITileRepository
|
|
{
|
|
private const int SlowQueryThresholdMs = 500;
|
|
|
|
private const string ColumnList = @"id, tile_zoom as TileZoom, tile_x as TileX, tile_y as TileY,
|
|
latitude, longitude,
|
|
tile_size_meters as TileSizeMeters, tile_size_pixels as TileSizePixels,
|
|
image_type as ImageType, maps_version as MapsVersion, version,
|
|
file_path as FilePath, source, captured_at as CapturedAt,
|
|
created_at as CreatedAt, updated_at as UpdatedAt,
|
|
flight_id as FlightId, location_hash as LocationHash,
|
|
content_sha256 as ContentSha256, legacy_id as LegacyId";
|
|
|
|
private readonly string _connectionString;
|
|
private readonly ILogger<TileRepository> _logger;
|
|
|
|
public TileRepository(string connectionString, ILogger<TileRepository> logger)
|
|
{
|
|
_connectionString = connectionString;
|
|
_logger = logger;
|
|
}
|
|
|
|
public async Task<TileEntity?> GetByIdAsync(Guid id)
|
|
{
|
|
using var connection = new NpgsqlConnection(_connectionString);
|
|
const string sql = $@"
|
|
SELECT {ColumnList}
|
|
FROM tiles
|
|
WHERE id = @Id";
|
|
|
|
return await connection.QuerySingleOrDefaultAsync<TileEntity>(sql, new { Id = id });
|
|
}
|
|
|
|
public async Task<TileEntity?> GetByTileCoordinatesAsync(int tileZoom, int tileX, int tileY)
|
|
{
|
|
using var connection = new NpgsqlConnection(_connectionString);
|
|
// AZ-505 read-rewrite: filter by `location_hash` so the new
|
|
// `tiles_leaflet_path` covering index drives the scan. Selection rule
|
|
// is unchanged from AZ-484: most-recent across sources/flights with
|
|
// deterministic tie-break on (captured_at DESC, updated_at DESC, id DESC).
|
|
// Heap fetch is unavoidable here (the column list spans columns not in
|
|
// the index INCLUDE list); the slim `SELECT file_path` Leaflet hot path
|
|
// — which is what AC-3 measures — is index-only-scannable.
|
|
var locationHash = Uuidv5.LocationHashForTile(tileZoom, tileX, tileY);
|
|
const string sql = $@"
|
|
SELECT {ColumnList}
|
|
FROM tiles
|
|
WHERE location_hash = @LocationHash
|
|
ORDER BY captured_at DESC, updated_at DESC, id DESC
|
|
LIMIT 1";
|
|
|
|
return await connection.QuerySingleOrDefaultAsync<TileEntity>(sql, new { LocationHash = locationHash });
|
|
}
|
|
|
|
public async Task<IReadOnlyDictionary<Guid, TileEntity>> GetTilesByLocationHashesAsync(IReadOnlyList<Guid> locationHashes)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(locationHashes);
|
|
if (locationHashes.Count == 0)
|
|
{
|
|
return new Dictionary<Guid, TileEntity>();
|
|
}
|
|
|
|
await using var connection = new NpgsqlConnection(_connectionString);
|
|
await connection.OpenAsync();
|
|
|
|
// AZ-505: one-row-per-hash bulk lookup. `DISTINCT ON (location_hash)`
|
|
// collapses the per-(z, x, y) cell to its most-recent variant across
|
|
// sources/flights using the same tie-break as AZ-484. Caller dedupes
|
|
// input + re-aligns response order; this query returns at most one
|
|
// row per distinct hash.
|
|
//
|
|
// The query is intentionally NOT routed through Dapper: Dapper's
|
|
// parameter expander rewrites any IEnumerable parameter (including
|
|
// `Guid[]`) into `(@p0, @p1, ...)`, which would turn `ANY(@p)` into
|
|
// `ANY((@p0, @p1, ...))` and break the SQL. Using NpgsqlParameter with
|
|
// `Array | Uuid` lets Npgsql bind the array as a single `uuid[]`,
|
|
// which is the form the AZ-505 spec query expects.
|
|
const string sql = @"
|
|
SELECT id, tile_zoom AS TileZoom, tile_x AS TileX, tile_y AS TileY,
|
|
latitude, longitude,
|
|
tile_size_meters AS TileSizeMeters, tile_size_pixels AS TileSizePixels,
|
|
image_type AS ImageType, maps_version AS MapsVersion, version,
|
|
file_path AS FilePath, source, captured_at AS CapturedAt,
|
|
created_at AS CreatedAt, updated_at AS UpdatedAt,
|
|
flight_id AS FlightId, location_hash AS LocationHash,
|
|
content_sha256 AS ContentSha256, legacy_id AS LegacyId
|
|
FROM (
|
|
SELECT DISTINCT ON (location_hash)
|
|
id, tile_zoom, tile_x, tile_y,
|
|
latitude, longitude,
|
|
tile_size_meters, tile_size_pixels,
|
|
image_type, maps_version, version,
|
|
file_path, source, captured_at,
|
|
created_at, updated_at,
|
|
flight_id, location_hash,
|
|
content_sha256, legacy_id
|
|
FROM tiles
|
|
WHERE location_hash = ANY(@LocationHashes)
|
|
ORDER BY location_hash, captured_at DESC, updated_at DESC, id DESC
|
|
) most_recent";
|
|
|
|
var distinctHashes = locationHashes.Distinct().ToArray();
|
|
|
|
await using var cmd = new NpgsqlCommand(sql, connection);
|
|
var arrayParam = new NpgsqlParameter("LocationHashes", NpgsqlTypes.NpgsqlDbType.Array | NpgsqlTypes.NpgsqlDbType.Uuid)
|
|
{
|
|
Value = distinctHashes
|
|
};
|
|
cmd.Parameters.Add(arrayParam);
|
|
|
|
var stopwatch = Stopwatch.StartNew();
|
|
var rows = new Dictionary<Guid, TileEntity>(distinctHashes.Length);
|
|
await using (var reader = await cmd.ExecuteReaderAsync())
|
|
{
|
|
while (await reader.ReadAsync())
|
|
{
|
|
var tile = new TileEntity
|
|
{
|
|
Id = reader.GetGuid(0),
|
|
TileZoom = reader.GetInt32(1),
|
|
TileX = reader.GetInt32(2),
|
|
TileY = reader.GetInt32(3),
|
|
Latitude = reader.GetDouble(4),
|
|
Longitude = reader.GetDouble(5),
|
|
TileSizeMeters = reader.GetDouble(6),
|
|
TileSizePixels = reader.GetInt32(7),
|
|
ImageType = reader.GetString(8),
|
|
MapsVersion = reader.IsDBNull(9) ? null : reader.GetString(9),
|
|
Version = reader.IsDBNull(10) ? null : reader.GetInt32(10),
|
|
FilePath = reader.GetString(11),
|
|
Source = reader.GetString(12),
|
|
CapturedAt = reader.GetDateTime(13),
|
|
CreatedAt = reader.GetDateTime(14),
|
|
UpdatedAt = reader.GetDateTime(15),
|
|
FlightId = reader.IsDBNull(16) ? null : reader.GetGuid(16),
|
|
LocationHash = reader.GetGuid(17),
|
|
ContentSha256 = reader.IsDBNull(18) ? null : (byte[])reader.GetValue(18),
|
|
LegacyId = reader.IsDBNull(19) ? null : reader.GetGuid(19)
|
|
};
|
|
rows[tile.LocationHash] = tile;
|
|
}
|
|
}
|
|
stopwatch.Stop();
|
|
|
|
if (stopwatch.ElapsedMilliseconds > SlowQueryThresholdMs)
|
|
{
|
|
_logger.LogWarning(
|
|
"Slow GetTilesByLocationHashesAsync: {ElapsedMs} ms (threshold {ThresholdMs} ms) for {RequestedHashes} requested ({DistinctHashes} distinct) hashes",
|
|
stopwatch.ElapsedMilliseconds, SlowQueryThresholdMs, locationHashes.Count, distinctHashes.Length);
|
|
}
|
|
|
|
return rows;
|
|
}
|
|
|
|
public async Task<IEnumerable<TileEntity>> GetTilesByRegionAsync(double latitude, double longitude, double sizeMeters, int zoomLevel)
|
|
{
|
|
using var connection = new NpgsqlConnection(_connectionString);
|
|
|
|
var latRad = latitude * Math.PI / 180.0;
|
|
var metersPerPixel = (GeoUtils.EarthEquatorialCircumferenceMeters * Math.Cos(latRad)) / (Math.Pow(2, zoomLevel) * MapConfig.DefaultTileSizePixels);
|
|
var tileSizeMeters = metersPerPixel * MapConfig.DefaultTileSizePixels;
|
|
|
|
var expandedSizeMeters = sizeMeters + (tileSizeMeters * 2);
|
|
|
|
var latRange = expandedSizeMeters / GeoUtils.MetersPerDegreeLatitude;
|
|
var lonRange = expandedSizeMeters / (GeoUtils.MetersPerDegreeLatitude * Math.Cos(latitude * Math.PI / 180.0));
|
|
|
|
// AZ-484 selection rule: at most one row per (lat, lon, zoom, size) cell, picking
|
|
// the most-recent across sources via DISTINCT ON, with deterministic tie-break on
|
|
// (captured_at DESC, updated_at DESC, id DESC). The outer ORDER BY restores the
|
|
// pre-AZ-484 caller-facing order (latitude DESC, longitude ASC); the pre-AZ-484
|
|
// updated_at DESC tiebreak is unreachable here because DISTINCT ON already
|
|
// guarantees one row per (latitude, longitude, ...) tuple.
|
|
const string sql = $@"
|
|
SELECT * FROM (
|
|
SELECT DISTINCT ON (latitude, longitude, tile_zoom, tile_size_meters)
|
|
{ColumnList}
|
|
FROM tiles
|
|
WHERE latitude BETWEEN @MinLat AND @MaxLat
|
|
AND longitude BETWEEN @MinLon AND @MaxLon
|
|
AND tile_zoom = @TileZoom
|
|
ORDER BY latitude, longitude, tile_zoom, tile_size_meters,
|
|
captured_at DESC, updated_at DESC, id DESC
|
|
) deduped
|
|
ORDER BY latitude DESC, longitude ASC";
|
|
|
|
var stopwatch = Stopwatch.StartNew();
|
|
var tiles = await connection.QueryAsync<TileEntity>(sql, new
|
|
{
|
|
MinLat = latitude - latRange / 2,
|
|
MaxLat = latitude + latRange / 2,
|
|
MinLon = longitude - lonRange / 2,
|
|
MaxLon = longitude + lonRange / 2,
|
|
TileZoom = zoomLevel
|
|
});
|
|
stopwatch.Stop();
|
|
|
|
if (stopwatch.ElapsedMilliseconds > SlowQueryThresholdMs)
|
|
{
|
|
_logger.LogWarning(
|
|
"Slow GetTilesByRegionAsync: {ElapsedMs} ms (threshold {ThresholdMs} ms) for lat={Latitude}, lon={Longitude}, sizeMeters={SizeMeters}, zoom={ZoomLevel}",
|
|
stopwatch.ElapsedMilliseconds, SlowQueryThresholdMs, latitude, longitude, sizeMeters, zoomLevel);
|
|
}
|
|
|
|
return tiles;
|
|
}
|
|
|
|
public async Task<Guid> InsertAsync(TileEntity tile)
|
|
{
|
|
using var connection = new NpgsqlConnection(_connectionString);
|
|
// AZ-503: integer-keyed UPSERT with per-flight separation. The conflict key
|
|
// is (tile_zoom, tile_x, tile_y, tile_size_meters, source, COALESCE(flight_id, '00...0')).
|
|
// Two UAV flights uploading the same (z, x, y) cell coexist as distinct rows
|
|
// because their flight_id values differ; legacy/google_maps rows collapse on
|
|
// the zero-UUID coalesce, preserving AZ-484 single-row-per-cell semantics for
|
|
// those producers. Float-based latitude/longitude is no longer part of the key
|
|
// so independently-rounded center coords always converge on the same row.
|
|
//
|
|
// `id` is deliberately NOT updated on conflict — legacy random ids and AZ-503
|
|
// deterministic ids both stay stable, matching AC-2 ("the `id` column is not
|
|
// regenerated").
|
|
const string sql = @"
|
|
INSERT INTO tiles (id, tile_zoom, tile_x, tile_y, latitude, longitude, tile_size_meters,
|
|
tile_size_pixels, image_type, maps_version, version, file_path,
|
|
source, captured_at, created_at, updated_at,
|
|
flight_id, location_hash, content_sha256, legacy_id)
|
|
VALUES (@Id, @TileZoom, @TileX, @TileY, @Latitude, @Longitude, @TileSizeMeters,
|
|
@TileSizePixels, @ImageType, @MapsVersion, @Version, @FilePath,
|
|
@Source, @CapturedAt, @CreatedAt, @UpdatedAt,
|
|
@FlightId, @LocationHash, @ContentSha256, @LegacyId)
|
|
ON CONFLICT (tile_zoom, tile_x, tile_y, tile_size_meters, source,
|
|
COALESCE(flight_id, '00000000-0000-0000-0000-000000000000'::uuid))
|
|
DO UPDATE SET
|
|
file_path = EXCLUDED.file_path,
|
|
latitude = EXCLUDED.latitude,
|
|
longitude = EXCLUDED.longitude,
|
|
captured_at = EXCLUDED.captured_at,
|
|
updated_at = EXCLUDED.updated_at,
|
|
content_sha256 = EXCLUDED.content_sha256
|
|
RETURNING id";
|
|
|
|
return await connection.ExecuteScalarAsync<Guid>(sql, tile);
|
|
}
|
|
|
|
public async Task<int> UpdateAsync(TileEntity tile)
|
|
{
|
|
using var connection = new NpgsqlConnection(_connectionString);
|
|
const string sql = @"
|
|
UPDATE tiles
|
|
SET tile_zoom = @TileZoom,
|
|
tile_x = @TileX,
|
|
tile_y = @TileY,
|
|
latitude = @Latitude,
|
|
longitude = @Longitude,
|
|
tile_size_meters = @TileSizeMeters,
|
|
tile_size_pixels = @TileSizePixels,
|
|
image_type = @ImageType,
|
|
maps_version = @MapsVersion,
|
|
version = @Version,
|
|
file_path = @FilePath,
|
|
source = @Source,
|
|
captured_at = @CapturedAt,
|
|
updated_at = @UpdatedAt,
|
|
flight_id = @FlightId,
|
|
location_hash = @LocationHash,
|
|
content_sha256 = @ContentSha256
|
|
WHERE id = @Id";
|
|
|
|
return await connection.ExecuteAsync(sql, tile);
|
|
}
|
|
|
|
public async Task<int> DeleteAsync(Guid id)
|
|
{
|
|
using var connection = new NpgsqlConnection(_connectionString);
|
|
const string sql = "DELETE FROM tiles WHERE id = @Id";
|
|
return await connection.ExecuteAsync(sql, new { Id = id });
|
|
}
|
|
}
|