mirror of
https://github.com/azaion/satellite-provider.git
synced 2026-06-22 05:41:15 +00:00
[AZ-484] Multi-source tile storage: source + captured_at
Add per-source tile rows to support multi-provider imagery (Google Maps + future UAV). Migration 013 (transactional) introduces source/captured_at columns, backfills existing rows to (source='google_maps', captured_at=created_at), and replaces the 4-column unique index with a 5-column index that includes source. TileRepository: - ColumnList includes source + captured_at - GetByTileCoordinatesAsync returns most-recent row across sources (ORDER BY captured_at DESC, updated_at DESC, id DESC) - GetTilesByRegionAsync uses DISTINCT ON to pick the most-recent tile per cell, restoring caller-facing row order - Insert/Update upsert on the new 5-column conflict key TileSource enum lives in Common.Enums. Snake_case wire format (google_maps, uav) is enforced by a focused TileSourceTypeHandler because the generic ToLowerInvariant pattern would emit "googlemaps", violating contract v1.0.0. TileService stamps Source=GoogleMaps + CapturedAt=UtcNow on every new tile. Tile-storage contract is now frozen at v1.0.0. AC coverage 7/7. New unit + integration tests cover all ACs; existing 200 unit + 5 smoke tests preserved. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Oleksandr Bezdieniezhnykh
@@ -15,9 +15,21 @@ public static class MigrationTests
|
||||
?? "Host=postgres;Port=5432;Database=satelliteprovider;Username=postgres;Password=postgres";
|
||||
|
||||
await DedupeSqlCollapsesDuplicatesByLatestUpdatedAt_AZ357_AC2(connectionString);
|
||||
await NewUniqueConstraintExistsOnFourColumns_AZ357_AC2(connectionString);
|
||||
|
||||
Console.WriteLine("✓ Migration 012 tests: PASSED");
|
||||
|
||||
Console.WriteLine();
|
||||
Console.WriteLine("Test: Migration 013 (AZ-484)");
|
||||
Console.WriteLine("============================");
|
||||
Console.WriteLine();
|
||||
|
||||
await BackfillUpdateAssignsGoogleMapsAndCapturedAt_AZ484_AC4(connectionString);
|
||||
await MultiSourceInsertCoexistsUnderNewIndex_AZ484_AC1(connectionString);
|
||||
await MostRecentAcrossSourcesSelection_AZ484_AC2(connectionString);
|
||||
await SameSourceUpsertReplacesPreviousRow_AZ484_AC3(connectionString);
|
||||
await NewUniqueConstraintIncludesSourceColumn_AZ484_AC1(connectionString);
|
||||
|
||||
Console.WriteLine("✓ Migration 013 tests: PASSED");
|
||||
}
|
||||
|
||||
private static async Task DedupeSqlCollapsesDuplicatesByLatestUpdatedAt_AZ357_AC2(string connectionString)
|
||||
@@ -103,50 +115,326 @@ public static class MigrationTests
|
||||
Console.WriteLine(" ✓ Unique row (idF) preserved");
|
||||
}
|
||||
|
||||
private static async Task NewUniqueConstraintExistsOnFourColumns_AZ357_AC2(string connectionString)
|
||||
private static async Task NewUniqueConstraintIncludesSourceColumn_AZ484_AC1(string connectionString)
|
||||
{
|
||||
Console.WriteLine();
|
||||
Console.WriteLine("AZ-357 AC-2 part 2: post-migration unique index has the new 4-column shape");
|
||||
Console.WriteLine("AZ-484 AC-1 part 2: post-migration-013 unique index includes the source column");
|
||||
|
||||
// Arrange / Act
|
||||
await using var conn = new NpgsqlConnection(connectionString);
|
||||
await conn.OpenAsync();
|
||||
|
||||
const string sql = @"
|
||||
SELECT indexdef
|
||||
SELECT indexname, indexdef
|
||||
FROM pg_indexes
|
||||
WHERE schemaname = 'public'
|
||||
AND tablename = 'tiles'
|
||||
AND indexname = 'idx_tiles_unique_location';";
|
||||
AND tablename = 'tiles';";
|
||||
|
||||
await using var cmd = new NpgsqlCommand(sql, conn);
|
||||
var indexDef = (string?)await cmd.ExecuteScalarAsync();
|
||||
var rows = new List<(string Name, string Def)>();
|
||||
await using (var cmd = new NpgsqlCommand(sql, conn))
|
||||
await using (var reader = await cmd.ExecuteReaderAsync())
|
||||
{
|
||||
while (await reader.ReadAsync())
|
||||
{
|
||||
rows.Add((reader.GetString(0), reader.GetString(1)));
|
||||
}
|
||||
}
|
||||
|
||||
// Assert
|
||||
if (indexDef == null)
|
||||
var newIndex = rows.FirstOrDefault(r => string.Equals(r.Name, "idx_tiles_unique_location_source", StringComparison.Ordinal));
|
||||
if (newIndex.Def is null)
|
||||
{
|
||||
throw new Exception("AZ-357 AC-2: idx_tiles_unique_location does not exist on tiles table after migration 012");
|
||||
throw new Exception(
|
||||
"AZ-484 AC-1: expected unique index 'idx_tiles_unique_location_source' on tiles after migration 013, but it is not present. " +
|
||||
$"Found indexes: {string.Join(", ", rows.Select(r => r.Name))}");
|
||||
}
|
||||
|
||||
// Expected shape after migration 012 — 4 cols, no version, UNIQUE
|
||||
var lower = indexDef.ToLowerInvariant();
|
||||
var lower = newIndex.Def.ToLowerInvariant();
|
||||
if (!lower.Contains("unique"))
|
||||
{
|
||||
throw new Exception($"AZ-357 AC-2: idx_tiles_unique_location is not UNIQUE. Definition: {indexDef}");
|
||||
throw new Exception($"AZ-484 AC-1: idx_tiles_unique_location_source is not UNIQUE. Definition: {newIndex.Def}");
|
||||
}
|
||||
foreach (var col in new[] { "latitude", "longitude", "tile_zoom", "tile_size_meters" })
|
||||
foreach (var col in new[] { "latitude", "longitude", "tile_zoom", "tile_size_meters", "source" })
|
||||
{
|
||||
if (!lower.Contains(col))
|
||||
{
|
||||
throw new Exception($"AZ-357 AC-2: idx_tiles_unique_location missing column '{col}'. Definition: {indexDef}");
|
||||
throw new Exception($"AZ-484 AC-1: idx_tiles_unique_location_source missing column '{col}'. Definition: {newIndex.Def}");
|
||||
}
|
||||
}
|
||||
if (lower.Contains("version"))
|
||||
|
||||
var oldIndex = rows.FirstOrDefault(r => string.Equals(r.Name, "idx_tiles_unique_location", StringComparison.Ordinal));
|
||||
if (oldIndex.Def is not null)
|
||||
{
|
||||
throw new Exception($"AZ-357 AC-2: idx_tiles_unique_location still includes 'version' column — migration did not drop it. Definition: {indexDef}");
|
||||
throw new Exception(
|
||||
"AZ-484 AC-1: legacy 4-column index 'idx_tiles_unique_location' still exists after migration 013 — migration did not drop it. " +
|
||||
$"Definition: {oldIndex.Def}");
|
||||
}
|
||||
|
||||
Console.WriteLine($" ✓ Index present with new shape: {indexDef}");
|
||||
Console.WriteLine($" ✓ New 5-column unique index present: {newIndex.Def}");
|
||||
Console.WriteLine(" ✓ Legacy 4-column unique index dropped");
|
||||
}
|
||||
|
||||
private static async Task BackfillUpdateAssignsGoogleMapsAndCapturedAt_AZ484_AC4(string connectionString)
|
||||
{
|
||||
Console.WriteLine();
|
||||
Console.WriteLine("AZ-484 AC-4: backfill UPDATE assigns source='google_maps' and captured_at = created_at, preserving row count");
|
||||
|
||||
// Arrange — TEMP table simulating the pre-migration tiles shape with 3 sample rows.
|
||||
await using var conn = new NpgsqlConnection(connectionString);
|
||||
await conn.OpenAsync();
|
||||
|
||||
await ExecAsync(conn, """
|
||||
CREATE TEMP TABLE tiles_backfill_test (
|
||||
id UUID PRIMARY KEY,
|
||||
created_at TIMESTAMP NOT NULL,
|
||||
source VARCHAR(32),
|
||||
captured_at TIMESTAMP
|
||||
);
|
||||
""");
|
||||
|
||||
var idA = Guid.Parse("aaaaaaaa-1111-1111-1111-111111111111");
|
||||
var idB = Guid.Parse("bbbbbbbb-2222-2222-2222-222222222222");
|
||||
var idC = Guid.Parse("cccccccc-3333-3333-3333-333333333333");
|
||||
|
||||
await ExecAsync(conn, """
|
||||
INSERT INTO tiles_backfill_test (id, created_at) VALUES
|
||||
(@idA, '2024-01-15 12:34:56'),
|
||||
(@idB, '2025-06-20 03:00:00'),
|
||||
(@idC, '2026-05-11 06:00:00');
|
||||
""",
|
||||
("idA", idA), ("idB", idB), ("idC", idC));
|
||||
|
||||
// Act — apply the same UPDATE pattern that migration 013 uses.
|
||||
await ExecAsync(conn, """
|
||||
UPDATE tiles_backfill_test SET source = 'google_maps' WHERE source IS NULL;
|
||||
UPDATE tiles_backfill_test SET captured_at = created_at WHERE captured_at IS NULL;
|
||||
""");
|
||||
|
||||
// Assert
|
||||
var rows = new List<(Guid Id, string Source, DateTime CreatedAt, DateTime CapturedAt)>();
|
||||
await using (var cmd = new NpgsqlCommand(
|
||||
"SELECT id, source, created_at, captured_at FROM tiles_backfill_test ORDER BY id;", conn))
|
||||
await using (var reader = await cmd.ExecuteReaderAsync())
|
||||
{
|
||||
while (await reader.ReadAsync())
|
||||
{
|
||||
rows.Add((
|
||||
reader.GetGuid(0),
|
||||
reader.GetString(1),
|
||||
reader.GetDateTime(2),
|
||||
reader.GetDateTime(3)));
|
||||
}
|
||||
}
|
||||
|
||||
if (rows.Count != 3)
|
||||
{
|
||||
throw new Exception($"AZ-484 AC-4 backfill changed row count. Expected 3, got {rows.Count}.");
|
||||
}
|
||||
foreach (var row in rows)
|
||||
{
|
||||
if (row.Source != "google_maps")
|
||||
{
|
||||
throw new Exception($"AZ-484 AC-4: row {row.Id} has source='{row.Source}', expected 'google_maps'.");
|
||||
}
|
||||
if (row.CapturedAt != row.CreatedAt)
|
||||
{
|
||||
throw new Exception($"AZ-484 AC-4: row {row.Id} captured_at={row.CapturedAt:o} does not equal created_at={row.CreatedAt:o}.");
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine(" ✓ All 3 backfilled rows have source='google_maps' and captured_at = created_at");
|
||||
}
|
||||
|
||||
private static async Task MultiSourceInsertCoexistsUnderNewIndex_AZ484_AC1(string connectionString)
|
||||
{
|
||||
Console.WriteLine();
|
||||
Console.WriteLine("AZ-484 AC-1: per-source unique index lets two producers store distinct rows for the same cell");
|
||||
|
||||
// Arrange — TEMP table replicating the 5-column unique index shape.
|
||||
await using var conn = new NpgsqlConnection(connectionString);
|
||||
await conn.OpenAsync();
|
||||
|
||||
await CreateTempTilesTable(conn, "tiles_multisource_test");
|
||||
await ExecAsync(conn, """
|
||||
CREATE UNIQUE INDEX idx_tiles_multisource_test_unique
|
||||
ON tiles_multisource_test (latitude, longitude, tile_zoom, tile_size_meters, source);
|
||||
""");
|
||||
|
||||
var idGoogle = Guid.NewGuid();
|
||||
var idUav = Guid.NewGuid();
|
||||
|
||||
// Act
|
||||
await ExecAsync(conn, """
|
||||
INSERT INTO tiles_multisource_test (id, latitude, longitude, tile_zoom, tile_size_meters, source, captured_at, file_path, updated_at)
|
||||
VALUES (@id, 47.5, 37.6, 18, 100, 'google_maps', '2026-05-10 00:00:00', 'tiles/google.jpg', '2026-05-10 00:00:00');
|
||||
""", ("id", idGoogle));
|
||||
|
||||
await ExecAsync(conn, """
|
||||
INSERT INTO tiles_multisource_test (id, latitude, longitude, tile_zoom, tile_size_meters, source, captured_at, file_path, updated_at)
|
||||
VALUES (@id, 47.5, 37.6, 18, 100, 'uav', '2026-05-11 00:00:00', 'tiles/uav.jpg', '2026-05-11 00:00:00');
|
||||
""", ("id", idUav));
|
||||
|
||||
// Assert
|
||||
var rowCount = await ScalarLongAsync(conn,
|
||||
"SELECT COUNT(*) FROM tiles_multisource_test WHERE latitude = 47.5 AND longitude = 37.6 AND tile_zoom = 18 AND tile_size_meters = 100;");
|
||||
if (rowCount != 2)
|
||||
{
|
||||
throw new Exception($"AZ-484 AC-1: expected 2 rows for the cell after multi-source insert, got {rowCount}.");
|
||||
}
|
||||
|
||||
Console.WriteLine(" ✓ Both google_maps and uav rows coexist under the 5-column unique index");
|
||||
}
|
||||
|
||||
private static async Task MostRecentAcrossSourcesSelection_AZ484_AC2(string connectionString)
|
||||
{
|
||||
Console.WriteLine();
|
||||
Console.WriteLine("AZ-484 AC-2: most-recent-across-sources selection rule returns the latest captured_at row");
|
||||
|
||||
// Arrange — TEMP table with two rows for the same cell, distinct sources, T2 > T1.
|
||||
await using var conn = new NpgsqlConnection(connectionString);
|
||||
await conn.OpenAsync();
|
||||
|
||||
await CreateTempTilesTable(conn, "tiles_selection_test");
|
||||
|
||||
var idGoogleT1 = Guid.NewGuid();
|
||||
var idUavT2 = Guid.NewGuid();
|
||||
|
||||
await ExecAsync(conn, """
|
||||
INSERT INTO tiles_selection_test (id, latitude, longitude, tile_zoom, tile_size_meters, source, captured_at, file_path, updated_at)
|
||||
VALUES
|
||||
(@idG, 48.0, 38.0, 18, 100, 'google_maps', '2026-04-01 00:00:00', 'g.jpg', '2026-04-01 00:00:00'),
|
||||
(@idU, 48.0, 38.0, 18, 100, 'uav', '2026-05-01 00:00:00', 'u.jpg', '2026-05-01 00:00:00');
|
||||
""",
|
||||
("idG", idGoogleT1), ("idU", idUavT2));
|
||||
|
||||
// Act — same SELECT shape used by TileRepository.GetByTileCoordinatesAsync.
|
||||
Guid? winnerId;
|
||||
string? winnerSource;
|
||||
await using (var cmd = new NpgsqlCommand("""
|
||||
SELECT id, source FROM tiles_selection_test
|
||||
WHERE latitude = 48.0 AND longitude = 38.0 AND tile_zoom = 18 AND tile_size_meters = 100
|
||||
ORDER BY captured_at DESC, updated_at DESC, id DESC
|
||||
LIMIT 1;
|
||||
""", conn))
|
||||
await using (var reader = await cmd.ExecuteReaderAsync())
|
||||
{
|
||||
if (!await reader.ReadAsync())
|
||||
{
|
||||
throw new Exception("AZ-484 AC-2: selection query returned no rows; expected the uav row.");
|
||||
}
|
||||
winnerId = reader.GetGuid(0);
|
||||
winnerSource = reader.GetString(1);
|
||||
}
|
||||
|
||||
// Assert
|
||||
if (winnerId != idUavT2 || winnerSource != "uav")
|
||||
{
|
||||
throw new Exception(
|
||||
$"AZ-484 AC-2: expected uav row (id={idUavT2}) to win, got id={winnerId} source='{winnerSource}'.");
|
||||
}
|
||||
|
||||
Console.WriteLine(" ✓ Selection rule picked the uav row (captured_at T2 > T1) deterministically");
|
||||
}
|
||||
|
||||
private static async Task SameSourceUpsertReplacesPreviousRow_AZ484_AC3(string connectionString)
|
||||
{
|
||||
Console.WriteLine();
|
||||
Console.WriteLine("AZ-484 AC-3: same-source UPSERT keeps a single row with refreshed captured_at and file_path");
|
||||
|
||||
// Arrange — TEMP table with the 5-column unique index so ON CONFLICT works.
|
||||
await using var conn = new NpgsqlConnection(connectionString);
|
||||
await conn.OpenAsync();
|
||||
|
||||
await CreateTempTilesTable(conn, "tiles_upsert_test");
|
||||
await ExecAsync(conn, """
|
||||
CREATE UNIQUE INDEX idx_tiles_upsert_test_unique
|
||||
ON tiles_upsert_test (latitude, longitude, tile_zoom, tile_size_meters, source);
|
||||
""");
|
||||
|
||||
var idFirst = Guid.NewGuid();
|
||||
var idSecond = Guid.NewGuid();
|
||||
|
||||
await ExecAsync(conn, """
|
||||
INSERT INTO tiles_upsert_test (id, latitude, longitude, tile_zoom, tile_size_meters, source, captured_at, file_path, updated_at)
|
||||
VALUES (@id, 49.0, 39.0, 18, 100, 'uav', '2026-04-01 00:00:00', 'first.jpg', '2026-04-01 00:00:00');
|
||||
""", ("id", idFirst));
|
||||
|
||||
// Act — second insert for the same cell+source uses the same UPSERT pattern as TileRepository.InsertAsync.
|
||||
await ExecAsync(conn, """
|
||||
INSERT INTO tiles_upsert_test (id, latitude, longitude, tile_zoom, tile_size_meters, source, captured_at, file_path, updated_at)
|
||||
VALUES (@id, 49.0, 39.0, 18, 100, 'uav', '2026-05-01 00:00:00', 'second.jpg', '2026-05-01 00:00:00')
|
||||
ON CONFLICT (latitude, longitude, tile_zoom, tile_size_meters, source)
|
||||
DO UPDATE SET
|
||||
file_path = EXCLUDED.file_path,
|
||||
captured_at = EXCLUDED.captured_at,
|
||||
updated_at = EXCLUDED.updated_at;
|
||||
""", ("id", idSecond));
|
||||
|
||||
// Assert
|
||||
long rowCount = 0;
|
||||
DateTime capturedAt = DateTime.MinValue;
|
||||
string? filePath = null;
|
||||
await using (var cmd = new NpgsqlCommand("""
|
||||
SELECT COUNT(*) OVER () AS total, captured_at, file_path
|
||||
FROM tiles_upsert_test
|
||||
WHERE latitude = 49.0 AND longitude = 39.0 AND tile_zoom = 18 AND tile_size_meters = 100 AND source = 'uav';
|
||||
""", conn))
|
||||
await using (var reader = await cmd.ExecuteReaderAsync())
|
||||
{
|
||||
if (!await reader.ReadAsync())
|
||||
{
|
||||
throw new Exception("AZ-484 AC-3: no rows after UPSERT — expected exactly one.");
|
||||
}
|
||||
rowCount = reader.GetInt64(0);
|
||||
capturedAt = reader.GetDateTime(1);
|
||||
filePath = reader.GetString(2);
|
||||
}
|
||||
|
||||
if (rowCount != 1)
|
||||
{
|
||||
throw new Exception($"AZ-484 AC-3: expected exactly 1 uav row after UPSERT, got {rowCount}.");
|
||||
}
|
||||
if (capturedAt != new DateTime(2026, 5, 1, 0, 0, 0, DateTimeKind.Unspecified))
|
||||
{
|
||||
throw new Exception($"AZ-484 AC-3: captured_at not refreshed. Got {capturedAt:o}, expected 2026-05-01.");
|
||||
}
|
||||
if (filePath != "second.jpg")
|
||||
{
|
||||
throw new Exception($"AZ-484 AC-3: file_path not refreshed. Got '{filePath}', expected 'second.jpg'.");
|
||||
}
|
||||
|
||||
Console.WriteLine(" ✓ Same-source UPSERT collapsed to 1 row with refreshed captured_at and file_path");
|
||||
}
|
||||
|
||||
private static async Task CreateTempTilesTable(NpgsqlConnection conn, string tableName)
|
||||
{
|
||||
// Mirrors the post-migration-013 column shape relevant to AZ-484 (omits
|
||||
// vestigial maps_version/version columns that AC-1..AC-3 do not exercise).
|
||||
await ExecAsync(conn, $$"""
|
||||
CREATE TEMP TABLE {{tableName}} (
|
||||
id UUID PRIMARY KEY,
|
||||
latitude DOUBLE PRECISION NOT NULL,
|
||||
longitude DOUBLE PRECISION NOT NULL,
|
||||
tile_zoom INT NOT NULL,
|
||||
tile_size_meters DOUBLE PRECISION NOT NULL,
|
||||
source VARCHAR(32) NOT NULL,
|
||||
captured_at TIMESTAMP NOT NULL,
|
||||
file_path VARCHAR(500) NOT NULL,
|
||||
updated_at TIMESTAMP NOT NULL
|
||||
);
|
||||
""");
|
||||
}
|
||||
|
||||
private static async Task<long> ScalarLongAsync(NpgsqlConnection conn, string sql)
|
||||
{
|
||||
await using var cmd = new NpgsqlCommand(sql, conn);
|
||||
var result = await cmd.ExecuteScalarAsync();
|
||||
return result switch
|
||||
{
|
||||
long l => l,
|
||||
int i => i,
|
||||
_ => throw new Exception($"Unexpected scalar type {result?.GetType()} for SQL: {sql}"),
|
||||
};
|
||||
}
|
||||
|
||||
private static async Task ExecAsync(NpgsqlConnection conn, string sql, params (string Name, object Value)[] parameters)
|
||||
|
||||
Reference in New Issue
Block a user