diff --git a/SatelliteProvider.Common/Enums/TileSource.cs b/SatelliteProvider.Common/Enums/TileSource.cs new file mode 100644 index 0000000..9f4950b --- /dev/null +++ b/SatelliteProvider.Common/Enums/TileSource.cs @@ -0,0 +1,7 @@ +namespace SatelliteProvider.Common.Enums; + +public enum TileSource +{ + GoogleMaps, + Uav, +} diff --git a/SatelliteProvider.DataAccess/Migrations/013_AddTileSourceAndCapturedAt.sql b/SatelliteProvider.DataAccess/Migrations/013_AddTileSourceAndCapturedAt.sql new file mode 100644 index 0000000..e61f70b --- /dev/null +++ b/SatelliteProvider.DataAccess/Migrations/013_AddTileSourceAndCapturedAt.sql @@ -0,0 +1,31 @@ +-- AZ-484: introduce per-source tile rows. +-- Adds `source` and `captured_at` columns, backfills existing rows to +-- (source='google_maps', captured_at=created_at), drops the 4-column unique index +-- created by migration 012 (idx_tiles_unique_location), and replaces it with a +-- 5-column unique index that includes `source`. The whole migration runs inside a +-- single transaction so a failure mid-flight cannot leave the table without its +-- unique index or with partially backfilled rows (per coderule.mdc and AZ-484 +-- Risk 1 mitigation). + +BEGIN; + +ALTER TABLE tiles ADD COLUMN IF NOT EXISTS source VARCHAR(32); +ALTER TABLE tiles ADD COLUMN IF NOT EXISTS captured_at TIMESTAMP; + +UPDATE tiles +SET source = 'google_maps' +WHERE source IS NULL; + +UPDATE tiles +SET captured_at = created_at +WHERE captured_at IS NULL; + +ALTER TABLE tiles ALTER COLUMN source SET NOT NULL; +ALTER TABLE tiles ALTER COLUMN captured_at SET NOT NULL; + +DROP INDEX IF EXISTS idx_tiles_unique_location; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_tiles_unique_location_source + ON tiles (latitude, longitude, tile_zoom, tile_size_meters, source); + +COMMIT; diff --git a/SatelliteProvider.DataAccess/Models/TileEntity.cs b/SatelliteProvider.DataAccess/Models/TileEntity.cs index c49cc13..b69c038 100644 --- a/SatelliteProvider.DataAccess/Models/TileEntity.cs +++ b/SatelliteProvider.DataAccess/Models/TileEntity.cs @@ -1,3 +1,5 @@ +using SatelliteProvider.Common.Enums; + namespace SatelliteProvider.DataAccess.Models; public class TileEntity @@ -14,6 +16,8 @@ public class TileEntity public string? MapsVersion { get; set; } public int? Version { get; set; } public string FilePath { get; set; } = string.Empty; + public TileSource Source { get; set; } + public DateTime CapturedAt { get; set; } public DateTime CreatedAt { get; set; } public DateTime UpdatedAt { get; set; } } diff --git a/SatelliteProvider.DataAccess/Repositories/TileRepository.cs b/SatelliteProvider.DataAccess/Repositories/TileRepository.cs index a93d03e..91bcad7 100644 --- a/SatelliteProvider.DataAccess/Repositories/TileRepository.cs +++ b/SatelliteProvider.DataAccess/Repositories/TileRepository.cs @@ -16,7 +16,8 @@ public class TileRepository : ITileRepository latitude, longitude, tile_size_meters as TileSizeMeters, tile_size_pixels as TileSizePixels, image_type as ImageType, maps_version as MapsVersion, version, - file_path as FilePath, created_at as CreatedAt, updated_at as UpdatedAt"; + file_path as FilePath, source, captured_at as CapturedAt, + created_at as CreatedAt, updated_at as UpdatedAt"; private readonly string _connectionString; private readonly ILogger _logger; @@ -41,11 +42,13 @@ public class TileRepository : ITileRepository public async Task GetByTileCoordinatesAsync(int tileZoom, int tileX, int tileY) { using var connection = new NpgsqlConnection(_connectionString); + // AZ-484 selection rule: most-recent across sources, deterministic tie-break on + // (captured_at DESC, updated_at DESC, id DESC). const string sql = $@" SELECT {ColumnList} FROM tiles WHERE tile_zoom = @TileZoom AND tile_x = @TileX AND tile_y = @TileY - ORDER BY updated_at DESC + ORDER BY captured_at DESC, updated_at DESC, id DESC LIMIT 1"; return await connection.QuerySingleOrDefaultAsync(sql, new { TileZoom = tileZoom, TileX = tileX, TileY = tileY }); @@ -64,12 +67,21 @@ public class TileRepository : ITileRepository var latRange = expandedSizeMeters / GeoUtils.MetersPerDegreeLatitude; var lonRange = expandedSizeMeters / (GeoUtils.MetersPerDegreeLatitude * Math.Cos(latitude * Math.PI / 180.0)); + // AZ-484 selection rule: at most one row per (lat, lon, zoom, size) cell, picking + // the most-recent across sources via DISTINCT ON, with deterministic tie-break on + // (captured_at DESC, updated_at DESC, id DESC). The outer ORDER BY restores the + // pre-AZ-484 caller-facing order (latitude DESC, longitude ASC, updated_at DESC). const string sql = $@" - SELECT {ColumnList} - FROM tiles - WHERE latitude BETWEEN @MinLat AND @MaxLat - AND longitude BETWEEN @MinLon AND @MaxLon - AND tile_zoom = @TileZoom + SELECT * FROM ( + SELECT DISTINCT ON (latitude, longitude, tile_zoom, tile_size_meters) + {ColumnList} + FROM tiles + WHERE latitude BETWEEN @MinLat AND @MaxLat + AND longitude BETWEEN @MinLon AND @MaxLon + AND tile_zoom = @TileZoom + ORDER BY latitude, longitude, tile_zoom, tile_size_meters, + captured_at DESC, updated_at DESC, id DESC + ) deduped ORDER BY latitude DESC, longitude ASC, updated_at DESC"; var stopwatch = Stopwatch.StartNew(); @@ -96,18 +108,23 @@ public class TileRepository : ITileRepository public async Task InsertAsync(TileEntity tile) { using var connection = new NpgsqlConnection(_connectionString); + // AZ-484: per-source UPSERT — conflict key now includes `source` so that two + // producers (e.g. google_maps + uav) can coexist for the same cell. A re-insert + // for the SAME source updates file_path / tile_x / tile_y plus refreshes + // captured_at and updated_at to reflect the new acquisition. const string sql = @" - INSERT INTO tiles (id, tile_zoom, tile_x, tile_y, latitude, longitude, tile_size_meters, - tile_size_pixels, image_type, maps_version, version, file_path, - created_at, updated_at) - VALUES (@Id, @TileZoom, @TileX, @TileY, @Latitude, @Longitude, @TileSizeMeters, - @TileSizePixels, @ImageType, @MapsVersion, @Version, @FilePath, - @CreatedAt, @UpdatedAt) - ON CONFLICT (latitude, longitude, tile_zoom, tile_size_meters) - DO UPDATE SET + INSERT INTO tiles (id, tile_zoom, tile_x, tile_y, latitude, longitude, tile_size_meters, + tile_size_pixels, image_type, maps_version, version, file_path, + source, captured_at, created_at, updated_at) + VALUES (@Id, @TileZoom, @TileX, @TileY, @Latitude, @Longitude, @TileSizeMeters, + @TileSizePixels, @ImageType, @MapsVersion, @Version, @FilePath, + @Source, @CapturedAt, @CreatedAt, @UpdatedAt) + ON CONFLICT (latitude, longitude, tile_zoom, tile_size_meters, source) + DO UPDATE SET file_path = EXCLUDED.file_path, tile_x = EXCLUDED.tile_x, tile_y = EXCLUDED.tile_y, + captured_at = EXCLUDED.captured_at, updated_at = EXCLUDED.updated_at RETURNING id"; @@ -118,11 +135,11 @@ public class TileRepository : ITileRepository { using var connection = new NpgsqlConnection(_connectionString); const string sql = @" - UPDATE tiles + UPDATE tiles SET tile_zoom = @TileZoom, tile_x = @TileX, tile_y = @TileY, - latitude = @Latitude, + latitude = @Latitude, longitude = @Longitude, tile_size_meters = @TileSizeMeters, tile_size_pixels = @TileSizePixels, @@ -130,6 +147,8 @@ public class TileRepository : ITileRepository maps_version = @MapsVersion, version = @Version, file_path = @FilePath, + source = @Source, + captured_at = @CapturedAt, updated_at = @UpdatedAt WHERE id = @Id"; diff --git a/SatelliteProvider.DataAccess/TypeHandlers/EnumStringTypeHandler.cs b/SatelliteProvider.DataAccess/TypeHandlers/EnumStringTypeHandler.cs index 221acb8..db9a138 100644 --- a/SatelliteProvider.DataAccess/TypeHandlers/EnumStringTypeHandler.cs +++ b/SatelliteProvider.DataAccess/TypeHandlers/EnumStringTypeHandler.cs @@ -47,5 +47,6 @@ public static class DapperEnumTypeHandlers SqlMapper.AddTypeHandler(new EnumStringTypeHandler()); SqlMapper.AddTypeHandler(new EnumStringTypeHandler()); + SqlMapper.AddTypeHandler(new TileSourceTypeHandler()); } } diff --git a/SatelliteProvider.DataAccess/TypeHandlers/TileSourceTypeHandler.cs b/SatelliteProvider.DataAccess/TypeHandlers/TileSourceTypeHandler.cs new file mode 100644 index 0000000..9fcdf77 --- /dev/null +++ b/SatelliteProvider.DataAccess/TypeHandlers/TileSourceTypeHandler.cs @@ -0,0 +1,49 @@ +using System.Data; +using Dapper; +using SatelliteProvider.Common.Enums; + +namespace SatelliteProvider.DataAccess.TypeHandlers; + +// AZ-484: TileSource needs an explicit string mapping because the multi-word +// member 'GoogleMaps' must round-trip as the snake_case wire value 'google_maps' +// per the v1.0.0 tile-storage contract. The generic EnumStringTypeHandler +// only does ToString().ToLowerInvariant(), which would emit 'googlemaps'. +public class TileSourceTypeHandler : SqlMapper.TypeHandler +{ + public const string GoogleMapsWireValue = "google_maps"; + public const string UavWireValue = "uav"; + + public override TileSource Parse(object value) + { + if (value is null || value is DBNull) + { + throw new DataException("Cannot parse null DB value into enum TileSource"); + } + + var s = value as string ?? value.ToString(); + if (string.IsNullOrEmpty(s)) + { + throw new DataException("Cannot parse empty DB value into enum TileSource"); + } + + return s.ToLowerInvariant() switch + { + GoogleMapsWireValue => TileSource.GoogleMaps, + UavWireValue => TileSource.Uav, + _ => throw new DataException($"DB value '{s}' is not a defined member of enum TileSource"), + }; + } + + public override void SetValue(IDbDataParameter parameter, TileSource value) + { + parameter.Value = ToWireValue(value); + parameter.DbType = DbType.String; + } + + public static string ToWireValue(TileSource value) => value switch + { + TileSource.GoogleMaps => GoogleMapsWireValue, + TileSource.Uav => UavWireValue, + _ => throw new ArgumentOutOfRangeException(nameof(value), value, "Unknown TileSource"), + }; +} diff --git a/SatelliteProvider.IntegrationTests/MigrationTests.cs b/SatelliteProvider.IntegrationTests/MigrationTests.cs index c00fd22..cb6fe43 100644 --- a/SatelliteProvider.IntegrationTests/MigrationTests.cs +++ b/SatelliteProvider.IntegrationTests/MigrationTests.cs @@ -15,9 +15,21 @@ public static class MigrationTests ?? "Host=postgres;Port=5432;Database=satelliteprovider;Username=postgres;Password=postgres"; await DedupeSqlCollapsesDuplicatesByLatestUpdatedAt_AZ357_AC2(connectionString); - await NewUniqueConstraintExistsOnFourColumns_AZ357_AC2(connectionString); Console.WriteLine("✓ Migration 012 tests: PASSED"); + + Console.WriteLine(); + Console.WriteLine("Test: Migration 013 (AZ-484)"); + Console.WriteLine("============================"); + Console.WriteLine(); + + await BackfillUpdateAssignsGoogleMapsAndCapturedAt_AZ484_AC4(connectionString); + await MultiSourceInsertCoexistsUnderNewIndex_AZ484_AC1(connectionString); + await MostRecentAcrossSourcesSelection_AZ484_AC2(connectionString); + await SameSourceUpsertReplacesPreviousRow_AZ484_AC3(connectionString); + await NewUniqueConstraintIncludesSourceColumn_AZ484_AC1(connectionString); + + Console.WriteLine("✓ Migration 013 tests: PASSED"); } private static async Task DedupeSqlCollapsesDuplicatesByLatestUpdatedAt_AZ357_AC2(string connectionString) @@ -103,50 +115,326 @@ public static class MigrationTests Console.WriteLine(" ✓ Unique row (idF) preserved"); } - private static async Task NewUniqueConstraintExistsOnFourColumns_AZ357_AC2(string connectionString) + private static async Task NewUniqueConstraintIncludesSourceColumn_AZ484_AC1(string connectionString) { Console.WriteLine(); - Console.WriteLine("AZ-357 AC-2 part 2: post-migration unique index has the new 4-column shape"); + Console.WriteLine("AZ-484 AC-1 part 2: post-migration-013 unique index includes the source column"); // Arrange / Act await using var conn = new NpgsqlConnection(connectionString); await conn.OpenAsync(); const string sql = @" - SELECT indexdef + SELECT indexname, indexdef FROM pg_indexes WHERE schemaname = 'public' - AND tablename = 'tiles' - AND indexname = 'idx_tiles_unique_location';"; + AND tablename = 'tiles';"; - await using var cmd = new NpgsqlCommand(sql, conn); - var indexDef = (string?)await cmd.ExecuteScalarAsync(); + var rows = new List<(string Name, string Def)>(); + await using (var cmd = new NpgsqlCommand(sql, conn)) + await using (var reader = await cmd.ExecuteReaderAsync()) + { + while (await reader.ReadAsync()) + { + rows.Add((reader.GetString(0), reader.GetString(1))); + } + } // Assert - if (indexDef == null) + var newIndex = rows.FirstOrDefault(r => string.Equals(r.Name, "idx_tiles_unique_location_source", StringComparison.Ordinal)); + if (newIndex.Def is null) { - throw new Exception("AZ-357 AC-2: idx_tiles_unique_location does not exist on tiles table after migration 012"); + throw new Exception( + "AZ-484 AC-1: expected unique index 'idx_tiles_unique_location_source' on tiles after migration 013, but it is not present. " + + $"Found indexes: {string.Join(", ", rows.Select(r => r.Name))}"); } - // Expected shape after migration 012 — 4 cols, no version, UNIQUE - var lower = indexDef.ToLowerInvariant(); + var lower = newIndex.Def.ToLowerInvariant(); if (!lower.Contains("unique")) { - throw new Exception($"AZ-357 AC-2: idx_tiles_unique_location is not UNIQUE. Definition: {indexDef}"); + throw new Exception($"AZ-484 AC-1: idx_tiles_unique_location_source is not UNIQUE. Definition: {newIndex.Def}"); } - foreach (var col in new[] { "latitude", "longitude", "tile_zoom", "tile_size_meters" }) + foreach (var col in new[] { "latitude", "longitude", "tile_zoom", "tile_size_meters", "source" }) { if (!lower.Contains(col)) { - throw new Exception($"AZ-357 AC-2: idx_tiles_unique_location missing column '{col}'. Definition: {indexDef}"); + throw new Exception($"AZ-484 AC-1: idx_tiles_unique_location_source missing column '{col}'. Definition: {newIndex.Def}"); } } - if (lower.Contains("version")) + + var oldIndex = rows.FirstOrDefault(r => string.Equals(r.Name, "idx_tiles_unique_location", StringComparison.Ordinal)); + if (oldIndex.Def is not null) { - throw new Exception($"AZ-357 AC-2: idx_tiles_unique_location still includes 'version' column — migration did not drop it. Definition: {indexDef}"); + throw new Exception( + "AZ-484 AC-1: legacy 4-column index 'idx_tiles_unique_location' still exists after migration 013 — migration did not drop it. " + + $"Definition: {oldIndex.Def}"); } - Console.WriteLine($" ✓ Index present with new shape: {indexDef}"); + Console.WriteLine($" ✓ New 5-column unique index present: {newIndex.Def}"); + Console.WriteLine(" ✓ Legacy 4-column unique index dropped"); + } + + private static async Task BackfillUpdateAssignsGoogleMapsAndCapturedAt_AZ484_AC4(string connectionString) + { + Console.WriteLine(); + Console.WriteLine("AZ-484 AC-4: backfill UPDATE assigns source='google_maps' and captured_at = created_at, preserving row count"); + + // Arrange — TEMP table simulating the pre-migration tiles shape with 3 sample rows. + await using var conn = new NpgsqlConnection(connectionString); + await conn.OpenAsync(); + + await ExecAsync(conn, """ + CREATE TEMP TABLE tiles_backfill_test ( + id UUID PRIMARY KEY, + created_at TIMESTAMP NOT NULL, + source VARCHAR(32), + captured_at TIMESTAMP + ); + """); + + var idA = Guid.Parse("aaaaaaaa-1111-1111-1111-111111111111"); + var idB = Guid.Parse("bbbbbbbb-2222-2222-2222-222222222222"); + var idC = Guid.Parse("cccccccc-3333-3333-3333-333333333333"); + + await ExecAsync(conn, """ + INSERT INTO tiles_backfill_test (id, created_at) VALUES + (@idA, '2024-01-15 12:34:56'), + (@idB, '2025-06-20 03:00:00'), + (@idC, '2026-05-11 06:00:00'); + """, + ("idA", idA), ("idB", idB), ("idC", idC)); + + // Act — apply the same UPDATE pattern that migration 013 uses. + await ExecAsync(conn, """ + UPDATE tiles_backfill_test SET source = 'google_maps' WHERE source IS NULL; + UPDATE tiles_backfill_test SET captured_at = created_at WHERE captured_at IS NULL; + """); + + // Assert + var rows = new List<(Guid Id, string Source, DateTime CreatedAt, DateTime CapturedAt)>(); + await using (var cmd = new NpgsqlCommand( + "SELECT id, source, created_at, captured_at FROM tiles_backfill_test ORDER BY id;", conn)) + await using (var reader = await cmd.ExecuteReaderAsync()) + { + while (await reader.ReadAsync()) + { + rows.Add(( + reader.GetGuid(0), + reader.GetString(1), + reader.GetDateTime(2), + reader.GetDateTime(3))); + } + } + + if (rows.Count != 3) + { + throw new Exception($"AZ-484 AC-4 backfill changed row count. Expected 3, got {rows.Count}."); + } + foreach (var row in rows) + { + if (row.Source != "google_maps") + { + throw new Exception($"AZ-484 AC-4: row {row.Id} has source='{row.Source}', expected 'google_maps'."); + } + if (row.CapturedAt != row.CreatedAt) + { + throw new Exception($"AZ-484 AC-4: row {row.Id} captured_at={row.CapturedAt:o} does not equal created_at={row.CreatedAt:o}."); + } + } + + Console.WriteLine(" ✓ All 3 backfilled rows have source='google_maps' and captured_at = created_at"); + } + + private static async Task MultiSourceInsertCoexistsUnderNewIndex_AZ484_AC1(string connectionString) + { + Console.WriteLine(); + Console.WriteLine("AZ-484 AC-1: per-source unique index lets two producers store distinct rows for the same cell"); + + // Arrange — TEMP table replicating the 5-column unique index shape. + await using var conn = new NpgsqlConnection(connectionString); + await conn.OpenAsync(); + + await CreateTempTilesTable(conn, "tiles_multisource_test"); + await ExecAsync(conn, """ + CREATE UNIQUE INDEX idx_tiles_multisource_test_unique + ON tiles_multisource_test (latitude, longitude, tile_zoom, tile_size_meters, source); + """); + + var idGoogle = Guid.NewGuid(); + var idUav = Guid.NewGuid(); + + // Act + await ExecAsync(conn, """ + INSERT INTO tiles_multisource_test (id, latitude, longitude, tile_zoom, tile_size_meters, source, captured_at, file_path, updated_at) + VALUES (@id, 47.5, 37.6, 18, 100, 'google_maps', '2026-05-10 00:00:00', 'tiles/google.jpg', '2026-05-10 00:00:00'); + """, ("id", idGoogle)); + + await ExecAsync(conn, """ + INSERT INTO tiles_multisource_test (id, latitude, longitude, tile_zoom, tile_size_meters, source, captured_at, file_path, updated_at) + VALUES (@id, 47.5, 37.6, 18, 100, 'uav', '2026-05-11 00:00:00', 'tiles/uav.jpg', '2026-05-11 00:00:00'); + """, ("id", idUav)); + + // Assert + var rowCount = await ScalarLongAsync(conn, + "SELECT COUNT(*) FROM tiles_multisource_test WHERE latitude = 47.5 AND longitude = 37.6 AND tile_zoom = 18 AND tile_size_meters = 100;"); + if (rowCount != 2) + { + throw new Exception($"AZ-484 AC-1: expected 2 rows for the cell after multi-source insert, got {rowCount}."); + } + + Console.WriteLine(" ✓ Both google_maps and uav rows coexist under the 5-column unique index"); + } + + private static async Task MostRecentAcrossSourcesSelection_AZ484_AC2(string connectionString) + { + Console.WriteLine(); + Console.WriteLine("AZ-484 AC-2: most-recent-across-sources selection rule returns the latest captured_at row"); + + // Arrange — TEMP table with two rows for the same cell, distinct sources, T2 > T1. + await using var conn = new NpgsqlConnection(connectionString); + await conn.OpenAsync(); + + await CreateTempTilesTable(conn, "tiles_selection_test"); + + var idGoogleT1 = Guid.NewGuid(); + var idUavT2 = Guid.NewGuid(); + + await ExecAsync(conn, """ + INSERT INTO tiles_selection_test (id, latitude, longitude, tile_zoom, tile_size_meters, source, captured_at, file_path, updated_at) + VALUES + (@idG, 48.0, 38.0, 18, 100, 'google_maps', '2026-04-01 00:00:00', 'g.jpg', '2026-04-01 00:00:00'), + (@idU, 48.0, 38.0, 18, 100, 'uav', '2026-05-01 00:00:00', 'u.jpg', '2026-05-01 00:00:00'); + """, + ("idG", idGoogleT1), ("idU", idUavT2)); + + // Act — same SELECT shape used by TileRepository.GetByTileCoordinatesAsync. + Guid? winnerId; + string? winnerSource; + await using (var cmd = new NpgsqlCommand(""" + SELECT id, source FROM tiles_selection_test + WHERE latitude = 48.0 AND longitude = 38.0 AND tile_zoom = 18 AND tile_size_meters = 100 + ORDER BY captured_at DESC, updated_at DESC, id DESC + LIMIT 1; + """, conn)) + await using (var reader = await cmd.ExecuteReaderAsync()) + { + if (!await reader.ReadAsync()) + { + throw new Exception("AZ-484 AC-2: selection query returned no rows; expected the uav row."); + } + winnerId = reader.GetGuid(0); + winnerSource = reader.GetString(1); + } + + // Assert + if (winnerId != idUavT2 || winnerSource != "uav") + { + throw new Exception( + $"AZ-484 AC-2: expected uav row (id={idUavT2}) to win, got id={winnerId} source='{winnerSource}'."); + } + + Console.WriteLine(" ✓ Selection rule picked the uav row (captured_at T2 > T1) deterministically"); + } + + private static async Task SameSourceUpsertReplacesPreviousRow_AZ484_AC3(string connectionString) + { + Console.WriteLine(); + Console.WriteLine("AZ-484 AC-3: same-source UPSERT keeps a single row with refreshed captured_at and file_path"); + + // Arrange — TEMP table with the 5-column unique index so ON CONFLICT works. + await using var conn = new NpgsqlConnection(connectionString); + await conn.OpenAsync(); + + await CreateTempTilesTable(conn, "tiles_upsert_test"); + await ExecAsync(conn, """ + CREATE UNIQUE INDEX idx_tiles_upsert_test_unique + ON tiles_upsert_test (latitude, longitude, tile_zoom, tile_size_meters, source); + """); + + var idFirst = Guid.NewGuid(); + var idSecond = Guid.NewGuid(); + + await ExecAsync(conn, """ + INSERT INTO tiles_upsert_test (id, latitude, longitude, tile_zoom, tile_size_meters, source, captured_at, file_path, updated_at) + VALUES (@id, 49.0, 39.0, 18, 100, 'uav', '2026-04-01 00:00:00', 'first.jpg', '2026-04-01 00:00:00'); + """, ("id", idFirst)); + + // Act — second insert for the same cell+source uses the same UPSERT pattern as TileRepository.InsertAsync. + await ExecAsync(conn, """ + INSERT INTO tiles_upsert_test (id, latitude, longitude, tile_zoom, tile_size_meters, source, captured_at, file_path, updated_at) + VALUES (@id, 49.0, 39.0, 18, 100, 'uav', '2026-05-01 00:00:00', 'second.jpg', '2026-05-01 00:00:00') + ON CONFLICT (latitude, longitude, tile_zoom, tile_size_meters, source) + DO UPDATE SET + file_path = EXCLUDED.file_path, + captured_at = EXCLUDED.captured_at, + updated_at = EXCLUDED.updated_at; + """, ("id", idSecond)); + + // Assert + long rowCount = 0; + DateTime capturedAt = DateTime.MinValue; + string? filePath = null; + await using (var cmd = new NpgsqlCommand(""" + SELECT COUNT(*) OVER () AS total, captured_at, file_path + FROM tiles_upsert_test + WHERE latitude = 49.0 AND longitude = 39.0 AND tile_zoom = 18 AND tile_size_meters = 100 AND source = 'uav'; + """, conn)) + await using (var reader = await cmd.ExecuteReaderAsync()) + { + if (!await reader.ReadAsync()) + { + throw new Exception("AZ-484 AC-3: no rows after UPSERT — expected exactly one."); + } + rowCount = reader.GetInt64(0); + capturedAt = reader.GetDateTime(1); + filePath = reader.GetString(2); + } + + if (rowCount != 1) + { + throw new Exception($"AZ-484 AC-3: expected exactly 1 uav row after UPSERT, got {rowCount}."); + } + if (capturedAt != new DateTime(2026, 5, 1, 0, 0, 0, DateTimeKind.Unspecified)) + { + throw new Exception($"AZ-484 AC-3: captured_at not refreshed. Got {capturedAt:o}, expected 2026-05-01."); + } + if (filePath != "second.jpg") + { + throw new Exception($"AZ-484 AC-3: file_path not refreshed. Got '{filePath}', expected 'second.jpg'."); + } + + Console.WriteLine(" ✓ Same-source UPSERT collapsed to 1 row with refreshed captured_at and file_path"); + } + + private static async Task CreateTempTilesTable(NpgsqlConnection conn, string tableName) + { + // Mirrors the post-migration-013 column shape relevant to AZ-484 (omits + // vestigial maps_version/version columns that AC-1..AC-3 do not exercise). + await ExecAsync(conn, $$""" + CREATE TEMP TABLE {{tableName}} ( + id UUID PRIMARY KEY, + latitude DOUBLE PRECISION NOT NULL, + longitude DOUBLE PRECISION NOT NULL, + tile_zoom INT NOT NULL, + tile_size_meters DOUBLE PRECISION NOT NULL, + source VARCHAR(32) NOT NULL, + captured_at TIMESTAMP NOT NULL, + file_path VARCHAR(500) NOT NULL, + updated_at TIMESTAMP NOT NULL + ); + """); + } + + private static async Task ScalarLongAsync(NpgsqlConnection conn, string sql) + { + await using var cmd = new NpgsqlCommand(sql, conn); + var result = await cmd.ExecuteScalarAsync(); + return result switch + { + long l => l, + int i => i, + _ => throw new Exception($"Unexpected scalar type {result?.GetType()} for SQL: {sql}"), + }; } private static async Task ExecAsync(NpgsqlConnection conn, string sql, params (string Name, object Value)[] parameters) diff --git a/SatelliteProvider.Services.TileDownloader/TileService.cs b/SatelliteProvider.Services.TileDownloader/TileService.cs index 106cdca..045a68f 100644 --- a/SatelliteProvider.Services.TileDownloader/TileService.cs +++ b/SatelliteProvider.Services.TileDownloader/TileService.cs @@ -3,6 +3,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using SatelliteProvider.Common.Configs; using SatelliteProvider.Common.DTO; +using SatelliteProvider.Common.Enums; using SatelliteProvider.Common.Interfaces; using SatelliteProvider.Common.Utils; using SatelliteProvider.DataAccess.Models; @@ -157,6 +158,8 @@ public class TileService : ITileService MapsVersion = null, Version = null, FilePath = downloaded.FilePath, + Source = TileSource.GoogleMaps, + CapturedAt = now, CreatedAt = now, UpdatedAt = now }; diff --git a/SatelliteProvider.Tests/EnumStringTypeHandlerTests.cs b/SatelliteProvider.Tests/EnumStringTypeHandlerTests.cs index 58b2c06..c040cb0 100644 --- a/SatelliteProvider.Tests/EnumStringTypeHandlerTests.cs +++ b/SatelliteProvider.Tests/EnumStringTypeHandlerTests.cs @@ -137,4 +137,107 @@ public class EnumStringTypeHandlerTests DapperEnumTypeHandlers.RegisterAll(); DapperEnumTypeHandlers.RegisterAll(); } + + [Theory] + [InlineData(TileSource.GoogleMaps, "google_maps")] + [InlineData(TileSource.Uav, "uav")] + public void TileSourceHandler_SetValue_WritesContractWireValue_AZ484(TileSource value, string expected) + { + // Arrange + var handler = new TileSourceTypeHandler(); + var param = new NpgsqlParameter(); + + // Act + handler.SetValue(param, value); + + // Assert + param.Value.Should().Be(expected); + param.DbType.Should().Be(DbType.String); + } + + [Theory] + [InlineData("google_maps", TileSource.GoogleMaps)] + [InlineData("GOOGLE_MAPS", TileSource.GoogleMaps)] + [InlineData("uav", TileSource.Uav)] + [InlineData("UAV", TileSource.Uav)] + public void TileSourceHandler_Parse_AcceptsContractWireValue_AZ484(string raw, TileSource expected) + { + // Arrange + var handler = new TileSourceTypeHandler(); + + // Act + var result = handler.Parse(raw); + + // Assert + result.Should().Be(expected); + } + + [Theory] + [InlineData(TileSource.GoogleMaps)] + [InlineData(TileSource.Uav)] + public void TileSourceHandler_RoundTrip_PreservesValue_AZ484(TileSource value) + { + // Arrange + var handler = new TileSourceTypeHandler(); + var param = new NpgsqlParameter(); + handler.SetValue(param, value); + + // Act + var roundTripped = handler.Parse(param.Value!); + + // Assert + roundTripped.Should().Be(value); + } + + [Fact] + public void TileSourceHandler_Parse_UnknownString_ThrowsDataException_AZ484() + { + // Arrange + var handler = new TileSourceTypeHandler(); + + // Act + Action act = () => handler.Parse("satar"); + + // Assert — Inv-1: unknown sources surface, not silently coerce (coderule.mdc: never suppress errors). + act.Should().Throw().WithMessage("*satar*"); + } + + [Fact] + public void TileSourceHandler_Parse_NullValue_ThrowsDataException_AZ484() + { + // Arrange + var handler = new TileSourceTypeHandler(); + + // Act + Action act = () => handler.Parse(null!); + + // Assert + act.Should().Throw(); + } + + [Fact] + public void TileSourceHandler_Parse_DbNullValue_ThrowsDataException_AZ484() + { + // Arrange + var handler = new TileSourceTypeHandler(); + + // Act + Action act = () => handler.Parse(DBNull.Value); + + // Assert + act.Should().Throw(); + } + + [Fact] + public void RegisterAll_RegistersTileSourceHandler_AZ484() + { + // Arrange / Act + DapperEnumTypeHandlers.RegisterAll(); + + // Assert — registration is idempotent and the handler emits the contract wire value. + var probe = new TileSourceTypeHandler(); + var param = new NpgsqlParameter(); + probe.SetValue(param, TileSource.GoogleMaps); + param.Value.Should().Be("google_maps"); + } } diff --git a/SatelliteProvider.Tests/RepositoryRefactorTests.cs b/SatelliteProvider.Tests/RepositoryRefactorTests.cs index 9fa8f71..d4663cd 100644 --- a/SatelliteProvider.Tests/RepositoryRefactorTests.cs +++ b/SatelliteProvider.Tests/RepositoryRefactorTests.cs @@ -125,7 +125,8 @@ public class RepositoryRefactorTests "latitude", "longitude", "tile_size_meters as TileSizeMeters", "tile_size_pixels as TileSizePixels", "image_type as ImageType", "maps_version as MapsVersion", "version", - "file_path as FilePath", "created_at as CreatedAt", "updated_at as UpdatedAt" + "file_path as FilePath", "source", "captured_at as CapturedAt", + "created_at as CreatedAt", "updated_at as UpdatedAt" }; var regionColumns = new[] { diff --git a/SatelliteProvider.Tests/TileServiceTests.cs b/SatelliteProvider.Tests/TileServiceTests.cs index 7d18c14..91732b3 100644 --- a/SatelliteProvider.Tests/TileServiceTests.cs +++ b/SatelliteProvider.Tests/TileServiceTests.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.Options; using Moq; using SatelliteProvider.Common.Configs; using SatelliteProvider.Common.DTO; +using SatelliteProvider.Common.Enums; using SatelliteProvider.Common.Interfaces; using SatelliteProvider.DataAccess.Models; using SatelliteProvider.DataAccess.Repositories; @@ -93,6 +94,8 @@ public class TileServiceTests FilePath = "tiles/18/0/0/cached.jpg", TileSizePixels = 256, ImageType = "jpg", + Source = TileSource.GoogleMaps, + CapturedAt = DateTime.UtcNow, }, }; tileRepo @@ -148,6 +151,8 @@ public class TileServiceTests FilePath = "tiles/18/0/0/cached_prior_year.jpg", TileSizePixels = 256, ImageType = "jpg", + Source = TileSource.GoogleMaps, + CapturedAt = DateTime.UtcNow.AddYears(-1), }, }; tileRepo @@ -271,6 +276,8 @@ public class TileServiceTests ImageType = "jpg", FilePath = "tiles/18/0/0/x.jpg", Version = 2026, + Source = TileSource.GoogleMaps, + CapturedAt = DateTime.UtcNow, }; var tileRepo = new Mock(); tileRepo.Setup(r => r.GetByIdAsync(id)).ReturnsAsync(entity); @@ -339,7 +346,16 @@ public class TileServiceTests var tileRepo = new Mock(); tileRepo .Setup(r => r.GetByTileCoordinatesAsync(z, x, y)) - .ReturnsAsync(new TileEntity { Id = Guid.NewGuid(), TileZoom = z, TileX = x, TileY = y, FilePath = tempPath }); + .ReturnsAsync(new TileEntity + { + Id = Guid.NewGuid(), + TileZoom = z, + TileX = x, + TileY = y, + FilePath = tempPath, + Source = TileSource.GoogleMaps, + CapturedAt = DateTime.UtcNow, + }); var service = BuildService(downloader, tileRepo); @@ -444,6 +460,38 @@ public class TileServiceTests capturedToken.Should().Be(cts.Token, "AZ-371 AC-3: caller-supplied CT must reach the downloader"); } + [Fact] + public void BuildTileEntity_SetsGoogleMapsSourceAndUtcCapturedAt_AZ484_AC5() + { + // Arrange + var downloader = new Mock(); + var tileRepo = new Mock(); + TileEntity? captured = null; + tileRepo + .Setup(r => r.InsertAsync(It.IsAny())) + .Callback(e => captured = e) + .ReturnsAsync(Guid.NewGuid()); + downloader + .Setup(d => d.DownloadSingleTileAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new DownloadedTileInfoV2(1, 2, 18, 47.46, 37.65, "tiles/18/1/2.jpg", 100.0)); + + var service = BuildService(downloader, tileRepo); + var before = DateTime.UtcNow; + + // Act + _ = service.DownloadAndStoreSingleTileAsync(47.46, 37.65, 18).GetAwaiter().GetResult(); + + // Assert + var after = DateTime.UtcNow; + captured.Should().NotBeNull(); + captured!.Source.Should().Be(TileSource.GoogleMaps, + "AZ-484 AC-5: the Google Maps download path stamps Source=GoogleMaps"); + captured.CapturedAt.Kind.Should().NotBe(DateTimeKind.Local, + "captured_at must be UTC per the v1.0.0 storage contract"); + captured.CapturedAt.Should().BeOnOrAfter(before).And.BeOnOrBefore(after, + "AZ-484 AC-5: CapturedAt is the UtcNow at download time"); + } + [Fact] public async Task DownloadAndStoreSingleTileAsync_DownloaderThrows_DoesNotInsert_AZ311_AC2b() { diff --git a/_docs/02_document/architecture.md b/_docs/02_document/architecture.md index 1f6621f..4f1a006 100644 --- a/_docs/02_document/architecture.md +++ b/_docs/02_document/architecture.md @@ -20,20 +20,22 @@ The three Layer-3 service components are compile-time siblings: each only refere **Architectural principles** (inferred): - Single-instance deployment, no horizontal scaling requirements (`inferred-from: Channel-based queue, no distributed state`) -- Immutable tile storage with year-based versioning for cache invalidation (`inferred-from: version column + unique index`) +- Append-by-source tile storage — multiple producers (Google Maps, UAV upload, future SatAR, …) can each persist a row per `(latitude, longitude, tile_zoom, tile_size_meters)` cell. Reads return the most-recent row across sources, ordered by `captured_at DESC` with deterministic `(updated_at DESC, id DESC)` tie-breaks. The single-row-per-cell-per-source invariant is enforced by the 5-column unique index `idx_tiles_unique_location_source` introduced in migration 013 (AZ-484). The `tiles.version` column is vestigial since AZ-357 dropped year-based cache invalidation in favour of cell-level overwrite. (`inferred-from: tiles table + AZ-484/AZ-357 migrations + tile-storage contract v1.0.0`) - Fire-and-forget async processing with status polling (`inferred-from: queue + background service + status endpoint`) - No authentication layer — designed as an internal/trusted network service (`inferred-from: no auth middleware in Program.cs`) **Planned features** (confirmed by user, currently stubs): - MGRS endpoint — tile access via Military Grid Reference System coordinates -- Upload endpoint — UAV nadir camera tile ingestion (Layer 2: orthogonal tiles uploaded post-flight, stored alongside Google Maps Layer 1; most recent layer returned on access) +- Upload endpoint — UAV nadir camera tile ingestion. Writes a row with `source='uav'` for the captured cell; the storage layer accepts it alongside any existing Google Maps row, and reads return whichever has the highest `captured_at`. AZ-484 has built the multi-source storage; the upload endpoint itself (T2 — AZ-485) and any quality-gate logic remain to be implemented. + +The N-source storage contract is authoritative in `_docs/02_document/contracts/data-access/tile-storage.md` (v1.0.0). Anything that reads or writes `tiles` MUST follow that contract rather than re-deriving the rules from prose here. **Drift signals**: - `geofence_polygons` mentioned in AGENTS.md as a routes table column but does not exist in schema or entity — documentation drift ## 1. System Context -**Problem being solved**: A GPS-denied UAV navigation service requires satellite imagery for positioning and route planning without GPS. This service pre-downloads Google Maps satellite tiles (Layer 1) for specified regions and routes, accepts UAV-captured nadir camera imagery uploaded post-flight (Layer 2), and serves the most recent tile layer on access. Tiles are stitched into composite images and packaged for offline use. +**Problem being solved**: A GPS-denied UAV navigation service requires satellite imagery for positioning and route planning without GPS. This service pre-downloads satellite tiles from one or more imagery sources (currently Google Maps; future sources including UAV nadir camera upload and additional providers such as SatAR) for specified regions and routes, stores them alongside each other under a per-source storage key, and serves the most-recent row across sources on access. Tiles are stitched into composite images and packaged for offline use. **System boundaries**: The Satellite Provider is a self-contained backend service. It receives HTTP requests (region/route definitions), downloads tiles from Google Maps, stores them on disk and in PostgreSQL, and produces output files (images, CSVs, ZIPs). @@ -41,8 +43,8 @@ The three Layer-3 service components are compile-time siblings: each only refere | System | Integration Type | Direction | Purpose | |--------|-----------------|-----------|---------| -| Satellite imagery provider (e.g., Google Maps) | HTTPS (tile download) | Outbound | Layer 1 satellite imagery source (provider-agnostic via `ISatelliteDownloader`) | -| GPS-Denied Service (UAV) | REST API | Inbound | Layer 2 nadir camera tile uploads post-flight | +| Satellite imagery provider (e.g., Google Maps) | HTTPS (tile download) | Outbound | First implementation of the multi-source `tiles` storage; provider-agnostic via `ISatelliteDownloader`. Stamps `source='google_maps'` on every persisted row. | +| GPS-Denied Service (UAV) | REST API | Inbound | Future producer of `source='uav'` rows via the upload endpoint (T2 — AZ-485). The storage layer (AZ-484) is already in place; the endpoint itself is still a stub. | | PostgreSQL | TCP (Npgsql) | Both | Tile metadata, region/route state | | File System | Local disk | Both | Tile image storage, output artifacts | | HTTP Clients | REST API | Inbound | Region/route requests, tile queries | diff --git a/_docs/02_document/contracts/data-access/tile-storage.md b/_docs/02_document/contracts/data-access/tile-storage.md index 35d705b..ff0846c 100644 --- a/_docs/02_document/contracts/data-access/tile-storage.md +++ b/_docs/02_document/contracts/data-access/tile-storage.md @@ -4,7 +4,7 @@ **Producer task**: AZ-484 — `_docs/02_tasks/todo/AZ-484_multi_source_tile_storage.md` **Consumer tasks**: AZ-485 (planned T2 — UAV upload endpoint); future tasks adding additional sources (e.g., SatAR) **Version**: 1.0.0 -**Status**: draft (frozen on AZ-484 implementation completion) +**Status**: frozen **Last Updated**: 2026-05-11 ## Purpose diff --git a/_docs/02_document/glossary.md b/_docs/02_document/glossary.md index 75fe7fd..07cf932 100644 --- a/_docs/02_document/glossary.md +++ b/_docs/02_document/glossary.md @@ -11,8 +11,10 @@ | Geofence | A rectangular geographic boundary (NW + SE corners) used to filter which route points receive map tile coverage | components/05_route_management/description.md | | Zoom Level | Google Maps tile resolution level (1–20); higher = more detail, smaller ground coverage per tile | modules/common_configs.md | | Stitch | Compositing multiple tiles into a single larger image with optional markers/borders | modules/services_region_service.md | -| Layer 1 | Satellite imagery from external providers (provider-agnostic; first implementation: Google Maps) | user clarification | -| Layer 2 | UAV-captured nadir camera imagery (orthogonal tiles uploaded post-flight) | user clarification | +| Layer 1 | Historic name for satellite imagery from external providers (provider-agnostic; first implementation: Google Maps). Generalised in AZ-484 to one of N values of `Tile Source`; the term is retained for continuity with earlier docs and tickets. | user clarification, AZ-484 | +| Layer 2 | Historic name for UAV-captured nadir camera imagery (orthogonal tiles uploaded post-flight). Generalised in AZ-484 to the `uav` `Tile Source` value; the term is retained for continuity with earlier docs and tickets. | user clarification, AZ-484 | +| Tile Source | The producer of a tile row, persisted in `tiles.source` as a contract-defined string (`google_maps`, `uav`, …). Each cell may have at most one row per source; reads return the most-recent across sources. Adding a new source requires a new `TileSource` enum member and a tile-storage contract version bump. | _docs/02_document/contracts/data-access/tile-storage.md (v1.0.0) | +| Captured At | Producer-defined UTC timestamp ("the moment this tile imagery represents") persisted in `tiles.captured_at`. For Google Maps it is `DateTime.UtcNow` at download time (provider does not expose original imagery date); for UAV it is the capture timestamp supplied by the upload client. Drives the most-recent-across-sources read selection rule. | _docs/02_document/contracts/data-access/tile-storage.md (v1.0.0) | | Nadir Camera | Downward-facing camera on a UAV capturing ground imagery during flight | user clarification | | GPS-Denied Service | The consuming system: a UAV navigation service operating without GPS, using satellite/UAV imagery for positioning | user clarification | | Slippy Map Coordinates | Tile X/Y indices in the Web Mercator projection grid (standard for web map tile servers) | data_model.md | diff --git a/_docs/02_document/module-layout.md b/_docs/02_document/module-layout.md index 3721e2b..cb66857 100644 --- a/_docs/02_document/module-layout.md +++ b/_docs/02_document/module-layout.md @@ -27,6 +27,9 @@ - `SatelliteProvider.Common/Configs/ProcessingConfig.cs` - `SatelliteProvider.Common/Configs/DatabaseConfig.cs` - `SatelliteProvider.Common/DTO/*.cs` (all DTOs) + - `SatelliteProvider.Common/Enums/RegionStatus.cs` + - `SatelliteProvider.Common/Enums/RoutePointType.cs` + - `SatelliteProvider.Common/Enums/TileSource.cs` (added by AZ-484; backed by the `tile-storage` v1.0.0 contract) - `SatelliteProvider.Common/Exceptions/RateLimitException.cs` - `SatelliteProvider.Common/Interfaces/*.cs` (all service interfaces) - `SatelliteProvider.Common/Utils/GeoUtils.cs` diff --git a/_docs/02_tasks/todo/AZ-484_multi_source_tile_storage.md b/_docs/02_tasks/done/AZ-484_multi_source_tile_storage.md similarity index 100% rename from _docs/02_tasks/todo/AZ-484_multi_source_tile_storage.md rename to _docs/02_tasks/done/AZ-484_multi_source_tile_storage.md diff --git a/_docs/03_implementation/batch_25_cycle1_report.md b/_docs/03_implementation/batch_25_cycle1_report.md new file mode 100644 index 0000000..ca0cf6a --- /dev/null +++ b/_docs/03_implementation/batch_25_cycle1_report.md @@ -0,0 +1,89 @@ +# Batch Report + +**Batch**: 25 (cycle 1) +**Tasks**: AZ-484 (Multi-source tile storage schema) +**Date**: 2026-05-11 + +## Task Results + +| Task | Status | Files Modified | Tests | AC Coverage | Issues | +|------|--------|---------------|-------|-------------|--------| +| AZ-484 Multi-source tile storage schema | Done | 11 source files + 4 docs | new + updated unit tests; new integration migration tests (handed off to Run Tests) | 7/7 ACs covered | None | + +## AC Test Coverage: All covered (7/7) + +| AC | Test | +|----|------| +| AC-1 | `MultiSourceInsertCoexistsUnderNewIndex_AZ484_AC1`, `NewUniqueConstraintIncludesSourceColumn_AZ484_AC1` (integration) | +| AC-2 | `MostRecentAcrossSourcesSelection_AZ484_AC2` (integration) | +| AC-3 | `SameSourceUpsertReplacesPreviousRow_AZ484_AC3` (integration) | +| AC-4 | `BackfillUpdateAssignsGoogleMapsAndCapturedAt_AZ484_AC4` (integration TEMP-table simulation of migration UPDATE) | +| AC-5 | `BuildTileEntity_SetsGoogleMapsSourceAndUtcCapturedAt_AZ484_AC5` (unit) | +| AC-6 | Existing 200 unit + 5 smoke pass unchanged — verified via the full suite run (handed off to autodev Step 11) | +| AC-7 | Documents amended in this batch; contract `tile-storage.md` Status flipped from `draft` to `frozen` | + +## Code Review Verdict: PASS +Report: `_docs/03_implementation/reviews/batch_25_cycle1_review.md` + +## Auto-Fix Attempts: 0 +## Stuck Agents: None + +## Pre-Implementation Audit (Risk 3 mitigation) + +`new TileEntity` and `Mock` sites surveyed before edits: + +| Site | Action | +|------|--------| +| `SatelliteProvider.Services.TileDownloader/TileService.cs:146` (`BuildTileEntity`) | Updated — sets `Source = TileSource.GoogleMaps`, `CapturedAt = DateTime.UtcNow` | +| `SatelliteProvider.Tests/TileServiceTests.cs:84` (BT-02 cached) | Updated — explicit `Source` + `CapturedAt = DateTime.UtcNow` | +| `SatelliteProvider.Tests/TileServiceTests.cs:139` (AZ-357 prior-year) | Updated — explicit `Source` + `CapturedAt = DateTime.UtcNow.AddYears(-1)` to mirror the prior-year semantic | +| `SatelliteProvider.Tests/TileServiceTests.cs:264` (`GetTileAsync` known-id) | Updated — explicit `Source` + `CapturedAt` | +| `SatelliteProvider.Tests/TileServiceTests.cs:342` (AZ-310 RepoHit) | Updated — inline `TileEntity` initializer expanded with explicit fields | +| `SatelliteProvider.Tests/InfrastructureTests.cs:23, :65` (mock-only, no `TileEntity` construction) | No change required — mocks return defaults that no test asserts on | +| `SatelliteProvider.Tests/RepositoryRefactorTests.cs` ColumnList assertion | Updated — added `source` + `captured_at as CapturedAt` to expected column list | + +**Note on the task spec's "RegionServiceTests ~3 sites" estimate**: that count was inaccurate — `SatelliteProvider.Tests/RegionServiceTests.cs` does not reference `TileEntity` or `ITileRepository`. No edit was needed there. + +## Files Changed + +### New +- `SatelliteProvider.DataAccess/Migrations/013_AddTileSourceAndCapturedAt.sql` +- `SatelliteProvider.Common/Enums/TileSource.cs` +- `SatelliteProvider.DataAccess/TypeHandlers/TileSourceTypeHandler.cs` + +### Modified — production code +- `SatelliteProvider.DataAccess/Models/TileEntity.cs` (added `Source`, `CapturedAt`) +- `SatelliteProvider.DataAccess/Repositories/TileRepository.cs` (ColumnList + 4 SQL methods) +- `SatelliteProvider.DataAccess/TypeHandlers/EnumStringTypeHandler.cs` (registered `TileSourceTypeHandler`) +- `SatelliteProvider.Services.TileDownloader/TileService.cs` (`BuildTileEntity` stamps Source + CapturedAt) + +### Modified — tests +- `SatelliteProvider.Tests/TileServiceTests.cs` +- `SatelliteProvider.Tests/RepositoryRefactorTests.cs` +- `SatelliteProvider.Tests/EnumStringTypeHandlerTests.cs` +- `SatelliteProvider.IntegrationTests/MigrationTests.cs` + +### Modified — documentation +- `_docs/02_document/architecture.md` (Architecture Vision + System Context) +- `_docs/02_document/glossary.md` (Tile Source, Captured At, Layer 1/2 disambiguation) +- `_docs/02_document/module-layout.md` (Common Public API listing) +- `_docs/02_document/contracts/data-access/tile-storage.md` (Status: `draft` → `frozen`) + +## Design Notes + +**Wire-format mismatch motivating `TileSourceTypeHandler`.** The generic `EnumStringTypeHandler` emits `value.ToString().ToLowerInvariant()`, which would produce `'googlemaps'` for `TileSource.GoogleMaps`. The v1.0.0 contract requires `'google_maps'`. A dedicated `TileSourceTypeHandler` keeps the snake_case mapping localized and avoids leaking case-conversion logic into the generic handler. Round-trip and unknown-value tests are colocated with the existing handler test class. + +**`DISTINCT ON` for region reads.** PostgreSQL's `DISTINCT ON` was chosen over a self-join or window function because the new 5-column unique index can serve as the prefix sort, keeping the change a near-zero overhead for a region query. The outer `ORDER BY latitude DESC, longitude ASC, updated_at DESC` preserves the pre-AZ-484 caller-facing row order. + +**Migration transactionality (Risk 1 mitigation).** The migration is wrapped in `BEGIN ... COMMIT`. The IntegrationTests TEMP-table tests cover the backfill semantics; the live-schema test verifies the final post-013 index shape (and that the legacy 4-column index was actually dropped). + +## Next Batch +None — AZ-484 is the only task in this cycle. AZ-485 (UAV upload + quality gate) is deferred to a future Step 9 loop and is recorded in `_docs/02_tasks/_dependencies_table.md` under Step 9 cycle 1. + +## Handoff to Step 11 (Run Tests) + +Per `/implement` skill Step 16: the autodev next step is Run Tests, so this batch does NOT execute the full suite locally. The `test-run` skill owns the full-suite gate. Pre-conditions required: +- `dotnet test SatelliteProvider.Tests` should pass (200 unit + new AZ-484 unit tests). +- `scripts/run-tests.sh --smoke` should pass with the live API + Postgres (5 smoke + new AZ-484 integration migration tests). + +If `test-run` reports a failure in either suite, surface it; the existing infrastructure tests for AZ-357 dedupe semantics and the new AZ-484 selection / UPSERT tests are the highest-signal checks. diff --git a/_docs/03_implementation/implementation_completeness_cycle1_report.md b/_docs/03_implementation/implementation_completeness_cycle1_report.md new file mode 100644 index 0000000..fecabe0 --- /dev/null +++ b/_docs/03_implementation/implementation_completeness_cycle1_report.md @@ -0,0 +1,45 @@ +# Product Implementation Completeness Gate — cycle 1 + +**Date**: 2026-05-11 +**Cycle**: 1 +**Tasks evaluated**: AZ-484 +**Verdict**: PASS + +## Per-Task Classification + +### AZ-484 — Multi-source tile storage schema (source + captured_at) — **PASS** + +**Evidence checked**: + +| Promise from spec | Production evidence | +|--------------------|---------------------| +| Migration 013 transactional, with column adds, backfill, index swap | `SatelliteProvider.DataAccess/Migrations/013_AddTileSourceAndCapturedAt.sql` (BEGIN/COMMIT, ALTER ADD COLUMN, UPDATE backfill, ALTER SET NOT NULL, DROP INDEX, CREATE UNIQUE INDEX) | +| `TileSource` enum in Common.Enums | `SatelliteProvider.Common/Enums/TileSource.cs` (`{ GoogleMaps, Uav }`) | +| `TileEntity` exposes `Source` + `CapturedAt` | `SatelliteProvider.DataAccess/Models/TileEntity.cs` | +| Repository read selection is most-recent across sources, deterministic | `TileRepository.GetByTileCoordinatesAsync` + `GetTilesByRegionAsync` (DISTINCT ON with `(captured_at DESC, updated_at DESC, id DESC)` tie-break) | +| Per-source UPSERT semantics on insert | `TileRepository.InsertAsync` (`ON CONFLICT (latitude, longitude, tile_zoom, tile_size_meters, source) DO UPDATE`) | +| `UpdateAsync` includes the new fields | `TileRepository.UpdateAsync` SET clause | +| Google Maps download path stamps `Source` + `CapturedAt` | `TileService.BuildTileEntity` | +| Snake_case wire format for `TileSource` | `TileSourceTypeHandler` (registered in `DapperEnumTypeHandlers.RegisterAll`) | +| Architecture Vision amended | `_docs/02_document/architecture.md` (Architecture Vision principles + System Context) | +| Glossary amended | `_docs/02_document/glossary.md` (`Tile Source`, `Captured At`; Layer 1 / Layer 2 disambiguation) | +| Module-layout amended | `_docs/02_document/module-layout.md` (Common Public API list) | +| Contract `tile-storage.md` v1.0.0 frozen | `_docs/02_document/contracts/data-access/tile-storage.md` Status: `frozen` | + +**Unresolved markers** (`TODO`, `placeholder`, `NotImplemented`, etc.) under owned files (`SatelliteProvider.Common/Enums/TileSource.cs`, `SatelliteProvider.DataAccess/Migrations/013_AddTileSourceAndCapturedAt.sql`, `SatelliteProvider.DataAccess/TypeHandlers/TileSourceTypeHandler.cs`): none found. + +**Named runtime dependencies**: AZ-484 introduces no new external dependency. It uses the existing Dapper / Npgsql / DbUp stack already integrated. The new `TileSourceTypeHandler` is wired into `DapperEnumTypeHandlers.RegisterAll`, which is invoked at API startup as part of DI registration. + +**Internal vs external**: every promised capability is an internal product capability and is implemented in production code. No promise is blocked on an external prerequisite. The deferred AZ-485 (UAV upload endpoint) is explicitly out of scope for this task. + +**Tests exercise real implementation path**: +- Unit: `BuildTileEntity_SetsGoogleMapsSourceAndUtcCapturedAt_AZ484_AC5` calls the real `TileService.DownloadAndStoreSingleTileAsync` path. +- Integration (migration): the AC-1..AC-4 tests run against the real Postgres container, asserting the live schema (5-column unique index, dropped legacy 4-column index) and the SQL semantics of the repository methods. + +## Cycle Verdict + +All product tasks (1/1) classified PASS. Proceeding to Final Test Run handoff (autodev Step 11). + +## Remediation Tasks Created + +None. diff --git a/_docs/03_implementation/implementation_report_multi_source_tile_storage_cycle1.md b/_docs/03_implementation/implementation_report_multi_source_tile_storage_cycle1.md new file mode 100644 index 0000000..8a06146 --- /dev/null +++ b/_docs/03_implementation/implementation_report_multi_source_tile_storage_cycle1.md @@ -0,0 +1,73 @@ +# Implementation Report — multi-source tile storage (cycle 1) + +**Date**: 2026-05-11 +**Cycle**: 1 +**Tasks completed**: AZ-484 +**Batches**: 25 +**Code review verdict**: PASS (`_docs/03_implementation/reviews/batch_25_cycle1_review.md`) +**Completeness gate verdict**: PASS (`_docs/03_implementation/implementation_completeness_cycle1_report.md`) +**Test handoff**: yes — full-suite execution deferred to autodev Step 11 (Run Tests) + +## Summary + +AZ-484 introduces multi-source tile storage to the `tiles` table and freezes the v1.0.0 `tile-storage` contract that future producers (T2 — UAV upload AZ-485, future SatAR provider, etc.) will consume. The implementation: + +- Migration `013_AddTileSourceAndCapturedAt.sql` adds `source` (`VARCHAR(32) NOT NULL`) and `captured_at` (`TIMESTAMP NOT NULL`) to `tiles`, backfills existing rows to `(source='google_maps', captured_at=created_at)`, drops the legacy 4-column unique index `idx_tiles_unique_location`, and creates the new 5-column unique index `idx_tiles_unique_location_source`. The whole migration runs inside a single transaction so a failure mid-flight cannot leave the table without its unique index or with partially backfilled rows. +- `SatelliteProvider.Common.Enums.TileSource { GoogleMaps, Uav }` is the producer enum. Because the v1.0.0 contract requires snake_case wire values (`google_maps`, `uav`) and the existing generic `EnumStringTypeHandler` only emits `value.ToString().ToLowerInvariant()`, AZ-484 introduces a focused `TileSourceTypeHandler` that owns the bidirectional mapping. Registration is added to `DapperEnumTypeHandlers.RegisterAll`. +- `TileEntity` exposes `Source` and `CapturedAt` properties. +- `TileRepository` is updated end-to-end: + - `ColumnList` includes `source` and `captured_at as CapturedAt`. + - `GetByTileCoordinatesAsync` returns the most-recent row across sources via `ORDER BY captured_at DESC, updated_at DESC, id DESC LIMIT 1`. + - `GetTilesByRegionAsync` uses `DISTINCT ON (latitude, longitude, tile_zoom, tile_size_meters)` with the same tie-break tuple, then restores the pre-AZ-484 caller-facing row order. + - `InsertAsync` UPSERTs on the 5-column conflict key and refreshes `captured_at` and `updated_at` on conflict. + - `UpdateAsync` writes `source` and `captured_at` alongside the other columns. +- `TileService.BuildTileEntity` stamps `Source = TileSource.GoogleMaps` and `CapturedAt = DateTime.UtcNow` so every Google-Maps-originated row carries the contract-required fields. +- Documentation: + - `_docs/02_document/architecture.md` Architecture Vision and System Context describe the N-source model, append-by-source storage, and most-recent-across-sources reads, and point at the contract as authoritative. + - `_docs/02_document/glossary.md` adds `Tile Source` and `Captured At`, and disambiguates the historic `Layer 1` / `Layer 2` terms against the new `TileSource` enum. + - `_docs/02_document/module-layout.md` lists `SatelliteProvider.Common/Enums/TileSource.cs` (and the previously implicit `RegionStatus.cs`, `RoutePointType.cs`) under the Common Public API surface. + - `_docs/02_document/contracts/data-access/tile-storage.md` is now Status `frozen` at v1.0.0. + +## AC Coverage (7/7 — see batch report for the full table) + +| AC | Coverage | +|----|----------| +| AC-1 | Integration: `MultiSourceInsertCoexistsUnderNewIndex_AZ484_AC1`, `NewUniqueConstraintIncludesSourceColumn_AZ484_AC1` | +| AC-2 | Integration: `MostRecentAcrossSourcesSelection_AZ484_AC2` | +| AC-3 | Integration: `SameSourceUpsertReplacesPreviousRow_AZ484_AC3` | +| AC-4 | Integration TEMP-table simulation: `BackfillUpdateAssignsGoogleMapsAndCapturedAt_AZ484_AC4` | +| AC-5 | Unit: `BuildTileEntity_SetsGoogleMapsSourceAndUtcCapturedAt_AZ484_AC5` | +| AC-6 | Existing 200 unit + 5 smoke pass unchanged — verified by Step 11 | +| AC-7 | Doc-state AC; verified inline (architecture, glossary, module-layout, contract status) | + +## Risk Mitigation + +| Risk | Mitigation in this cycle | +|------|--------------------------| +| 1 — Migration fails partway against a non-empty production-like DB | Migration wrapped in `BEGIN ... COMMIT`. AC-4 TEMP-table simulation asserts the backfill semantics; AC-1 live-schema test asserts the post-migration index shape and that the legacy index was dropped. | +| 2 — Read-path selection rule breaks region cache hit logic | AC-6 covers this via the existing smoke tests (BT-03 200 m region exercises the cache-hit path). Pre/post tile-row counts must match. Handed off to Step 11. | +| 3 — TileEntity field additions break test mock construction | Pre-implementation audit listed every `new TileEntity` site (5 locations across `TileServiceTests.cs` and `TileService.cs`); each was updated explicitly with `Source = TileSource.GoogleMaps` and a sensible `CapturedAt`. The task spec's "RegionServiceTests ~3 sites" estimate was inaccurate (those tests don't reference `TileEntity` / `ITileRepository`); no edits were needed there. | +| 4 — `EnumStringTypeHandler` registration drift | `TileSourceTypeHandler` is registered in `DapperEnumTypeHandlers.RegisterAll` alongside the existing handlers. New unit test `RegisterAll_RegistersTileSourceHandler_AZ484` asserts the registration is in place and emits the contract wire value. | + +## Deviations from Spec + +- **Wire-format handler**: the spec says "string-stored via the existing `EnumStringTypeHandler` pattern". The existing pattern emits `ToString().ToLowerInvariant()`, which would produce `'googlemaps'`. The contract requires `'google_maps'`. Implementation follows the contract by introducing a focused `TileSourceTypeHandler`. The "pattern" — generic Dapper type handler registered through `DapperEnumTypeHandlers.RegisterAll` — is preserved. +- **Test site count**: the spec estimated ~12 sites needing mock fixups including ~3 in `RegionServiceTests`. Actual count: 5 sites in `TileServiceTests.cs` and 1 ColumnList assertion in `RepositoryRefactorTests.cs`. `RegionServiceTests.cs` and `InfrastructureTests.cs` did not require `TileEntity` field updates (the latter only constructs mocks, not entities). + +## Handoff to Step 11 (Run Tests) + +Per `/implement` skill Step 16: this cycle does not execute the full test suite locally because the autodev next step is Run Tests, which owns the full-suite gate. + +Recommended Step-11 invocation: +- Unit suite: `scripts/run-tests.sh --unit-only` (covers the new AC-5 + handler tests). +- Smoke suite: `scripts/run-tests.sh --smoke` (covers AC-1..AC-4 integration migration tests AND verifies AC-6 regression — the 5 smoke scenarios continue to pass). + +If `test-run` reports a failure, the highest-signal checks are: +1. `BuildTileEntity_SetsGoogleMapsSourceAndUtcCapturedAt_AZ484_AC5` — confirms the production write path. +2. `MostRecentAcrossSourcesSelection_AZ484_AC2` — confirms the new SQL ORDER BY shape. +3. `NewUniqueConstraintIncludesSourceColumn_AZ484_AC1` — confirms migration 013 ran correctly against the live schema. +4. The 5 smoke scenarios — confirms region/route flows behave identically to the pre-T1 baseline (AC-6). + +## Deferred work + +- AZ-485 — UAV upload endpoint + quality gate. Tracked in `_docs/02_tasks/_dependencies_table.md` § Step 9 cycle 1 as planned, depending on AZ-484 and the now-frozen v1.0.0 contract. diff --git a/_docs/03_implementation/reviews/batch_25_cycle1_review.md b/_docs/03_implementation/reviews/batch_25_cycle1_review.md new file mode 100644 index 0000000..d1b6212 --- /dev/null +++ b/_docs/03_implementation/reviews/batch_25_cycle1_review.md @@ -0,0 +1,68 @@ +# Code Review Report + +**Batch**: 25 (cycle 1) +**Tasks**: AZ-484 (Multi-source tile storage schema) +**Date**: 2026-05-11 +**Verdict**: PASS + +## Findings + +None. + +## Phase Summary + +### Phase 1 — Context +Read AZ-484 task spec, `_docs/02_document/contracts/data-access/tile-storage.md` v1.0.0, the existing `_docs/02_document/architecture.md` Architecture Vision section, and the existing `module-layout.md` per-component map. Mapped the 15 changed files to AZ-484 (single-task batch). + +### Phase 2 — Spec Compliance +Walked every AC against code: + +| AC | Promise | Validating test | +|----|---------|-----------------| +| AC-1 | Per-source coexistence on the same cell | `MultiSourceInsertCoexistsUnderNewIndex_AZ484_AC1` (TEMP), `NewUniqueConstraintIncludesSourceColumn_AZ484_AC1` (live schema) | +| AC-2 | Most-recent across sources on read | `MostRecentAcrossSourcesSelection_AZ484_AC2` | +| AC-3 | Same-source UPSERT collapses to one row | `SameSourceUpsertReplacesPreviousRow_AZ484_AC3` | +| AC-4 | Migration backfill leaves no orphans | `BackfillUpdateAssignsGoogleMapsAndCapturedAt_AZ484_AC4` (TEMP simulation of the migration UPDATE) | +| AC-5 | Google Maps path stamps Source + CapturedAt | `BuildTileEntity_SetsGoogleMapsSourceAndUtcCapturedAt_AZ484_AC5` | +| AC-6 | Existing 200 unit + 5 smoke pass unchanged | Verified via the full suite run (handed off to autodev Step 11) | +| AC-7 | Architecture / glossary / module-layout / contract updated | Documents amended in this batch; contract Status flipped from `draft` to `frozen` | + +**Contract verification** against `_docs/02_document/contracts/data-access/tile-storage.md` v1.0.0: +- Shape: `source VARCHAR(32) NOT NULL`, `captured_at TIMESTAMP NOT NULL` — matches migration 013. +- 5-column unique index `idx_tiles_unique_location_source` — created by migration 013. +- Producer write API: `InsertAsync` UPSERT on the 5-column key, refreshes `captured_at`/`updated_at`/`file_path`/`tile_x`/`tile_y` — matches. +- Consumer read API: `GetByTileCoordinatesAsync` LIMIT 1 ordered by `(captured_at DESC, updated_at DESC, id DESC)`; `GetTilesByRegionAsync` uses `DISTINCT ON (latitude, longitude, tile_zoom, tile_size_meters)` with the same tie-break tuple — matches. +- Wire format: `TileSource.GoogleMaps → 'google_maps'`, `TileSource.Uav → 'uav'` enforced by `TileSourceTypeHandler` (necessary because the generic `EnumStringTypeHandler` would emit `'googlemaps'`). +- Inv-1 / Inv-2 / Inv-5: `NOT NULL` columns + handler `Parse` throws `DataException` on unknown values (no silent coercion per `coderule.mdc`). +- Inv-3: 5-column unique index. +- Inv-4: identical tie-break tuple in `GetByTileCoordinatesAsync` and the inner `DISTINCT ON` of `GetTilesByRegionAsync` guarantees identical winner per cell. + +### Phase 3 — Code Quality +- SRP: `TileSourceTypeHandler` is a focused persistence concern (the bidirectional wire-format mapping); kept separate from the generic `EnumStringTypeHandler` instead of leaking snake_case logic into the generic. +- Comments: only added where intent is non-obvious (snake_case wire-format requirement, new ORDER BY tuple, per-source UPSERT semantics, transactional migration rationale). No narration-of-code comments. +- Tests: every new test uses Arrange / Act / Assert. +- DRY: `CreateTempTilesTable` factored out across the three TEMP-table integration tests. + +### Phase 4 — Security Quick-Scan +- All SQL parameters bound (`@Source`, `@CapturedAt`, etc.) — no string interpolation of caller-supplied values. +- Migration backfill literal is `'google_maps'`, not user input. +- No new secrets or credentials introduced. + +### Phase 5 — Performance Scan +- The new `DISTINCT ON` in `GetTilesByRegionAsync` can use `idx_tiles_unique_location_source` for the partition prefix; no extra round-trip; slow-query log threshold preserved. +- No N+1 patterns introduced. + +### Phase 6 — Cross-Task Consistency +Single-task batch. Internal consistency: enum members, wire values, migration backfill literal, and test assertions all agree on `'google_maps'` / `'uav'`. + +### Phase 7 — Architecture Compliance +- Layering: `TileSource` enum lives in `SatelliteProvider.Common.Enums` (Layer 1 Foundation). DataAccess (Layer 1) and TileDownloader (Layer 3) both consume it through Common — no new cross-sibling ProjectReferences. +- Public API respect: `TileSource` and `TileSourceTypeHandler` are public; `module-layout.md` Common Public API list updated to include `TileSource.cs`. +- No new cycles. +- No duplicate symbols across components. + +### Baseline Delta +Not computed inline — this batch makes no structural changes that would shift the existing `_docs/02_document/architecture_compliance_baseline.md` deltas. The AZ-484 changes stay within the existing layering invariants confirmed in earlier baseline scans. + +## Verdict Logic +No Critical, High, Medium, or Low findings → **PASS**. diff --git a/_docs/_autodev_state.md b/_docs/_autodev_state.md index b0f304f..7789e7f 100644 --- a/_docs/_autodev_state.md +++ b/_docs/_autodev_state.md @@ -2,13 +2,13 @@ ## Current Step flow: existing-code -step: 10 -name: Implement +step: 11 +name: Run Tests status: not_started sub_step: phase: 0 name: awaiting-invocation - detail: "AZ-484 only; T2 (AZ-485) deferred to a future cycle" + detail: "" retry_count: 0 cycle: 1 tracker: jira