[AZ-503] Tile identity → UUIDv5 + integer UPSERT (foundation)
ci/woodpecker/push/01-test Pipeline was successful
ci/woodpecker/push/02-build-push Pipeline was successful

Foundation half of original AZ-503 (split during /autodev step 10 batch 2
on user choice; deferred work moved to AZ-505 with a Blocks link).

Adds deterministic tile identity (UUIDv5 over (z, x, y, source, flight_id))
shared cross-repo with gps-denied-onboard via the pinned TileNamespace
5b8d0c2e-7f1a-4d3b-9c5e-1f3a8e7d2b6c, switches the tiles UPSERT key from
floats to integers with per-flight separation, plumbs FlightId through
UavTileMetadata + handler, and writes UAV evidence to per-flight
on-disk directories so two flights at the same (z, x, y) coexist.

- Common: pure-C# RFC 9562 Uuidv5 (no third-party dep) + FlightId DTO
  field; 10 Python-reference unit vectors verify byte parity.
- DataAccess: migration 014 adds flight_id (uuid NULL), location_hash
  (uuid NOT NULL, backfilled via session-scoped pg_temp.uuidv5),
  content_sha256 (bytea NULL), legacy_id (uuid NULL = preserves
  pre-AZ-503 random id one cycle); drops idx_tiles_unique_location_source
  (AZ-484) and adds idx_tiles_unique_identity keyed on
  (tile_zoom, tile_x, tile_y, tile_size_meters, source,
   COALESCE(flight_id, '00000000-...'::uuid)) + idx_tiles_location_hash.
- TileRepository: ColumnList + UPSERT updated; id never updated on
  conflict (preserves AC-2 idempotence). UpdateAsync extended.
- Services: TileService and UavTileUploadHandler compute deterministic
  Id + LocationHash + ContentSha256 before insert; UAV file path
  becomes ./tiles/uav/{flight_id or 'none'}/{z}/{x}/{y}.jpg.
- Tests: Uuidv5Tests (10 reference vectors), UavTileFilePathTests
  (per-flight + anonymous paths), UavTileUploadHandlerTests (AC-2,
  AC-3, AC-7, AC-11 unit-level), UavUploadTests (AC-3 + AC-4
  integration: multi-flight DB coexistence with shared location_hash
  + distinct file_path; float-different lat/lon collapse to 1 row),
  MigrationTests (column shape, idx_tiles_unique_identity supersedes
  AZ-484 index, deterministic backfill).
- IntegrationTests project references Common to reuse Uuidv5 in raw
  SQL seeds.
- AZ-488 MultiSourceCoexistence seed fixed to populate location_hash
  (otherwise migration 014's NOT NULL constraint fails).

ACs covered: AC-1, AC-2, AC-3, AC-4, AC-7, AC-8, AC-11.
ACs deferred to AZ-505: AC-5, AC-6, AC-9, AC-10, AC-12.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Oleksandr Bezdieniezhnykh
2026-05-12 17:07:35 +03:00
parent f6197499a4
commit c646aa93e2
17 changed files with 1154 additions and 117 deletions
@@ -27,9 +27,20 @@ public static class MigrationTests
await MultiSourceInsertCoexistsUnderNewIndex_AZ484_AC1(connectionString);
await MostRecentAcrossSourcesSelection_AZ484_AC2(connectionString);
await SameSourceUpsertReplacesPreviousRow_AZ484_AC3(connectionString);
await NewUniqueConstraintIncludesSourceColumn_AZ484_AC1(connectionString);
await Az503MigrationSupersedesAz484UniqueIndex(connectionString);
Console.WriteLine("✓ Migration 013 tests: PASSED");
Console.WriteLine();
Console.WriteLine("Test: Migration 014 (AZ-503-foundation)");
Console.WriteLine("========================================");
Console.WriteLine();
await Az503ColumnsExistAndLocationHashIsNotNull(connectionString);
await Az503NewUniqueIndexCoversIntegerKeyAndFlightId(connectionString);
await Az503LocationHashBackfillIsDeterministic(connectionString);
Console.WriteLine("✓ Migration 014 tests: PASSED");
}
private static async Task DedupeSqlCollapsesDuplicatesByLatestUpdatedAt_AZ357_AC2(string connectionString)
@@ -115,15 +126,236 @@ public static class MigrationTests
Console.WriteLine(" ✓ Unique row (idF) preserved");
}
private static async Task NewUniqueConstraintIncludesSourceColumn_AZ484_AC1(string connectionString)
private static async Task Az503MigrationSupersedesAz484UniqueIndex(string connectionString)
{
Console.WriteLine();
Console.WriteLine("AZ-484 AC-1 part 2: post-migration-013 unique index includes the source column");
Console.WriteLine("AZ-484/AZ-503 supersession: AZ-503 migration 014 drops the AZ-484 unique index in favour of the integer-key + flight_id index");
// Arrange / Act
await using var conn = new NpgsqlConnection(connectionString);
await conn.OpenAsync();
var rows = await QueryIndexesAsync(conn);
// Assert — AZ-484's idx_tiles_unique_location_source must NOT exist anymore after migration 014.
var supersededIndex = rows.FirstOrDefault(r => string.Equals(r.Name, "idx_tiles_unique_location_source", StringComparison.Ordinal));
if (supersededIndex.Def is not null)
{
throw new Exception(
"AZ-503: legacy AZ-484 index 'idx_tiles_unique_location_source' still exists after migration 014 — migration did not drop it. " +
$"Definition: {supersededIndex.Def}");
}
// Pre-AZ-484 4-column index must also remain dropped.
var preAz484Index = rows.FirstOrDefault(r => string.Equals(r.Name, "idx_tiles_unique_location", StringComparison.Ordinal));
if (preAz484Index.Def is not null)
{
throw new Exception(
"AZ-503: pre-AZ-484 4-column index 'idx_tiles_unique_location' reappeared after migration 014. " +
$"Definition: {preAz484Index.Def}");
}
Console.WriteLine(" ✓ AZ-484 'idx_tiles_unique_location_source' dropped by migration 014 (superseded)");
Console.WriteLine(" ✓ Pre-AZ-484 'idx_tiles_unique_location' remains dropped");
}
private static async Task Az503ColumnsExistAndLocationHashIsNotNull(string connectionString)
{
Console.WriteLine("AZ-503 AC-6: migration 014 adds flight_id, location_hash, content_sha256, legacy_id with correct nullability");
// Arrange
await using var conn = new NpgsqlConnection(connectionString);
await conn.OpenAsync();
const string sql = @"
SELECT column_name, data_type, is_nullable
FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = 'tiles'
AND column_name IN ('flight_id', 'location_hash', 'content_sha256', 'legacy_id');";
var columns = new Dictionary<string, (string DataType, bool IsNullable)>(StringComparer.Ordinal);
await using (var cmd = new NpgsqlCommand(sql, conn))
await using (var reader = await cmd.ExecuteReaderAsync())
{
while (await reader.ReadAsync())
{
columns[reader.GetString(0)] = (
reader.GetString(1),
string.Equals(reader.GetString(2), "YES", StringComparison.OrdinalIgnoreCase));
}
}
// Assert — flight_id, location_hash, content_sha256, legacy_id must exist with the contractual shape.
AssertColumn(columns, "flight_id", expectedType: "uuid", expectedNullable: true);
AssertColumn(columns, "location_hash", expectedType: "uuid", expectedNullable: false);
AssertColumn(columns, "content_sha256", expectedType: "bytea", expectedNullable: true);
AssertColumn(columns, "legacy_id", expectedType: "uuid", expectedNullable: true);
Console.WriteLine(" ✓ flight_id (uuid, nullable), location_hash (uuid, NOT NULL), content_sha256 (bytea, nullable), legacy_id (uuid, nullable)");
}
private static async Task Az503NewUniqueIndexCoversIntegerKeyAndFlightId(string connectionString)
{
Console.WriteLine();
Console.WriteLine("AZ-503 AC-9: idx_tiles_unique_identity is unique on (tile_zoom, tile_x, tile_y, tile_size_meters, source, COALESCE(flight_id, ...))");
// Arrange / Act
await using var conn = new NpgsqlConnection(connectionString);
await conn.OpenAsync();
var rows = await QueryIndexesAsync(conn);
// Assert
var newIndex = rows.FirstOrDefault(r => string.Equals(r.Name, "idx_tiles_unique_identity", StringComparison.Ordinal));
if (newIndex.Def is null)
{
throw new Exception(
"AZ-503 AC-9: expected unique index 'idx_tiles_unique_identity' on tiles after migration 014, but it is not present. " +
$"Found indexes: {string.Join(", ", rows.Select(r => r.Name))}");
}
var lower = newIndex.Def.ToLowerInvariant();
if (!lower.Contains("unique"))
{
throw new Exception($"AZ-503 AC-9: idx_tiles_unique_identity is not UNIQUE. Definition: {newIndex.Def}");
}
foreach (var col in new[] { "tile_zoom", "tile_x", "tile_y", "tile_size_meters", "source", "flight_id" })
{
if (!lower.Contains(col))
{
throw new Exception($"AZ-503 AC-9: idx_tiles_unique_identity missing column '{col}'. Definition: {newIndex.Def}");
}
}
if (!lower.Contains("coalesce"))
{
throw new Exception(
$"AZ-503 AC-9: idx_tiles_unique_identity must wrap flight_id in COALESCE so NULL flights collide deterministically. Definition: {newIndex.Def}");
}
// A non-unique index on location_hash should also exist so the upcoming AZ-505 covering scan has a starting point.
var locationHashIndex = rows.FirstOrDefault(r => string.Equals(r.Name, "idx_tiles_location_hash", StringComparison.Ordinal));
if (locationHashIndex.Def is null)
{
throw new Exception(
"AZ-503 AC-9: expected supporting index 'idx_tiles_location_hash' after migration 014, but it is not present.");
}
Console.WriteLine($" ✓ New unique index present: {newIndex.Def}");
Console.WriteLine($" ✓ Supporting location_hash index present: {locationHashIndex.Def}");
}
private static async Task Az503LocationHashBackfillIsDeterministic(string connectionString)
{
Console.WriteLine();
Console.WriteLine("AZ-503 AC-6: the location_hash backfill function used by migration 014 is deterministic and matches RFC 9562 §5.5");
// Arrange — the migration installs pg_temp.uuidv5 then drops it; replay the same SHA-1 logic in a session
// to confirm that two identical inputs produce byte-identical UUIDv5 values, and that two distinct inputs
// produce different values.
await using var conn = new NpgsqlConnection(connectionString);
await conn.OpenAsync();
await ExecAsync(conn, "CREATE EXTENSION IF NOT EXISTS pgcrypto;");
await ExecAsync(conn, """
CREATE OR REPLACE FUNCTION pg_temp.uuidv5_probe(namespace uuid, name text)
RETURNS uuid
LANGUAGE plpgsql
IMMUTABLE
AS $$
DECLARE
namespace_bytes bytea;
input_bytes bytea;
hash_bytes bytea;
v5_bytes bytea;
BEGIN
namespace_bytes := decode(replace(namespace::text, '-', ''), 'hex');
input_bytes := namespace_bytes || convert_to(name, 'UTF8');
hash_bytes := digest(input_bytes, 'sha1');
v5_bytes := substring(hash_bytes from 1 for 16);
v5_bytes := set_byte(v5_bytes, 6, (get_byte(v5_bytes, 6) & 15) | 80);
v5_bytes := set_byte(v5_bytes, 8, (get_byte(v5_bytes, 8) & 63) | 128);
RETURN encode(v5_bytes, 'hex')::uuid;
END;
$$;
""");
// Act — location_hash canonical name is "{zoom}/{x}/{y}" (matches the migration backfill
// and SatelliteProvider.Services.TileDownloader.TileService.BuildTileEntity).
const string probeSql = @"
SELECT
pg_temp.uuidv5_probe('5b8d0c2e-7f1a-4d3b-9c5e-1f3a8e7d2b6c'::uuid, '18/12345/23456') AS v1,
pg_temp.uuidv5_probe('5b8d0c2e-7f1a-4d3b-9c5e-1f3a8e7d2b6c'::uuid, '18/12345/23456') AS v1_again,
pg_temp.uuidv5_probe('5b8d0c2e-7f1a-4d3b-9c5e-1f3a8e7d2b6c'::uuid, '18/12346/23456') AS v2;";
Guid v1, v1Again, v2;
await using (var cmd = new NpgsqlCommand(probeSql, conn))
await using (var reader = await cmd.ExecuteReaderAsync())
{
if (!await reader.ReadAsync())
{
throw new Exception("AZ-503 AC-6: backfill probe returned no rows.");
}
v1 = reader.GetGuid(0);
v1Again = reader.GetGuid(1);
v2 = reader.GetGuid(2);
}
// Assert
if (v1 != v1Again)
{
throw new Exception(
$"AZ-503 AC-6: location_hash backfill is non-deterministic. v1={v1}, v1_again={v1Again}.");
}
if (v1 == v2)
{
throw new Exception(
$"AZ-503 AC-6: location_hash backfill produced the same UUID for different (x,y) tuples. v1={v1}, v2={v2}.");
}
// Cross-check that the live tiles.location_hash column matches the same function for at least one row, if any rows exist.
// (Pre-existing rows are backfilled by migration 014; new rows would be written by app code that uses the C# Uuidv5.Create.)
long sampleRowCount = await ScalarLongAsync(conn, "SELECT COUNT(*) FROM tiles;");
if (sampleRowCount > 0)
{
const string sampleSql = @"
SELECT
location_hash,
pg_temp.uuidv5_probe(
'5b8d0c2e-7f1a-4d3b-9c5e-1f3a8e7d2b6c'::uuid,
tile_zoom::text || '/' || tile_x::text || '/' || tile_y::text
) AS expected_hash
FROM tiles
LIMIT 1;";
Guid storedHash, expectedHash;
await using (var cmd = new NpgsqlCommand(sampleSql, conn))
await using (var reader = await cmd.ExecuteReaderAsync())
{
if (await reader.ReadAsync())
{
storedHash = reader.GetGuid(0);
expectedHash = reader.GetGuid(1);
if (storedHash != expectedHash)
{
throw new Exception(
$"AZ-503 AC-6: tiles.location_hash drift for sample row. stored={storedHash}, expected={expectedHash}. " +
"Backfill formula and live UUIDv5 implementation must agree on the canonical name string.");
}
Console.WriteLine($" ✓ Sample row location_hash matches the canonical UUIDv5 formula: {storedHash}");
}
}
}
else
{
Console.WriteLine(" (no rows in tiles table; deterministic-probe-only assertion)");
}
Console.WriteLine(" ✓ UUIDv5 backfill probe is deterministic across two identical inputs");
Console.WriteLine(" ✓ UUIDv5 backfill probe distinguishes different (x,y) tuples");
}
private static async Task<List<(string Name, string Def)>> QueryIndexesAsync(NpgsqlConnection conn)
{
const string sql = @"
SELECT indexname, indexdef
FROM pg_indexes
@@ -131,47 +363,37 @@ public static class MigrationTests
AND tablename = 'tiles';";
var rows = new List<(string Name, string Def)>();
await using (var cmd = new NpgsqlCommand(sql, conn))
await using (var reader = await cmd.ExecuteReaderAsync())
await using var cmd = new NpgsqlCommand(sql, conn);
await using var reader = await cmd.ExecuteReaderAsync();
while (await reader.ReadAsync())
{
while (await reader.ReadAsync())
{
rows.Add((reader.GetString(0), reader.GetString(1)));
}
rows.Add((reader.GetString(0), reader.GetString(1)));
}
return rows;
}
// Assert
var newIndex = rows.FirstOrDefault(r => string.Equals(r.Name, "idx_tiles_unique_location_source", StringComparison.Ordinal));
if (newIndex.Def is null)
private static void AssertColumn(
Dictionary<string, (string DataType, bool IsNullable)> columns,
string columnName,
string expectedType,
bool expectedNullable)
{
if (!columns.TryGetValue(columnName, out var info))
{
throw new Exception(
"AZ-484 AC-1: expected unique index 'idx_tiles_unique_location_source' on tiles after migration 013, but it is not present. " +
$"Found indexes: {string.Join(", ", rows.Select(r => r.Name))}");
$"AZ-503 AC-6: column 'tiles.{columnName}' was not created by migration 014. " +
$"Found columns: {string.Join(", ", columns.Keys)}");
}
var lower = newIndex.Def.ToLowerInvariant();
if (!lower.Contains("unique"))
{
throw new Exception($"AZ-484 AC-1: idx_tiles_unique_location_source is not UNIQUE. Definition: {newIndex.Def}");
}
foreach (var col in new[] { "latitude", "longitude", "tile_zoom", "tile_size_meters", "source" })
{
if (!lower.Contains(col))
{
throw new Exception($"AZ-484 AC-1: idx_tiles_unique_location_source missing column '{col}'. Definition: {newIndex.Def}");
}
}
var oldIndex = rows.FirstOrDefault(r => string.Equals(r.Name, "idx_tiles_unique_location", StringComparison.Ordinal));
if (oldIndex.Def is not null)
if (!string.Equals(info.DataType, expectedType, StringComparison.OrdinalIgnoreCase))
{
throw new Exception(
"AZ-484 AC-1: legacy 4-column index 'idx_tiles_unique_location' still exists after migration 013 — migration did not drop it. " +
$"Definition: {oldIndex.Def}");
$"AZ-503 AC-6: column 'tiles.{columnName}' has data_type='{info.DataType}', expected '{expectedType}'.");
}
if (info.IsNullable != expectedNullable)
{
throw new Exception(
$"AZ-503 AC-6: column 'tiles.{columnName}' is_nullable={info.IsNullable}, expected {expectedNullable}.");
}
Console.WriteLine($" ✓ New 5-column unique index present: {newIndex.Def}");
Console.WriteLine(" ✓ Legacy 4-column unique index dropped");
}
private static async Task BackfillUpdateAssignsGoogleMapsAndCapturedAt_AZ484_AC4(string connectionString)