From 275ee1b554260ebd44f9230692c0cc42c422d014 Mon Sep 17 00:00:00 2001 From: Oleksandr Bezdieniezhnykh Date: Tue, 23 Jun 2026 13:18:59 +0300 Subject: [PATCH] Add TileProvision configuration and gRPC service for tile delivery - Introduced new TileProvision settings in appsettings.json, including MaxTilesPerBatch and ProgressEmitIntervalSeconds. - Configured TileProvisionConfig in Program.cs to bind the new settings. - Added gRPC service for RouteTileDelivery in Program.cs to handle tile delivery requests. - Updated SatelliteProvider.Api.csproj to include Grpc.AspNetCore package and added protobuf file for tile provision. - Enhanced AuthenticationServiceCollectionExtensions to handle JWT token extraction from the Authorization header. - Registered additional services in RouteManagementServiceCollectionExtensions for tile processing. These changes enhance the API's capability to manage tile provisioning and delivery efficiently. --- ...thenticationServiceCollectionExtensions.cs | 14 + .../Grpc/RouteTileDeliveryGrpcService.cs | 205 +++++++++++ SatelliteProvider.Api/Program.cs | 10 + .../Protos/tile_provision.proto | 95 +++++ .../SatelliteProvider.Api.csproj | 11 +- SatelliteProvider.Api/appsettings.json | 4 + .../Configs/TileProvisionConfig.cs | 7 + ...teManagementServiceCollectionExtensions.cs | 5 + ...teProvider.Services.RouteManagement.csproj | 1 + .../TileProvision/ClientTileCatalog.cs | 49 +++ .../TileProvision/RouteTileCandidate.cs | 3 + .../TileProvision/RouteTileDeliveryModels.cs | 27 ++ .../RouteTileDeliveryOrchestrator.cs | 336 ++++++++++++++++++ .../TileProvision/RouteTileExpander.cs | 116 ++++++ .../TileProvision/ServerTileProspect.cs | 6 + .../TileProvision/TileResolutionHelper.cs | 19 + .../ClientTileCatalogTests.cs | 64 ++++ .../RouteTileDeliveryOrchestratorTests.cs | 193 ++++++++++ .../RouteTileExpanderTests.cs | 68 ++++ .../c11_tilemanager/tile_provision.proto | 95 +++++ .../c11_tilemanager/tile_provision_grpc.md | 143 ++++++++ docker-compose.yml | 1 + 22 files changed, 1469 insertions(+), 3 deletions(-) create mode 100644 SatelliteProvider.Api/Grpc/RouteTileDeliveryGrpcService.cs create mode 100644 SatelliteProvider.Api/Protos/tile_provision.proto create mode 100644 SatelliteProvider.Common/Configs/TileProvisionConfig.cs create mode 100644 SatelliteProvider.Services.RouteManagement/TileProvision/ClientTileCatalog.cs create mode 100644 SatelliteProvider.Services.RouteManagement/TileProvision/RouteTileCandidate.cs create mode 100644 SatelliteProvider.Services.RouteManagement/TileProvision/RouteTileDeliveryModels.cs create mode 100644 SatelliteProvider.Services.RouteManagement/TileProvision/RouteTileDeliveryOrchestrator.cs create mode 100644 SatelliteProvider.Services.RouteManagement/TileProvision/RouteTileExpander.cs create mode 100644 SatelliteProvider.Services.RouteManagement/TileProvision/ServerTileProspect.cs create mode 100644 SatelliteProvider.Services.RouteManagement/TileProvision/TileResolutionHelper.cs create mode 100644 SatelliteProvider.Tests/ClientTileCatalogTests.cs create mode 100644 SatelliteProvider.Tests/RouteTileDeliveryOrchestratorTests.cs create mode 100644 SatelliteProvider.Tests/RouteTileExpanderTests.cs create mode 100644 _docs/02_document/contracts/c11_tilemanager/tile_provision.proto create mode 100644 _docs/02_document/contracts/c11_tilemanager/tile_provision_grpc.md diff --git a/SatelliteProvider.Api/Authentication/AuthenticationServiceCollectionExtensions.cs b/SatelliteProvider.Api/Authentication/AuthenticationServiceCollectionExtensions.cs index 17a716a..94ba91e 100644 --- a/SatelliteProvider.Api/Authentication/AuthenticationServiceCollectionExtensions.cs +++ b/SatelliteProvider.Api/Authentication/AuthenticationServiceCollectionExtensions.cs @@ -41,6 +41,20 @@ public static class AuthenticationServiceCollectionExtensions RequireSignedTokens = true, RequireExpirationTime = true }; + options.Events = new JwtBearerEvents + { + OnMessageReceived = context => + { + var authHeader = context.Request.Headers.Authorization.FirstOrDefault(); + if (!string.IsNullOrEmpty(authHeader) + && authHeader.StartsWith("Bearer ", StringComparison.OrdinalIgnoreCase)) + { + context.Token = authHeader["Bearer ".Length..].Trim(); + } + + return Task.CompletedTask; + }, + }; }); return services; diff --git a/SatelliteProvider.Api/Grpc/RouteTileDeliveryGrpcService.cs b/SatelliteProvider.Api/Grpc/RouteTileDeliveryGrpcService.cs new file mode 100644 index 0000000..495c339 --- /dev/null +++ b/SatelliteProvider.Api/Grpc/RouteTileDeliveryGrpcService.cs @@ -0,0 +1,205 @@ +using Google.Protobuf.WellKnownTypes; +using Grpc.Core; +using Microsoft.AspNetCore.Authorization; +using Satellite.V1; +using SatelliteProvider.Services.RouteManagement.TileProvision; + +namespace SatelliteProvider.Api.Grpc; + +[Authorize] +public sealed class RouteTileDeliveryGrpcService : RouteTileDelivery.RouteTileDeliveryBase +{ + private readonly RouteTileDeliveryOrchestrator _orchestrator; + private readonly ILogger _logger; + + public RouteTileDeliveryGrpcService( + RouteTileDeliveryOrchestrator orchestrator, + ILogger logger) + { + _orchestrator = orchestrator; + _logger = logger; + } + + public override async Task DeliverRouteTiles( + DeliverRouteTilesRequest request, + IServerStreamWriter responseStream, + ServerCallContext context) + { + if (request.Route is null) + { + await WriteErrorAsync(responseStream, "INVALID_REQUEST", "route is required", retryable: false, context.CancellationToken); + return; + } + + if (!Guid.TryParse(request.Route.RouteId, out var routeId)) + { + await WriteErrorAsync(responseStream, "INVALID_REQUEST", "route_id must be a UUID", retryable: false, context.CancellationToken); + return; + } + + var job = MapJob(request, routeId); + var sink = new GrpcRouteTileDeliverySink(responseStream, context.CancellationToken); + + try + { + await _orchestrator.DeliverAsync(job, sink, context.CancellationToken); + } + catch (ArgumentException ex) + { + _logger.LogWarning(ex, "Invalid route tile delivery request for route {RouteId}", routeId); + await WriteErrorAsync(responseStream, "INVALID_REQUEST", ex.Message, retryable: false, context.CancellationToken); + } + catch (OperationCanceledException) when (context.CancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + _logger.LogError(ex, "Route tile delivery failed for route {RouteId}", routeId); + await WriteErrorAsync(responseStream, "INTERNAL_ERROR", ex.Message, retryable: true, context.CancellationToken); + } + } + + private static RouteTileDeliveryJob MapJob(DeliverRouteTilesRequest request, Guid routeId) + { + var route = request.Route!; + var waypoints = route.Waypoints + .Select(w => (w.Lat, w.Lon)) + .ToList(); + + var geofences = route.Geofences + .Select(polygon => (IReadOnlyList<(double Lat, double Lon)>)polygon.Vertices + .Select(v => (v.Lat, v.Lon)) + .ToList()) + .ToList(); + + var clientTiles = request.ClientTiles + .Select(MapClientTile) + .ToList(); + + return new RouteTileDeliveryJob( + routeId, + waypoints, + route.RegionSizeMeters, + route.Zoom, + geofences, + route.IncludeGeofenceTiles, + clientTiles); + } + + private static ClientTileSnapshot MapClientTile(ClientTileRecord record) + { + var capturedAt = record.CapturedAt?.ToDateTime().ToUniversalTime() ?? DateTime.MinValue; + byte[]? hash = record.ContentSha256 is { Length: > 0 } sha ? sha.ToByteArray() : null; + return new ClientTileSnapshot( + record.Z, + record.X, + record.Y, + record.ResolutionMPerPx, + capturedAt, + hash); + } + + private static async Task WriteErrorAsync( + IServerStreamWriter responseStream, + string code, + string message, + bool retryable, + CancellationToken cancellationToken) + { + await responseStream.WriteAsync(new RouteTileEvent + { + Error = new DeliveryError + { + Code = code, + Message = message, + Retryable = retryable, + }, + }, cancellationToken); + } + + private sealed class GrpcRouteTileDeliverySink : IRouteTileDeliverySink + { + private readonly IServerStreamWriter _stream; + + public GrpcRouteTileDeliverySink(IServerStreamWriter stream, CancellationToken cancellationToken) + { + _stream = stream; + _ = cancellationToken; + } + + public async ValueTask WriteManifestAsync(uint totalCandidates, uint skippedByClient, uint toDeliver, CancellationToken cancellationToken) + { + await _stream.WriteAsync(new RouteTileEvent + { + Manifest = new RouteManifest + { + TotalCandidates = totalCandidates, + SkippedByClient = skippedByClient, + ToDeliver = toDeliver, + }, + }, cancellationToken); + } + + public async ValueTask WriteBatchAsync(uint batchSeq, IReadOnlyList tiles, CancellationToken cancellationToken) + { + var batch = new TileBatch { BatchSeq = batchSeq }; + foreach (var tile in tiles) + { + batch.Tiles.Add(new TilePayload + { + Z = tile.Candidate.Z, + X = tile.Candidate.X, + Y = tile.Candidate.Y, + ResolutionMPerPx = tile.ResolutionMetersPerPx, + CapturedAt = Timestamp.FromDateTime(DateTime.SpecifyKind(tile.CapturedAtUtc, DateTimeKind.Utc)), + Source = tile.Source, + Jpeg = Google.Protobuf.ByteString.CopyFrom(tile.Jpeg), + ContentSha256 = Google.Protobuf.ByteString.CopyFrom(tile.ContentSha256), + RoutePriority = tile.Candidate.RoutePriority, + }); + } + + await _stream.WriteAsync(new RouteTileEvent { Batch = batch }, cancellationToken); + } + + public async ValueTask WriteProgressAsync(uint delivered, uint total, uint downloading, CancellationToken cancellationToken) + { + await _stream.WriteAsync(new RouteTileEvent + { + Progress = new ProgressUpdate + { + Delivered = delivered, + Total = total, + Downloading = downloading, + }, + }, cancellationToken); + } + + public async ValueTask WriteCompleteAsync(uint delivered, uint skippedClient, uint skippedServerFilter, CancellationToken cancellationToken) + { + await _stream.WriteAsync(new RouteTileEvent + { + Complete = new DeliveryComplete + { + Delivered = delivered, + SkippedClient = skippedClient, + SkippedServerFilter = skippedServerFilter, + }, + }, cancellationToken); + } + + public async ValueTask WriteErrorAsync(string code, string message, bool retryable, CancellationToken cancellationToken) + { + await _stream.WriteAsync(new RouteTileEvent + { + Error = new DeliveryError + { + Code = code, + Message = message, + Retryable = retryable, + }, + }, cancellationToken); + } + } +} diff --git a/SatelliteProvider.Api/Program.cs b/SatelliteProvider.Api/Program.cs index 580c81b..4998f22 100644 --- a/SatelliteProvider.Api/Program.cs +++ b/SatelliteProvider.Api/Program.cs @@ -8,6 +8,7 @@ using Swashbuckle.AspNetCore.SwaggerGen; using SatelliteProvider.Api; using SatelliteProvider.Api.Authentication; using SatelliteProvider.Api.DTOs; +using SatelliteProvider.Api.Grpc; using SatelliteProvider.Api.Swagger; using SatelliteProvider.Api.Validators; using SatelliteProvider.DataAccess; @@ -35,6 +36,7 @@ builder.Services.Configure(builder.Configuration.GetSection("MapConfi builder.Services.Configure(builder.Configuration.GetSection("StorageConfig")); builder.Services.Configure(builder.Configuration.GetSection("ProcessingConfig")); builder.Services.Configure(builder.Configuration.GetSection("UavQuality")); +builder.Services.Configure(builder.Configuration.GetSection("TileProvision")); var uavQuality = builder.Configuration.GetSection("UavQuality").Get() ?? new UavQualityConfig(); var uavBatchBodyLimit = checked((long)uavQuality.MaxBatchSize * uavQuality.MaxBytes); @@ -127,6 +129,12 @@ GlobalValidatorConfig.ApplyOnce(); // options constructor deps. Transient so each request gets a fresh instance. builder.Services.AddTransient(); +builder.Services.AddGrpc(options => +{ + options.MaxReceiveMessageSize = 16 * 1024 * 1024; + options.MaxSendMessageSize = 64 * 1024 * 1024; +}); + builder.Services.AddEndpointsApiExplorer(); builder.Services.AddSwaggerGen(c => { @@ -205,6 +213,8 @@ app.UseCors("TilesCors"); app.UseAuthentication(); app.UseAuthorization(); +app.MapGrpcService(); + app.MapGet("/tiles/{z:int}/{x:int}/{y:int}", ServeTile) .RequireAuthorization() .WithOpenApi(op => new(op) { Summary = "Get satellite tile image by z/x/y coordinates (Slippy Map tile server)" }); diff --git a/SatelliteProvider.Api/Protos/tile_provision.proto b/SatelliteProvider.Api/Protos/tile_provision.proto new file mode 100644 index 0000000..5e1583f --- /dev/null +++ b/SatelliteProvider.Api/Protos/tile_provision.proto @@ -0,0 +1,95 @@ +syntax = "proto3"; + +package satellite.v1; + +import "google/protobuf/timestamp.proto"; + +option csharp_namespace = "Satellite.V1"; + +service RouteTileDelivery { + rpc DeliverRouteTiles(DeliverRouteTilesRequest) returns (stream RouteTileEvent); +} + +message DeliverRouteTilesRequest { + RouteSpec route = 1; + repeated ClientTileRecord client_tiles = 2; +} + +message RouteSpec { + string route_id = 1; + repeated Waypoint waypoints = 2; + double region_size_meters = 3; + int32 zoom = 4; + repeated GeofencePolygon geofences = 5; + bool include_geofence_tiles = 6; +} + +message Waypoint { + double lat = 1; + double lon = 2; +} + +message GeofencePolygon { + repeated Waypoint vertices = 1; +} + +message ClientTileRecord { + int32 z = 1; + int32 x = 2; + int32 y = 3; + double resolution_m_per_px = 4; + google.protobuf.Timestamp captured_at = 5; + optional string source = 6; + bytes content_sha256 = 7; +} + +message RouteTileEvent { + oneof payload { + RouteManifest manifest = 1; + TileBatch batch = 2; + ProgressUpdate progress = 3; + DeliveryComplete complete = 4; + DeliveryError error = 5; + } +} + +message RouteManifest { + uint32 total_candidates = 1; + uint32 skipped_by_client = 2; + uint32 to_deliver = 3; +} + +message TileBatch { + uint32 batch_seq = 1; + repeated TilePayload tiles = 2; +} + +message TilePayload { + int32 z = 1; + int32 x = 2; + int32 y = 3; + double resolution_m_per_px = 4; + google.protobuf.Timestamp captured_at = 5; + string source = 6; + bytes jpeg = 7; + bytes content_sha256 = 8; + uint32 route_priority = 9; +} + +message ProgressUpdate { + uint32 delivered = 1; + uint32 total = 2; + uint32 downloading = 3; +} + +message DeliveryComplete { + uint32 delivered = 1; + uint32 skipped_client = 2; + uint32 skipped_server_filter = 3; +} + +message DeliveryError { + string code = 1; + string message = 2; + bool retryable = 3; +} diff --git a/SatelliteProvider.Api/SatelliteProvider.Api.csproj b/SatelliteProvider.Api/SatelliteProvider.Api.csproj index f0aacbf..37805d0 100644 --- a/SatelliteProvider.Api/SatelliteProvider.Api.csproj +++ b/SatelliteProvider.Api/SatelliteProvider.Api.csproj @@ -7,9 +7,10 @@ - - - + + + + @@ -26,4 +27,8 @@ + + + + diff --git a/SatelliteProvider.Api/appsettings.json b/SatelliteProvider.Api/appsettings.json index 9c6a6f5..6779ef1 100644 --- a/SatelliteProvider.Api/appsettings.json +++ b/SatelliteProvider.Api/appsettings.json @@ -61,6 +61,10 @@ "MaxRoutePointSpacingMeters": 200.0, "LatLonTolerance": 0.0001 }, + "TileProvision": { + "MaxTilesPerBatch": 200, + "ProgressEmitIntervalSeconds": 2 + }, "CorsConfig": { "AllowedOrigins": [] } diff --git a/SatelliteProvider.Common/Configs/TileProvisionConfig.cs b/SatelliteProvider.Common/Configs/TileProvisionConfig.cs new file mode 100644 index 0000000..85d30b4 --- /dev/null +++ b/SatelliteProvider.Common/Configs/TileProvisionConfig.cs @@ -0,0 +1,7 @@ +namespace SatelliteProvider.Common.Configs; + +public sealed class TileProvisionConfig +{ + public int MaxTilesPerBatch { get; set; } = 200; + public int ProgressEmitIntervalSeconds { get; set; } = 2; +} diff --git a/SatelliteProvider.Services.RouteManagement/RouteManagementServiceCollectionExtensions.cs b/SatelliteProvider.Services.RouteManagement/RouteManagementServiceCollectionExtensions.cs index 9f00768..d3f35d5 100644 --- a/SatelliteProvider.Services.RouteManagement/RouteManagementServiceCollectionExtensions.cs +++ b/SatelliteProvider.Services.RouteManagement/RouteManagementServiceCollectionExtensions.cs @@ -1,6 +1,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using SatelliteProvider.Common.Interfaces; +using SatelliteProvider.Services.RouteManagement.TileProvision; namespace SatelliteProvider.Services.RouteManagement; @@ -13,6 +14,10 @@ public static class RouteManagementServiceCollectionExtensions services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddHostedService(); return services; diff --git a/SatelliteProvider.Services.RouteManagement/SatelliteProvider.Services.RouteManagement.csproj b/SatelliteProvider.Services.RouteManagement/SatelliteProvider.Services.RouteManagement.csproj index 6fbf32a..13cfd4d 100644 --- a/SatelliteProvider.Services.RouteManagement/SatelliteProvider.Services.RouteManagement.csproj +++ b/SatelliteProvider.Services.RouteManagement/SatelliteProvider.Services.RouteManagement.csproj @@ -17,6 +17,7 @@ + diff --git a/SatelliteProvider.Services.RouteManagement/TileProvision/ClientTileCatalog.cs b/SatelliteProvider.Services.RouteManagement/TileProvision/ClientTileCatalog.cs new file mode 100644 index 0000000..33af4ed --- /dev/null +++ b/SatelliteProvider.Services.RouteManagement/TileProvision/ClientTileCatalog.cs @@ -0,0 +1,49 @@ +namespace SatelliteProvider.Services.RouteManagement.TileProvision; + +public sealed record ClientTileSnapshot( + int Z, + int X, + int Y, + double ResolutionMetersPerPx, + DateTime CapturedAtUtc, + byte[]? ContentSha256); + +public static class ClientTileCatalog +{ + public static Dictionary<(int Z, int X, int Y), ClientTileSnapshot> IndexByZxy( + IEnumerable clientTiles) + { + var index = new Dictionary<(int Z, int X, int Y), ClientTileSnapshot>(); + foreach (var tile in clientTiles) + { + index[(tile.Z, tile.X, tile.Y)] = tile; + } + + return index; + } + + public static bool ShouldSkipForClient( + ClientTileSnapshot? client, + ServerTileProspect server) + { + if (client is null) + { + return false; + } + + if (client.ContentSha256 is { Length: 32 } clientHash + && server.ContentSha256 is { Length: 32 } serverHash + && clientHash.AsSpan().SequenceEqual(serverHash)) + { + return true; + } + + if (client.ResolutionMetersPerPx <= server.ResolutionMetersPerPx + && client.CapturedAtUtc >= server.CapturedAtUtc) + { + return true; + } + + return false; + } +} diff --git a/SatelliteProvider.Services.RouteManagement/TileProvision/RouteTileCandidate.cs b/SatelliteProvider.Services.RouteManagement/TileProvision/RouteTileCandidate.cs new file mode 100644 index 0000000..b631442 --- /dev/null +++ b/SatelliteProvider.Services.RouteManagement/TileProvision/RouteTileCandidate.cs @@ -0,0 +1,3 @@ +namespace SatelliteProvider.Services.RouteManagement.TileProvision; + +public sealed record RouteTileCandidate(int Z, int X, int Y, uint RoutePriority); diff --git a/SatelliteProvider.Services.RouteManagement/TileProvision/RouteTileDeliveryModels.cs b/SatelliteProvider.Services.RouteManagement/TileProvision/RouteTileDeliveryModels.cs new file mode 100644 index 0000000..f93647f --- /dev/null +++ b/SatelliteProvider.Services.RouteManagement/TileProvision/RouteTileDeliveryModels.cs @@ -0,0 +1,27 @@ +namespace SatelliteProvider.Services.RouteManagement.TileProvision; + +public sealed record RouteTileDeliveryJob( + Guid RouteId, + IReadOnlyList<(double Lat, double Lon)> Waypoints, + double RegionSizeMeters, + int Zoom, + IReadOnlyList> GeofenceVertices, + bool IncludeGeofenceTiles, + IReadOnlyList ClientTiles); + +public sealed record PreparedTileDelivery( + RouteTileCandidate Candidate, + byte[] Jpeg, + byte[] ContentSha256, + double ResolutionMetersPerPx, + DateTime CapturedAtUtc, + string Source); + +public interface IRouteTileDeliverySink +{ + ValueTask WriteManifestAsync(uint totalCandidates, uint skippedByClient, uint toDeliver, CancellationToken cancellationToken); + ValueTask WriteBatchAsync(uint batchSeq, IReadOnlyList tiles, CancellationToken cancellationToken); + ValueTask WriteProgressAsync(uint delivered, uint total, uint downloading, CancellationToken cancellationToken); + ValueTask WriteCompleteAsync(uint delivered, uint skippedClient, uint skippedServerFilter, CancellationToken cancellationToken); + ValueTask WriteErrorAsync(string code, string message, bool retryable, CancellationToken cancellationToken); +} diff --git a/SatelliteProvider.Services.RouteManagement/TileProvision/RouteTileDeliveryOrchestrator.cs b/SatelliteProvider.Services.RouteManagement/TileProvision/RouteTileDeliveryOrchestrator.cs new file mode 100644 index 0000000..2a756f4 --- /dev/null +++ b/SatelliteProvider.Services.RouteManagement/TileProvision/RouteTileDeliveryOrchestrator.cs @@ -0,0 +1,336 @@ +using System.Security.Cryptography; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using SatelliteProvider.Common.Configs; +using SatelliteProvider.Common.Interfaces; +using SatelliteProvider.Common.Utils; +using SatelliteProvider.DataAccess.Models; +using SatelliteProvider.DataAccess.Repositories; + +namespace SatelliteProvider.Services.RouteManagement.TileProvision; + +public sealed class RouteTileDeliveryOrchestrator +{ + private readonly RouteTileExpander _expander; + private readonly ITileRepository _tileRepository; + private readonly ITileService _tileService; + private readonly MapConfig _mapConfig; + private readonly ProcessingConfig _processingConfig; + private readonly TileProvisionConfig _provisionConfig; + private readonly ILogger _logger; + + public RouteTileDeliveryOrchestrator( + RouteTileExpander expander, + ITileRepository tileRepository, + ITileService tileService, + IOptions mapConfig, + IOptions processingConfig, + IOptions provisionConfig, + ILogger logger) + { + _expander = expander; + _tileRepository = tileRepository; + _tileService = tileService; + _mapConfig = mapConfig.Value; + _processingConfig = processingConfig.Value; + _provisionConfig = provisionConfig.Value; + _logger = logger; + } + + public async Task DeliverAsync( + RouteTileDeliveryJob job, + IRouteTileDeliverySink sink, + CancellationToken cancellationToken) + { + ValidateJob(job); + + var jobStartedAt = DateTime.UtcNow; + var candidates = _expander.Expand( + job.Waypoints, + job.RegionSizeMeters, + job.Zoom, + job.GeofenceVertices, + job.IncludeGeofenceTiles); + + var clientIndex = ClientTileCatalog.IndexByZxy(job.ClientTiles); + var locationHashes = candidates + .Select(c => Uuidv5.LocationHashForTile(c.Z, c.X, c.Y)) + .Distinct() + .ToArray(); + var dbTiles = await _tileRepository.GetTilesByLocationHashesAsync(locationHashes); + + var workItems = new List(candidates.Count); + var skippedByClient = 0u; + + foreach (var candidate in candidates) + { + var hash = Uuidv5.LocationHashForTile(candidate.Z, candidate.X, candidate.Y); + dbTiles.TryGetValue(hash, out var dbTile); + var prospect = BuildServerProspect(candidate, dbTile, jobStartedAt); + clientIndex.TryGetValue((candidate.Z, candidate.X, candidate.Y), out var clientTile); + + if (ClientTileCatalog.ShouldSkipForClient(clientTile, prospect)) + { + skippedByClient++; + continue; + } + + workItems.Add(new TileWorkItem(candidate, dbTile)); + } + + var totalCandidates = (uint)candidates.Count; + var toDeliver = (uint)workItems.Count; + await sink.WriteManifestAsync(totalCandidates, skippedByClient, toDeliver, cancellationToken); + + if (toDeliver == 0) + { + await sink.WriteCompleteAsync(0, skippedByClient, 0, cancellationToken); + return; + } + + var delivered = 0u; + var batchSeq = 0u; + var pendingBatch = new List(_provisionConfig.MaxTilesPerBatch); + + async Task FlushBatchAsync() + { + if (pendingBatch.Count == 0) + { + return; + } + + pendingBatch.Sort((a, b) => a.Candidate.RoutePriority.CompareTo(b.Candidate.RoutePriority)); + await sink.WriteBatchAsync(batchSeq++, pendingBatch.ToList(), cancellationToken); + delivered += (uint)pendingBatch.Count; + pendingBatch.Clear(); + } + + var cachedItems = workItems.Where(w => w.DbTile is not null).ToList(); + var missingItems = workItems.Where(w => w.DbTile is null).ToList(); + + foreach (var item in cachedItems.OrderBy(w => w.Candidate.RoutePriority)) + { + cancellationToken.ThrowIfCancellationRequested(); + var prepared = await TryPrepareFromDatabaseAsync(item.Candidate, item.DbTile!, cancellationToken); + if (prepared is null) + { + missingItems.Add(item); + continue; + } + + pendingBatch.Add(prepared); + if (pendingBatch.Count >= _provisionConfig.MaxTilesPerBatch) + { + await FlushBatchAsync(); + } + } + + await FlushBatchAsync(); + + if (missingItems.Count > 0) + { + var downloadChannel = System.Threading.Channels.Channel.CreateBounded( + new System.Threading.Channels.BoundedChannelOptions(_provisionConfig.MaxTilesPerBatch * 2) + { + FullMode = System.Threading.Channels.BoundedChannelFullMode.Wait, + SingleReader = true, + SingleWriter = false, + }); + + var inFlight = 0; + var lastProgressAt = DateTime.UtcNow; + + async Task EmitProgressIfDueAsync() + { + var now = DateTime.UtcNow; + if ((now - lastProgressAt).TotalSeconds < _provisionConfig.ProgressEmitIntervalSeconds) + { + return; + } + + lastProgressAt = now; + await sink.WriteProgressAsync(delivered, toDeliver, (uint)Volatile.Read(ref inFlight), cancellationToken); + } + + var downloadLimiter = new SemaphoreSlim( + _processingConfig.MaxConcurrentDownloads, + _processingConfig.MaxConcurrentDownloads); + + try + { + var producer = Task.Run(async () => + { + try + { + var tasks = missingItems + .OrderBy(w => w.Candidate.RoutePriority) + .Select(async item => + { + await downloadLimiter.WaitAsync(cancellationToken); + Interlocked.Increment(ref inFlight); + try + { + var prepared = await TryDownloadAndPrepareAsync(item.Candidate, cancellationToken); + if (prepared is not null) + { + await downloadChannel.Writer.WriteAsync(prepared, cancellationToken); + } + } + finally + { + Interlocked.Decrement(ref inFlight); + downloadLimiter.Release(); + } + }); + + await Task.WhenAll(tasks); + } + finally + { + downloadChannel.Writer.Complete(); + } + }, cancellationToken); + + await foreach (var prepared in downloadChannel.Reader.ReadAllAsync(cancellationToken)) + { + await EmitProgressIfDueAsync(); + pendingBatch.Add(prepared); + if (pendingBatch.Count >= _provisionConfig.MaxTilesPerBatch) + { + await FlushBatchAsync(); + } + } + + await producer; + } + finally + { + downloadLimiter.Dispose(); + } + } + + await FlushBatchAsync(); + var serverFiltered = toDeliver - delivered; + await sink.WriteCompleteAsync(delivered, skippedByClient, serverFiltered, cancellationToken); + } + + private void ValidateJob(RouteTileDeliveryJob job) + { + if (job.RouteId == Guid.Empty) + { + throw new ArgumentException("route_id must be a non-empty UUID", nameof(job)); + } + + if (job.Waypoints.Count < 2) + { + throw new ArgumentException("Route must have at least 2 waypoints", nameof(job)); + } + + if (job.RegionSizeMeters <= 0) + { + throw new ArgumentOutOfRangeException(nameof(job), "region_size_meters must be positive"); + } + + if (!_mapConfig.AllowedZoomLevels.Contains(job.Zoom)) + { + throw new ArgumentException( + $"zoom {job.Zoom} is not allowed. Allowed: {string.Join(", ", _mapConfig.AllowedZoomLevels)}", + nameof(job)); + } + } + + private ServerTileProspect BuildServerProspect( + RouteTileCandidate candidate, + TileEntity? dbTile, + DateTime jobStartedAtUtc) + { + var center = GeoUtils.TileToWorldPos(candidate.X, candidate.Y, candidate.Z); + var resolution = TileResolutionHelper.ResolutionMetersPerPixel( + candidate.Z, + center.Lat, + _mapConfig.TileSizePixels); + + if (dbTile is null) + { + return new ServerTileProspect(resolution, jobStartedAtUtc, null); + } + + return new ServerTileProspect( + dbTile.TileSizePixels > 0 ? dbTile.TileSizeMeters / dbTile.TileSizePixels : resolution, + dbTile.CapturedAt, + dbTile.ContentSha256); + } + + private async Task TryPrepareFromDatabaseAsync( + RouteTileCandidate candidate, + TileEntity dbTile, + CancellationToken cancellationToken) + { + if (!File.Exists(dbTile.FilePath)) + { + return null; + } + + try + { + var jpeg = await File.ReadAllBytesAsync(dbTile.FilePath, cancellationToken); + var hash = dbTile.ContentSha256 is { Length: 32 } existingHash + ? existingHash + : SHA256.HashData(jpeg); + var resolution = dbTile.TileSizePixels > 0 + ? dbTile.TileSizeMeters / dbTile.TileSizePixels + : TileResolutionHelper.ResolutionMetersPerPixel( + candidate.Z, + dbTile.Latitude, + _mapConfig.TileSizePixels); + + return new PreparedTileDelivery( + candidate, + jpeg, + hash, + resolution, + dbTile.CapturedAt, + dbTile.Source); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to read cached tile ({Z}, {X}, {Y})", candidate.Z, candidate.X, candidate.Y); + return null; + } + } + + private async Task TryDownloadAndPrepareAsync( + RouteTileCandidate candidate, + CancellationToken cancellationToken) + { + try + { + var center = GeoUtils.TileToWorldPos(candidate.X, candidate.Y, candidate.Z); + await _tileService.DownloadAndStoreSingleTileAsync( + center.Lat, + center.Lon, + candidate.Z, + cancellationToken); + + var dbTile = await _tileRepository.GetByTileCoordinatesAsync(candidate.Z, candidate.X, candidate.Y); + if (dbTile is null || !File.Exists(dbTile.FilePath)) + { + return null; + } + + return await TryPrepareFromDatabaseAsync(candidate, dbTile, cancellationToken); + } + catch (Exception ex) + { + _logger.LogWarning( + ex, + "Failed to download tile ({Z}, {X}, {Y}) for route delivery", + candidate.Z, + candidate.X, + candidate.Y); + return null; + } + } + + private sealed record TileWorkItem(RouteTileCandidate Candidate, TileEntity? DbTile); +} diff --git a/SatelliteProvider.Services.RouteManagement/TileProvision/RouteTileExpander.cs b/SatelliteProvider.Services.RouteManagement/TileProvision/RouteTileExpander.cs new file mode 100644 index 0000000..990894b --- /dev/null +++ b/SatelliteProvider.Services.RouteManagement/TileProvision/RouteTileExpander.cs @@ -0,0 +1,116 @@ +using Microsoft.Extensions.Options; +using SatelliteProvider.Common.Configs; +using SatelliteProvider.Common.DTO; +using SatelliteProvider.Common.Utils; + +namespace SatelliteProvider.Services.RouteManagement.TileProvision; + +public sealed class RouteTileExpander +{ + private readonly RoutePointGraphBuilder _pointGraphBuilder; + private readonly GeofenceGridCalculator _geofenceGridCalculator; + + public RouteTileExpander( + RoutePointGraphBuilder pointGraphBuilder, + GeofenceGridCalculator geofenceGridCalculator) + { + _pointGraphBuilder = pointGraphBuilder; + _geofenceGridCalculator = geofenceGridCalculator; + } + + public IReadOnlyList Expand( + IReadOnlyList<(double Lat, double Lon)> waypoints, + double regionSizeMeters, + int zoom, + IReadOnlyList> geofenceVertices, + bool includeGeofenceTiles) + { + if (waypoints.Count < 2) + { + throw new ArgumentException("Route must have at least 2 waypoints", nameof(waypoints)); + } + + if (regionSizeMeters <= 0) + { + throw new ArgumentOutOfRangeException(nameof(regionSizeMeters), "Region size must be positive"); + } + + var routePoints = waypoints + .Select(w => new RoutePoint { Latitude = w.Lat, Longitude = w.Lon }) + .ToList(); + var graph = _pointGraphBuilder.Build(routePoints); + + var tiles = new Dictionary<(int Z, int X, int Y), uint>(); + + for (var priority = 0; priority < graph.Points.Count; priority++) + { + var point = graph.Points[priority]; + AddCorridorTiles( + tiles, + new GeoPoint(point.Latitude, point.Longitude), + regionSizeMeters, + zoom, + (uint)priority); + } + + if (includeGeofenceTiles) + { + var geofencePriority = (uint)graph.Points.Count; + foreach (var vertices in geofenceVertices) + { + if (vertices.Count < 3) + { + continue; + } + + var minLat = vertices.Min(v => v.Lat); + var maxLat = vertices.Max(v => v.Lat); + var minLon = vertices.Min(v => v.Lon); + var maxLon = vertices.Max(v => v.Lon); + + var northWest = new GeoPoint(maxLat, minLon); + var southEast = new GeoPoint(minLat, maxLon); + var centers = _geofenceGridCalculator.GenerateRegions(northWest, southEast, regionSizeMeters); + + foreach (var center in centers) + { + AddCorridorTiles(tiles, center, regionSizeMeters, zoom, geofencePriority); + } + + geofencePriority++; + } + } + + return tiles + .OrderBy(t => t.Value) + .ThenBy(t => t.Key.Y) + .ThenBy(t => t.Key.X) + .Select(t => new RouteTileCandidate(t.Key.Z, t.Key.X, t.Key.Y, t.Value)) + .ToList(); + } + + private static void AddCorridorTiles( + Dictionary<(int Z, int X, int Y), uint> tiles, + GeoPoint center, + double regionSizeMeters, + int zoom, + uint routePriority) + { + var radiusMeters = regionSizeMeters / 2.0; + var (latMin, latMax, lonMin, lonMax) = GeoUtils.GetBoundingBox(center, radiusMeters); + var (xMin, yMin) = GeoUtils.WorldToTilePos(new GeoPoint(latMax, lonMin), zoom); + var (xMax, yMax) = GeoUtils.WorldToTilePos(new GeoPoint(latMin, lonMax), zoom); + + for (var y = yMin; y <= yMax; y++) + { + for (var x = xMin; x <= xMax; x++) + { + var key = (zoom, x, y); + if (!tiles.TryGetValue(key, out var existing) || routePriority < existing) + { + tiles[key] = routePriority; + } + } + } + } +} diff --git a/SatelliteProvider.Services.RouteManagement/TileProvision/ServerTileProspect.cs b/SatelliteProvider.Services.RouteManagement/TileProvision/ServerTileProspect.cs new file mode 100644 index 0000000..de98d5c --- /dev/null +++ b/SatelliteProvider.Services.RouteManagement/TileProvision/ServerTileProspect.cs @@ -0,0 +1,6 @@ +namespace SatelliteProvider.Services.RouteManagement.TileProvision; + +public sealed record ServerTileProspect( + double ResolutionMetersPerPx, + DateTime CapturedAtUtc, + byte[]? ContentSha256); diff --git a/SatelliteProvider.Services.RouteManagement/TileProvision/TileResolutionHelper.cs b/SatelliteProvider.Services.RouteManagement/TileProvision/TileResolutionHelper.cs new file mode 100644 index 0000000..1933824 --- /dev/null +++ b/SatelliteProvider.Services.RouteManagement/TileProvision/TileResolutionHelper.cs @@ -0,0 +1,19 @@ +using SatelliteProvider.Common.Utils; + +namespace SatelliteProvider.Services.RouteManagement.TileProvision; + +public static class TileResolutionHelper +{ + public static double ResolutionMetersPerPixel(int zoomLevel, double latitude, int tileSizePixels) + { + var latRad = latitude * Math.PI / 180.0; + var metersPerPixel = (GeoUtils.EarthEquatorialCircumferenceMeters * Math.Cos(latRad)) + / (Math.Pow(2, zoomLevel) * tileSizePixels); + return metersPerPixel; + } + + public static double TileSizeMeters(int zoomLevel, double latitude, int tileSizePixels) + { + return ResolutionMetersPerPixel(zoomLevel, latitude, tileSizePixels) * tileSizePixels; + } +} diff --git a/SatelliteProvider.Tests/ClientTileCatalogTests.cs b/SatelliteProvider.Tests/ClientTileCatalogTests.cs new file mode 100644 index 0000000..dbd1cce --- /dev/null +++ b/SatelliteProvider.Tests/ClientTileCatalogTests.cs @@ -0,0 +1,64 @@ +using System.Security.Cryptography; +using FluentAssertions; +using SatelliteProvider.Services.RouteManagement.TileProvision; + +namespace SatelliteProvider.Tests; + +public class ClientTileCatalogTests +{ + private static readonly byte[] ServerHash = SHA256.HashData("server-tile"u8.ToArray()); + private static readonly byte[] ClientHash = SHA256.HashData("client-tile"u8.ToArray()); + + [Fact] + public void ShouldSkipForClient_HashMatch_Skips() + { + var client = new ClientTileSnapshot(18, 1, 2, 1.0, DateTime.UtcNow.AddDays(-1), ServerHash); + var server = new ServerTileProspect(2.0, DateTime.UtcNow, ServerHash); + + ClientTileCatalog.ShouldSkipForClient(client, server).Should().BeTrue(); + } + + [Fact] + public void ShouldSkipForClient_MetadataSufficient_Skips() + { + var client = new ClientTileSnapshot(18, 1, 2, 0.5, DateTime.UtcNow, null); + var server = new ServerTileProspect(1.0, DateTime.UtcNow.AddHours(-1), null); + + ClientTileCatalog.ShouldSkipForClient(client, server).Should().BeTrue(); + } + + [Fact] + public void ShouldSkipForClient_WorseResolution_DoesNotSkip() + { + var client = new ClientTileSnapshot(18, 1, 2, 2.0, DateTime.UtcNow, null); + var server = new ServerTileProspect(1.0, DateTime.UtcNow, null); + + ClientTileCatalog.ShouldSkipForClient(client, server).Should().BeFalse(); + } + + [Fact] + public void ShouldSkipForClient_OlderCapture_DoesNotSkip() + { + var client = new ClientTileSnapshot(18, 1, 2, 0.5, DateTime.UtcNow.AddDays(-2), null); + var server = new ServerTileProspect(1.0, DateTime.UtcNow, null); + + ClientTileCatalog.ShouldSkipForClient(client, server).Should().BeFalse(); + } + + [Fact] + public void ShouldSkipForClient_NoClientRecord_DoesNotSkip() + { + var server = new ServerTileProspect(1.0, DateTime.UtcNow, ServerHash); + + ClientTileCatalog.ShouldSkipForClient(null, server).Should().BeFalse(); + } + + [Fact] + public void ShouldSkipForClient_DifferentHashWithInsufficientMetadata_DoesNotSkip() + { + var client = new ClientTileSnapshot(18, 1, 2, 2.0, DateTime.UtcNow.AddDays(-2), ClientHash); + var server = new ServerTileProspect(1.0, DateTime.UtcNow, ServerHash); + + ClientTileCatalog.ShouldSkipForClient(client, server).Should().BeFalse(); + } +} diff --git a/SatelliteProvider.Tests/RouteTileDeliveryOrchestratorTests.cs b/SatelliteProvider.Tests/RouteTileDeliveryOrchestratorTests.cs new file mode 100644 index 0000000..4ba9423 --- /dev/null +++ b/SatelliteProvider.Tests/RouteTileDeliveryOrchestratorTests.cs @@ -0,0 +1,193 @@ +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Moq; +using SatelliteProvider.Common.Configs; +using SatelliteProvider.Common.Enums; +using SatelliteProvider.Common.Interfaces; +using SatelliteProvider.Common.Utils; +using SatelliteProvider.DataAccess.Models; +using SatelliteProvider.DataAccess.Repositories; +using SatelliteProvider.Services.RouteManagement; +using SatelliteProvider.Services.RouteManagement.TileProvision; + +namespace SatelliteProvider.Tests; + +public class RouteTileDeliveryOrchestratorTests +{ + [Fact] + public async Task DeliverAsync_AllTilesSkippedByClient_EmitsManifestAndCompleteWithZeroDelivered() + { + var expander = new RouteTileExpander( + new RoutePointGraphBuilder(Options.Create(new ProcessingConfig { MaxRoutePointSpacingMeters = 200 })), + new GeofenceGridCalculator()); + + var tileRepo = new Mock(); + tileRepo + .Setup(r => r.GetTilesByLocationHashesAsync(It.IsAny>())) + .ReturnsAsync(new Dictionary()); + + var tileService = new Mock(); + var orchestrator = new RouteTileDeliveryOrchestrator( + expander, + tileRepo.Object, + tileService.Object, + Options.Create(new MapConfig { TileSizePixels = 256, AllowedZoomLevels = [18] }), + Options.Create(new ProcessingConfig { MaxConcurrentDownloads = 2 }), + Options.Create(new TileProvisionConfig { MaxTilesPerBatch = 100 }), + NullLogger.Instance); + + var waypoints = new List<(double Lat, double Lon)> { (47.0, 37.0), (47.001, 37.001) }; + var candidates = expander.Expand(waypoints, 400, 18, [], false); + candidates.Should().NotBeEmpty(); + + var clientTiles = candidates.Select(c => + { + var center = GeoUtils.TileToWorldPos(c.X, c.Y, c.Z); + var resolution = TileResolutionHelper.ResolutionMetersPerPixel(c.Z, center.Lat, 256); + return new ClientTileSnapshot(c.Z, c.X, c.Y, resolution, DateTime.UtcNow.AddHours(1), null); + }).ToList(); + + var sink = new RecordingSink(); + var job = new RouteTileDeliveryJob( + Guid.NewGuid(), + waypoints, + 400, + 18, + [], + false, + clientTiles); + + await orchestrator.DeliverAsync(job, sink, CancellationToken.None); + + sink.Manifest.Should().NotBeNull(); + sink.Manifest!.Value.Total.Should().Be((uint)candidates.Count); + sink.Manifest.Value.Skipped.Should().Be((uint)candidates.Count); + sink.Manifest.Value.ToDeliver.Should().Be(0); + sink.Complete.Should().NotBeNull(); + sink.Complete!.Value.Delivered.Should().Be(0); + sink.Complete.Value.SkippedClient.Should().Be((uint)candidates.Count); + sink.Batches.Should().BeEmpty(); + tileService.Verify( + s => s.DownloadAndStoreSingleTileAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + } + + [Fact] + public async Task DeliverAsync_CachedTileOnDisk_EmitsBatchWithoutDownload() + { + var tilesDir = Path.Combine(Path.GetTempPath(), "sp-grpc-test-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tilesDir); + try + { + var expander = new RouteTileExpander( + new RoutePointGraphBuilder(Options.Create(new ProcessingConfig { MaxRoutePointSpacingMeters = 200 })), + new GeofenceGridCalculator()); + + var waypoints = new List<(double Lat, double Lon)> { (47.0, 37.0), (47.001, 37.001) }; + var candidates = expander.Expand(waypoints, 400, 18, [], false); + var candidate = candidates[0]; + + var jpeg = new byte[] { 0xFF, 0xD8, 0xFF, 0xD9 }; + var tilePath = Path.Combine(tilesDir, "tile.jpg"); + await File.WriteAllBytesAsync(tilePath, jpeg); + var hash = System.Security.Cryptography.SHA256.HashData(jpeg); + var center = GeoUtils.TileToWorldPos(candidate.X, candidate.Y, candidate.Z); + var locationHash = Uuidv5.LocationHashForTile(candidate.Z, candidate.X, candidate.Y); + + var entity = new TileEntity + { + Id = Guid.NewGuid(), + TileZoom = candidate.Z, + TileX = candidate.X, + TileY = candidate.Y, + Latitude = center.Lat, + Longitude = center.Lon, + TileSizeMeters = 100, + TileSizePixels = 256, + ImageType = "jpg", + FilePath = tilePath, + Source = TileSourceConverter.GoogleMapsWireValue, + CapturedAt = DateTime.UtcNow, + CreatedAt = DateTime.UtcNow, + UpdatedAt = DateTime.UtcNow, + LocationHash = locationHash, + ContentSha256 = hash, + }; + + var tileRepo = new Mock(); + tileRepo + .Setup(r => r.GetTilesByLocationHashesAsync(It.IsAny>())) + .ReturnsAsync((IReadOnlyList hashes) => + { + var dict = new Dictionary(); + if (hashes.Contains(locationHash)) + { + dict[locationHash] = entity; + } + + return dict; + }); + + var tileService = new Mock(); + var orchestrator = new RouteTileDeliveryOrchestrator( + expander, + tileRepo.Object, + tileService.Object, + Options.Create(new MapConfig { TileSizePixels = 256, AllowedZoomLevels = [18] }), + Options.Create(new ProcessingConfig { MaxConcurrentDownloads = 2 }), + Options.Create(new TileProvisionConfig { MaxTilesPerBatch = 100 }), + NullLogger.Instance); + + var sink = new RecordingSink(); + var job = new RouteTileDeliveryJob(Guid.NewGuid(), waypoints, 400, 18, [], false, []); + + await orchestrator.DeliverAsync(job, sink, CancellationToken.None); + + sink.Manifest.Should().NotBeNull(); + sink.Manifest!.Value.ToDeliver.Should().BeGreaterThan(0); + sink.Batches.Should().NotBeEmpty(); + sink.Batches.SelectMany(b => b).Should().Contain(t => t.Candidate.X == candidate.X && t.Candidate.Y == candidate.Y); + sink.Complete.Should().NotBeNull(); + sink.Complete!.Value.Delivered.Should().BeGreaterThan(0u); + } + finally + { + if (Directory.Exists(tilesDir)) + { + Directory.Delete(tilesDir, recursive: true); + } + } + } + + private sealed class RecordingSink : IRouteTileDeliverySink + { + public (uint Total, uint Skipped, uint ToDeliver)? Manifest { get; private set; } + public List> Batches { get; } = new(); + public (uint Delivered, uint SkippedClient, uint SkippedServer)? Complete { get; private set; } + + public ValueTask WriteManifestAsync(uint totalCandidates, uint skippedByClient, uint toDeliver, CancellationToken cancellationToken) + { + Manifest = (totalCandidates, skippedByClient, toDeliver); + return ValueTask.CompletedTask; + } + + public ValueTask WriteBatchAsync(uint batchSeq, IReadOnlyList tiles, CancellationToken cancellationToken) + { + Batches.Add(tiles); + return ValueTask.CompletedTask; + } + + public ValueTask WriteProgressAsync(uint delivered, uint total, uint downloading, CancellationToken cancellationToken) + => ValueTask.CompletedTask; + + public ValueTask WriteCompleteAsync(uint delivered, uint skippedClient, uint skippedServerFilter, CancellationToken cancellationToken) + { + Complete = (delivered, skippedClient, skippedServerFilter); + return ValueTask.CompletedTask; + } + + public ValueTask WriteErrorAsync(string code, string message, bool retryable, CancellationToken cancellationToken) + => ValueTask.CompletedTask; + } +} diff --git a/SatelliteProvider.Tests/RouteTileExpanderTests.cs b/SatelliteProvider.Tests/RouteTileExpanderTests.cs new file mode 100644 index 0000000..ac955c5 --- /dev/null +++ b/SatelliteProvider.Tests/RouteTileExpanderTests.cs @@ -0,0 +1,68 @@ +using FluentAssertions; +using Microsoft.Extensions.Options; +using SatelliteProvider.Common.Configs; +using SatelliteProvider.Services.RouteManagement; +using SatelliteProvider.Services.RouteManagement.TileProvision; + +namespace SatelliteProvider.Tests; + +public class RouteTileExpanderTests +{ + private readonly RouteTileExpander _expander = new( + new RoutePointGraphBuilder(Options.Create(new ProcessingConfig { MaxRoutePointSpacingMeters = 200 })), + new GeofenceGridCalculator()); + + [Fact] + public void Expand_ShortRoute_ProducesTilesForEachWaypointCorridor() + { + var waypoints = new List<(double Lat, double Lon)> + { + (47.0, 37.0), + (47.001, 37.001), + }; + + var tiles = _expander.Expand(waypoints, regionSizeMeters: 400, zoom: 18, [], includeGeofenceTiles: false); + + tiles.Should().NotBeEmpty(); + tiles.Select(t => (t.Z, t.X, t.Y)).Should().OnlyHaveUniqueItems(); + tiles.Should().OnlyContain(t => t.Z == 18); + } + + [Fact] + public void Expand_LongSegment_AddsIntermediateCorridors() + { + var waypoints = new List<(double Lat, double Lon)> + { + (47.0, 37.0), + (47.01, 37.0), + }; + + var tiles = _expander.Expand(waypoints, regionSizeMeters: 400, zoom: 18, [], includeGeofenceTiles: false); + + tiles.Should().NotBeEmpty(); + tiles.Select(t => t.RoutePriority).Max().Should().BeGreaterThan(1u); + } + + [Fact] + public void Expand_WithGeofence_IncludesGeofenceTilesWhenRequested() + { + var waypoints = new List<(double Lat, double Lon)> + { + (47.0, 37.0), + (47.001, 37.001), + }; + + var geofence = new List<(double Lat, double Lon)> + { + (47.002, 37.002), + (47.003, 37.002), + (47.003, 37.003), + (47.002, 37.003), + }; + + var without = _expander.Expand(waypoints, 400, 18, [], false); + var with = _expander.Expand(waypoints, 400, 18, [geofence], true); + + with.Count.Should().BeGreaterThan(without.Count); + } +} diff --git a/_docs/02_document/contracts/c11_tilemanager/tile_provision.proto b/_docs/02_document/contracts/c11_tilemanager/tile_provision.proto new file mode 100644 index 0000000..5e1583f --- /dev/null +++ b/_docs/02_document/contracts/c11_tilemanager/tile_provision.proto @@ -0,0 +1,95 @@ +syntax = "proto3"; + +package satellite.v1; + +import "google/protobuf/timestamp.proto"; + +option csharp_namespace = "Satellite.V1"; + +service RouteTileDelivery { + rpc DeliverRouteTiles(DeliverRouteTilesRequest) returns (stream RouteTileEvent); +} + +message DeliverRouteTilesRequest { + RouteSpec route = 1; + repeated ClientTileRecord client_tiles = 2; +} + +message RouteSpec { + string route_id = 1; + repeated Waypoint waypoints = 2; + double region_size_meters = 3; + int32 zoom = 4; + repeated GeofencePolygon geofences = 5; + bool include_geofence_tiles = 6; +} + +message Waypoint { + double lat = 1; + double lon = 2; +} + +message GeofencePolygon { + repeated Waypoint vertices = 1; +} + +message ClientTileRecord { + int32 z = 1; + int32 x = 2; + int32 y = 3; + double resolution_m_per_px = 4; + google.protobuf.Timestamp captured_at = 5; + optional string source = 6; + bytes content_sha256 = 7; +} + +message RouteTileEvent { + oneof payload { + RouteManifest manifest = 1; + TileBatch batch = 2; + ProgressUpdate progress = 3; + DeliveryComplete complete = 4; + DeliveryError error = 5; + } +} + +message RouteManifest { + uint32 total_candidates = 1; + uint32 skipped_by_client = 2; + uint32 to_deliver = 3; +} + +message TileBatch { + uint32 batch_seq = 1; + repeated TilePayload tiles = 2; +} + +message TilePayload { + int32 z = 1; + int32 x = 2; + int32 y = 3; + double resolution_m_per_px = 4; + google.protobuf.Timestamp captured_at = 5; + string source = 6; + bytes jpeg = 7; + bytes content_sha256 = 8; + uint32 route_priority = 9; +} + +message ProgressUpdate { + uint32 delivered = 1; + uint32 total = 2; + uint32 downloading = 3; +} + +message DeliveryComplete { + uint32 delivered = 1; + uint32 skipped_client = 2; + uint32 skipped_server_filter = 3; +} + +message DeliveryError { + string code = 1; + string message = 2; + bool retryable = 3; +} diff --git a/_docs/02_document/contracts/c11_tilemanager/tile_provision_grpc.md b/_docs/02_document/contracts/c11_tilemanager/tile_provision_grpc.md new file mode 100644 index 0000000..97f170f --- /dev/null +++ b/_docs/02_document/contracts/c11_tilemanager/tile_provision_grpc.md @@ -0,0 +1,143 @@ +# Contract: RouteTileDelivery (gRPC) + +**Component**: c11_tilemanager (consumer), satellite-provider (producer) +**Epic**: AZ-976 +**ADR**: ADR-013 (architecture.md) +**Proto**: `tile_provision.proto` — `package satellite.v1` +**Version**: 0.3.0 +**Status**: proposed +**Last Updated**: 2026-06-19 + +## Purpose + +Operator-side **pre-flight cache provisioning**. Client sends route + onboard tile catalog once; server streams `RouteTileEvent` messages until `DeliveryComplete` or `DeliveryError`. + +satellite-provider does **not** receive `flight_id` — that is a C6 bookkeeping concern on the gps-denied side only (`route_id` is the wire correlation id). + +C11/C12 on the **operator workstation** only. ADR-004: airborne image must not import stubs or open this channel. + +## RPC + +```protobuf +service RouteTileDelivery { + rpc DeliverRouteTiles(DeliverRouteTilesRequest) returns (stream RouteTileEvent); +} +``` + +| Concern | Rule | +|---------|------| +| Auth | gRPC metadata `authorization: Bearer ` | +| TLS | Required in production; `SATELLITE_PROVIDER_TLS_INSECURE=1` dev knob | +| Idempotency | `RouteSpec.route_id` (UUID string) | +| Resume | Client persists last acked `batch_seq` per `route_id` locally (not on wire) | + +## Request + +### `DeliverRouteTilesRequest` + +| Field | Description | +|-------|-------------| +| `route` | Corridor geometry + single zoom | +| `client_tiles` | Onboard inventory snapshot (route intersection only) | + +### `RouteSpec` + +| Field | Maps from gps-denied | +|-------|----------------------| +| `route_id` | Client-generated UUID per provision job | +| `waypoints` | `replay_input.tlog_route.RouteSpec.waypoints` | +| `region_size_meters` | `RouteSpec.suggested_region_size_meters` | +| `zoom` | Single slippy zoom level (confirmed sufficient) | +| `geofences` | Optional inclusion polygons | +| `include_geofence_tiles` | Union geofence tiles with corridor grid | + +### `ClientTileRecord` + +Canonical key: **`(z, x, y)`**. `source` is informational only — **not** used in skip logic. + +| Field | C6 mapping | +|-------|------------| +| `resolution_m_per_px` | RESTRICT-SAT-4 (lower = better) | +| `captured_at` | `TileMetadata.capture_timestamp` | +| `content_sha256` | `TileMetadata.content_sha256_hex` (raw 32 bytes) | + +## Server skip rule (client catalog) + +For each server candidate tile, **omit from stream** when `client_tiles` has matching `(z,x,y)` and **any** of: + +1. `client.content_sha256` is non-empty and **equals** server payload hash → skip (byte-identical) +2. `client.resolution_m_per_px <= server.resolution_m_per_px` **and** `client.captured_at >= server.captured_at` → skip (metadata-sufficient) + +`source` is **not** compared. + +`RouteManifest.skipped_by_client` counts tiles removed by this rule. + +## Sector — not on this wire + +**Sector** (`active_conflict` vs `stable_rear`) controls **how stale a tile may be before C6 rejects it on write** (AC-NEW-6 freshness). It is an operator decision about the geographic area, not something satellite-provider needs to deliver tiles. + +| Layer | Who applies sector | +|-------|-------------------| +| satellite-provider | Does not need sector — streams tiles by route geometry | +| C11 client write | Reads sector from **C11/C12 config** (same as today) when calling C6 freshness gate | + +No `SectorClass` field on the gRPC request. + +## Response stream: `RouteTileEvent` + +Typical sequence: + +1. **`RouteManifest`** — `total_candidates`, `skipped_by_client`, `to_deliver` +2. **`TileBatch`** — monotonic `batch_seq`; on-disk hits first, then freshly fetched +3. **`ProgressUpdate`** — optional +4. **`DeliveryComplete`** or **`DeliveryError`** + +### `DeliveryComplete` counters + +| Field | Meaning | +|-------|---------| +| `delivered` | Tiles actually sent in `TileBatch` streams | +| `skipped_client` | Same as manifest `skipped_by_client` (echo for client verify) | +| `skipped_server_filter` | Tiles SP required but **did not send** after client dedup — see below | + +#### `skipped_server_filter` — what counts + +Tiles that entered the post-client-dedup work queue but never appeared in a batch: + +| Reason | Example | +|--------|---------| +| **Fetch failed** | External imagery provider 404/timeout after retries | +| **Below SP min resolution** | SP refuses to store/serve below its configured floor | +| **Geometry clip** | Tile dropped after server-side corridor/geofence validation | +| **Operational cap** | Job hit max-tiles / rate limit (if SP enforces) | + +Tiles skipped by the **client catalog rule** are **not** included here (they are `skipped_client`). + +If SP has no server-side filters in v1, `skipped_server_filter` may be **0**; the field is reserved for observability. + +### `TilePayload` + +| Field | Notes | +|-------|-------| +| `content_sha256` | 32-byte SHA-256 of `jpeg`; matches C6 DB invariant | +| `route_priority` | Lower = earlier along route | + +## Client write path (gps-denied) + +`RouteTileDeliveryClient` (C11): + +- Assigns C6 `flight_id` from operator context locally (not from SP) +- Applies RESTRICT-SAT-4, **sector-based freshness**, AZ-308 budget, download journal +- Resumes via persisted `route_id` + `batch_seq` + +## Migration + +REST `route_client` + `HttpTileDownloader` remain fallback until AZ-979 benchmark. + +## Change log + +| Version | Date | Change | +|---------|------|--------| +| 0.3.0 | 2026-06-19 | `ClientTileRecord.content_sha256`; sequential field nums on `TilePayload`; sector/flight_id off wire; skip rule + `skipped_server_filter` defined | +| 0.2.0 | 2026-06-19 | `satellite.v1.RouteTileDelivery` + `RouteTileEvent` oneof | +| 0.1.0 | 2026-06-19 | Initial draft (superseded) | diff --git a/docker-compose.yml b/docker-compose.yml index bc1284c..8783d19 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,7 @@ services: retries: 5 api: + platform: linux/amd64 build: context: . dockerfile: SatelliteProvider.Api/Dockerfile