mirror of
https://github.com/azaion/satellite-provider.git
synced 2026-06-23 15:11:15 +00:00
Add TileProvision configuration and gRPC service for tile delivery
- Introduced new TileProvision settings in appsettings.json, including MaxTilesPerBatch and ProgressEmitIntervalSeconds. - Configured TileProvisionConfig in Program.cs to bind the new settings. - Added gRPC service for RouteTileDelivery in Program.cs to handle tile delivery requests. - Updated SatelliteProvider.Api.csproj to include Grpc.AspNetCore package and added protobuf file for tile provision. - Enhanced AuthenticationServiceCollectionExtensions to handle JWT token extraction from the Authorization header. - Registered additional services in RouteManagementServiceCollectionExtensions for tile processing. These changes enhance the API's capability to manage tile provisioning and delivery efficiently.
This commit is contained in:
Oleksandr Bezdieniezhnykh
@@ -41,6 +41,20 @@ public static class AuthenticationServiceCollectionExtensions
|
||||
RequireSignedTokens = true,
|
||||
RequireExpirationTime = true
|
||||
};
|
||||
options.Events = new JwtBearerEvents
|
||||
{
|
||||
OnMessageReceived = context =>
|
||||
{
|
||||
var authHeader = context.Request.Headers.Authorization.FirstOrDefault();
|
||||
if (!string.IsNullOrEmpty(authHeader)
|
||||
&& authHeader.StartsWith("Bearer ", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
context.Token = authHeader["Bearer ".Length..].Trim();
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
return services;
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Grpc.Core;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Satellite.V1;
|
||||
using SatelliteProvider.Services.RouteManagement.TileProvision;
|
||||
|
||||
namespace SatelliteProvider.Api.Grpc;
|
||||
|
||||
[Authorize]
|
||||
public sealed class RouteTileDeliveryGrpcService : RouteTileDelivery.RouteTileDeliveryBase
|
||||
{
|
||||
private readonly RouteTileDeliveryOrchestrator _orchestrator;
|
||||
private readonly ILogger<RouteTileDeliveryGrpcService> _logger;
|
||||
|
||||
public RouteTileDeliveryGrpcService(
|
||||
RouteTileDeliveryOrchestrator orchestrator,
|
||||
ILogger<RouteTileDeliveryGrpcService> logger)
|
||||
{
|
||||
_orchestrator = orchestrator;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public override async Task DeliverRouteTiles(
|
||||
DeliverRouteTilesRequest request,
|
||||
IServerStreamWriter<RouteTileEvent> responseStream,
|
||||
ServerCallContext context)
|
||||
{
|
||||
if (request.Route is null)
|
||||
{
|
||||
await WriteErrorAsync(responseStream, "INVALID_REQUEST", "route is required", retryable: false, context.CancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!Guid.TryParse(request.Route.RouteId, out var routeId))
|
||||
{
|
||||
await WriteErrorAsync(responseStream, "INVALID_REQUEST", "route_id must be a UUID", retryable: false, context.CancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
var job = MapJob(request, routeId);
|
||||
var sink = new GrpcRouteTileDeliverySink(responseStream, context.CancellationToken);
|
||||
|
||||
try
|
||||
{
|
||||
await _orchestrator.DeliverAsync(job, sink, context.CancellationToken);
|
||||
}
|
||||
catch (ArgumentException ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Invalid route tile delivery request for route {RouteId}", routeId);
|
||||
await WriteErrorAsync(responseStream, "INVALID_REQUEST", ex.Message, retryable: false, context.CancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (context.CancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Route tile delivery failed for route {RouteId}", routeId);
|
||||
await WriteErrorAsync(responseStream, "INTERNAL_ERROR", ex.Message, retryable: true, context.CancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
private static RouteTileDeliveryJob MapJob(DeliverRouteTilesRequest request, Guid routeId)
|
||||
{
|
||||
var route = request.Route!;
|
||||
var waypoints = route.Waypoints
|
||||
.Select(w => (w.Lat, w.Lon))
|
||||
.ToList();
|
||||
|
||||
var geofences = route.Geofences
|
||||
.Select(polygon => (IReadOnlyList<(double Lat, double Lon)>)polygon.Vertices
|
||||
.Select(v => (v.Lat, v.Lon))
|
||||
.ToList())
|
||||
.ToList();
|
||||
|
||||
var clientTiles = request.ClientTiles
|
||||
.Select(MapClientTile)
|
||||
.ToList();
|
||||
|
||||
return new RouteTileDeliveryJob(
|
||||
routeId,
|
||||
waypoints,
|
||||
route.RegionSizeMeters,
|
||||
route.Zoom,
|
||||
geofences,
|
||||
route.IncludeGeofenceTiles,
|
||||
clientTiles);
|
||||
}
|
||||
|
||||
private static ClientTileSnapshot MapClientTile(ClientTileRecord record)
|
||||
{
|
||||
var capturedAt = record.CapturedAt?.ToDateTime().ToUniversalTime() ?? DateTime.MinValue;
|
||||
byte[]? hash = record.ContentSha256 is { Length: > 0 } sha ? sha.ToByteArray() : null;
|
||||
return new ClientTileSnapshot(
|
||||
record.Z,
|
||||
record.X,
|
||||
record.Y,
|
||||
record.ResolutionMPerPx,
|
||||
capturedAt,
|
||||
hash);
|
||||
}
|
||||
|
||||
private static async Task WriteErrorAsync(
|
||||
IServerStreamWriter<RouteTileEvent> responseStream,
|
||||
string code,
|
||||
string message,
|
||||
bool retryable,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await responseStream.WriteAsync(new RouteTileEvent
|
||||
{
|
||||
Error = new DeliveryError
|
||||
{
|
||||
Code = code,
|
||||
Message = message,
|
||||
Retryable = retryable,
|
||||
},
|
||||
}, cancellationToken);
|
||||
}
|
||||
|
||||
private sealed class GrpcRouteTileDeliverySink : IRouteTileDeliverySink
|
||||
{
|
||||
private readonly IServerStreamWriter<RouteTileEvent> _stream;
|
||||
|
||||
public GrpcRouteTileDeliverySink(IServerStreamWriter<RouteTileEvent> stream, CancellationToken cancellationToken)
|
||||
{
|
||||
_stream = stream;
|
||||
_ = cancellationToken;
|
||||
}
|
||||
|
||||
public async ValueTask WriteManifestAsync(uint totalCandidates, uint skippedByClient, uint toDeliver, CancellationToken cancellationToken)
|
||||
{
|
||||
await _stream.WriteAsync(new RouteTileEvent
|
||||
{
|
||||
Manifest = new RouteManifest
|
||||
{
|
||||
TotalCandidates = totalCandidates,
|
||||
SkippedByClient = skippedByClient,
|
||||
ToDeliver = toDeliver,
|
||||
},
|
||||
}, cancellationToken);
|
||||
}
|
||||
|
||||
public async ValueTask WriteBatchAsync(uint batchSeq, IReadOnlyList<PreparedTileDelivery> tiles, CancellationToken cancellationToken)
|
||||
{
|
||||
var batch = new TileBatch { BatchSeq = batchSeq };
|
||||
foreach (var tile in tiles)
|
||||
{
|
||||
batch.Tiles.Add(new TilePayload
|
||||
{
|
||||
Z = tile.Candidate.Z,
|
||||
X = tile.Candidate.X,
|
||||
Y = tile.Candidate.Y,
|
||||
ResolutionMPerPx = tile.ResolutionMetersPerPx,
|
||||
CapturedAt = Timestamp.FromDateTime(DateTime.SpecifyKind(tile.CapturedAtUtc, DateTimeKind.Utc)),
|
||||
Source = tile.Source,
|
||||
Jpeg = Google.Protobuf.ByteString.CopyFrom(tile.Jpeg),
|
||||
ContentSha256 = Google.Protobuf.ByteString.CopyFrom(tile.ContentSha256),
|
||||
RoutePriority = tile.Candidate.RoutePriority,
|
||||
});
|
||||
}
|
||||
|
||||
await _stream.WriteAsync(new RouteTileEvent { Batch = batch }, cancellationToken);
|
||||
}
|
||||
|
||||
public async ValueTask WriteProgressAsync(uint delivered, uint total, uint downloading, CancellationToken cancellationToken)
|
||||
{
|
||||
await _stream.WriteAsync(new RouteTileEvent
|
||||
{
|
||||
Progress = new ProgressUpdate
|
||||
{
|
||||
Delivered = delivered,
|
||||
Total = total,
|
||||
Downloading = downloading,
|
||||
},
|
||||
}, cancellationToken);
|
||||
}
|
||||
|
||||
public async ValueTask WriteCompleteAsync(uint delivered, uint skippedClient, uint skippedServerFilter, CancellationToken cancellationToken)
|
||||
{
|
||||
await _stream.WriteAsync(new RouteTileEvent
|
||||
{
|
||||
Complete = new DeliveryComplete
|
||||
{
|
||||
Delivered = delivered,
|
||||
SkippedClient = skippedClient,
|
||||
SkippedServerFilter = skippedServerFilter,
|
||||
},
|
||||
}, cancellationToken);
|
||||
}
|
||||
|
||||
public async ValueTask WriteErrorAsync(string code, string message, bool retryable, CancellationToken cancellationToken)
|
||||
{
|
||||
await _stream.WriteAsync(new RouteTileEvent
|
||||
{
|
||||
Error = new DeliveryError
|
||||
{
|
||||
Code = code,
|
||||
Message = message,
|
||||
Retryable = retryable,
|
||||
},
|
||||
}, cancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ using Swashbuckle.AspNetCore.SwaggerGen;
|
||||
using SatelliteProvider.Api;
|
||||
using SatelliteProvider.Api.Authentication;
|
||||
using SatelliteProvider.Api.DTOs;
|
||||
using SatelliteProvider.Api.Grpc;
|
||||
using SatelliteProvider.Api.Swagger;
|
||||
using SatelliteProvider.Api.Validators;
|
||||
using SatelliteProvider.DataAccess;
|
||||
@@ -35,6 +36,7 @@ builder.Services.Configure<MapConfig>(builder.Configuration.GetSection("MapConfi
|
||||
builder.Services.Configure<StorageConfig>(builder.Configuration.GetSection("StorageConfig"));
|
||||
builder.Services.Configure<ProcessingConfig>(builder.Configuration.GetSection("ProcessingConfig"));
|
||||
builder.Services.Configure<UavQualityConfig>(builder.Configuration.GetSection("UavQuality"));
|
||||
builder.Services.Configure<TileProvisionConfig>(builder.Configuration.GetSection("TileProvision"));
|
||||
|
||||
var uavQuality = builder.Configuration.GetSection("UavQuality").Get<UavQualityConfig>() ?? new UavQualityConfig();
|
||||
var uavBatchBodyLimit = checked((long)uavQuality.MaxBatchSize * uavQuality.MaxBytes);
|
||||
@@ -127,6 +129,12 @@ GlobalValidatorConfig.ApplyOnce();
|
||||
// options constructor deps. Transient so each request gets a fresh instance.
|
||||
builder.Services.AddTransient<UavUploadValidationFilter>();
|
||||
|
||||
builder.Services.AddGrpc(options =>
|
||||
{
|
||||
options.MaxReceiveMessageSize = 16 * 1024 * 1024;
|
||||
options.MaxSendMessageSize = 64 * 1024 * 1024;
|
||||
});
|
||||
|
||||
builder.Services.AddEndpointsApiExplorer();
|
||||
builder.Services.AddSwaggerGen(c =>
|
||||
{
|
||||
@@ -205,6 +213,8 @@ app.UseCors("TilesCors");
|
||||
app.UseAuthentication();
|
||||
app.UseAuthorization();
|
||||
|
||||
app.MapGrpcService<RouteTileDeliveryGrpcService>();
|
||||
|
||||
app.MapGet("/tiles/{z:int}/{x:int}/{y:int}", ServeTile)
|
||||
.RequireAuthorization()
|
||||
.WithOpenApi(op => new(op) { Summary = "Get satellite tile image by z/x/y coordinates (Slippy Map tile server)" });
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package satellite.v1;
|
||||
|
||||
import "google/protobuf/timestamp.proto";
|
||||
|
||||
option csharp_namespace = "Satellite.V1";
|
||||
|
||||
service RouteTileDelivery {
|
||||
rpc DeliverRouteTiles(DeliverRouteTilesRequest) returns (stream RouteTileEvent);
|
||||
}
|
||||
|
||||
message DeliverRouteTilesRequest {
|
||||
RouteSpec route = 1;
|
||||
repeated ClientTileRecord client_tiles = 2;
|
||||
}
|
||||
|
||||
message RouteSpec {
|
||||
string route_id = 1;
|
||||
repeated Waypoint waypoints = 2;
|
||||
double region_size_meters = 3;
|
||||
int32 zoom = 4;
|
||||
repeated GeofencePolygon geofences = 5;
|
||||
bool include_geofence_tiles = 6;
|
||||
}
|
||||
|
||||
message Waypoint {
|
||||
double lat = 1;
|
||||
double lon = 2;
|
||||
}
|
||||
|
||||
message GeofencePolygon {
|
||||
repeated Waypoint vertices = 1;
|
||||
}
|
||||
|
||||
message ClientTileRecord {
|
||||
int32 z = 1;
|
||||
int32 x = 2;
|
||||
int32 y = 3;
|
||||
double resolution_m_per_px = 4;
|
||||
google.protobuf.Timestamp captured_at = 5;
|
||||
optional string source = 6;
|
||||
bytes content_sha256 = 7;
|
||||
}
|
||||
|
||||
message RouteTileEvent {
|
||||
oneof payload {
|
||||
RouteManifest manifest = 1;
|
||||
TileBatch batch = 2;
|
||||
ProgressUpdate progress = 3;
|
||||
DeliveryComplete complete = 4;
|
||||
DeliveryError error = 5;
|
||||
}
|
||||
}
|
||||
|
||||
message RouteManifest {
|
||||
uint32 total_candidates = 1;
|
||||
uint32 skipped_by_client = 2;
|
||||
uint32 to_deliver = 3;
|
||||
}
|
||||
|
||||
message TileBatch {
|
||||
uint32 batch_seq = 1;
|
||||
repeated TilePayload tiles = 2;
|
||||
}
|
||||
|
||||
message TilePayload {
|
||||
int32 z = 1;
|
||||
int32 x = 2;
|
||||
int32 y = 3;
|
||||
double resolution_m_per_px = 4;
|
||||
google.protobuf.Timestamp captured_at = 5;
|
||||
string source = 6;
|
||||
bytes jpeg = 7;
|
||||
bytes content_sha256 = 8;
|
||||
uint32 route_priority = 9;
|
||||
}
|
||||
|
||||
message ProgressUpdate {
|
||||
uint32 delivered = 1;
|
||||
uint32 total = 2;
|
||||
uint32 downloading = 3;
|
||||
}
|
||||
|
||||
message DeliveryComplete {
|
||||
uint32 delivered = 1;
|
||||
uint32 skipped_client = 2;
|
||||
uint32 skipped_server_filter = 3;
|
||||
}
|
||||
|
||||
message DeliveryError {
|
||||
string code = 1;
|
||||
string message = 2;
|
||||
bool retryable = 3;
|
||||
}
|
||||
@@ -9,6 +9,7 @@
|
||||
<ItemGroup>
|
||||
<PackageReference Include="FluentValidation" Version="12.0.0" />
|
||||
<PackageReference Include="FluentValidation.DependencyInjectionExtensions" Version="12.0.0" />
|
||||
<PackageReference Include="Grpc.AspNetCore" Version="2.71.0" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="10.0.7" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="10.0.7"/>
|
||||
<PackageReference Include="Newtonsoft.Json" Version="13.0.4" />
|
||||
@@ -26,4 +27,8 @@
|
||||
<ProjectReference Include="..\SatelliteProvider.Services.RouteManagement\SatelliteProvider.Services.RouteManagement.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Protobuf Include="Protos\tile_provision.proto" GrpcServices="Server" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -61,6 +61,10 @@
|
||||
"MaxRoutePointSpacingMeters": 200.0,
|
||||
"LatLonTolerance": 0.0001
|
||||
},
|
||||
"TileProvision": {
|
||||
"MaxTilesPerBatch": 200,
|
||||
"ProgressEmitIntervalSeconds": 2
|
||||
},
|
||||
"CorsConfig": {
|
||||
"AllowedOrigins": []
|
||||
}
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
namespace SatelliteProvider.Common.Configs;
|
||||
|
||||
public sealed class TileProvisionConfig
|
||||
{
|
||||
public int MaxTilesPerBatch { get; set; } = 200;
|
||||
public int ProgressEmitIntervalSeconds { get; set; } = 2;
|
||||
}
|
||||
+5
@@ -1,6 +1,7 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using SatelliteProvider.Common.Interfaces;
|
||||
using SatelliteProvider.Services.RouteManagement.TileProvision;
|
||||
|
||||
namespace SatelliteProvider.Services.RouteManagement;
|
||||
|
||||
@@ -13,6 +14,10 @@ public static class RouteManagementServiceCollectionExtensions
|
||||
services.AddSingleton<RouteImageRenderer>();
|
||||
services.AddSingleton<TilesZipBuilder>();
|
||||
services.AddSingleton<RegionFileCleaner>();
|
||||
services.AddSingleton<GeofenceGridCalculator>();
|
||||
services.AddSingleton<RoutePointGraphBuilder>();
|
||||
services.AddSingleton<RouteTileExpander>();
|
||||
services.AddSingleton<RouteTileDeliveryOrchestrator>();
|
||||
services.AddSingleton<IRouteService, RouteService>();
|
||||
services.AddHostedService<RouteProcessingService>();
|
||||
return services;
|
||||
|
||||
+1
@@ -17,6 +17,7 @@
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\SatelliteProvider.Common\SatelliteProvider.Common.csproj" />
|
||||
<ProjectReference Include="..\SatelliteProvider.DataAccess\SatelliteProvider.DataAccess.csproj" />
|
||||
<ProjectReference Include="..\SatelliteProvider.Services.TileDownloader\SatelliteProvider.Services.TileDownloader.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
namespace SatelliteProvider.Services.RouteManagement.TileProvision;
|
||||
|
||||
public sealed record ClientTileSnapshot(
|
||||
int Z,
|
||||
int X,
|
||||
int Y,
|
||||
double ResolutionMetersPerPx,
|
||||
DateTime CapturedAtUtc,
|
||||
byte[]? ContentSha256);
|
||||
|
||||
public static class ClientTileCatalog
|
||||
{
|
||||
public static Dictionary<(int Z, int X, int Y), ClientTileSnapshot> IndexByZxy(
|
||||
IEnumerable<ClientTileSnapshot> clientTiles)
|
||||
{
|
||||
var index = new Dictionary<(int Z, int X, int Y), ClientTileSnapshot>();
|
||||
foreach (var tile in clientTiles)
|
||||
{
|
||||
index[(tile.Z, tile.X, tile.Y)] = tile;
|
||||
}
|
||||
|
||||
return index;
|
||||
}
|
||||
|
||||
public static bool ShouldSkipForClient(
|
||||
ClientTileSnapshot? client,
|
||||
ServerTileProspect server)
|
||||
{
|
||||
if (client is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (client.ContentSha256 is { Length: 32 } clientHash
|
||||
&& server.ContentSha256 is { Length: 32 } serverHash
|
||||
&& clientHash.AsSpan().SequenceEqual(serverHash))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (client.ResolutionMetersPerPx <= server.ResolutionMetersPerPx
|
||||
&& client.CapturedAtUtc >= server.CapturedAtUtc)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
namespace SatelliteProvider.Services.RouteManagement.TileProvision;
|
||||
|
||||
public sealed record RouteTileCandidate(int Z, int X, int Y, uint RoutePriority);
|
||||
@@ -0,0 +1,27 @@
|
||||
namespace SatelliteProvider.Services.RouteManagement.TileProvision;
|
||||
|
||||
public sealed record RouteTileDeliveryJob(
|
||||
Guid RouteId,
|
||||
IReadOnlyList<(double Lat, double Lon)> Waypoints,
|
||||
double RegionSizeMeters,
|
||||
int Zoom,
|
||||
IReadOnlyList<IReadOnlyList<(double Lat, double Lon)>> GeofenceVertices,
|
||||
bool IncludeGeofenceTiles,
|
||||
IReadOnlyList<ClientTileSnapshot> ClientTiles);
|
||||
|
||||
public sealed record PreparedTileDelivery(
|
||||
RouteTileCandidate Candidate,
|
||||
byte[] Jpeg,
|
||||
byte[] ContentSha256,
|
||||
double ResolutionMetersPerPx,
|
||||
DateTime CapturedAtUtc,
|
||||
string Source);
|
||||
|
||||
public interface IRouteTileDeliverySink
|
||||
{
|
||||
ValueTask WriteManifestAsync(uint totalCandidates, uint skippedByClient, uint toDeliver, CancellationToken cancellationToken);
|
||||
ValueTask WriteBatchAsync(uint batchSeq, IReadOnlyList<PreparedTileDelivery> tiles, CancellationToken cancellationToken);
|
||||
ValueTask WriteProgressAsync(uint delivered, uint total, uint downloading, CancellationToken cancellationToken);
|
||||
ValueTask WriteCompleteAsync(uint delivered, uint skippedClient, uint skippedServerFilter, CancellationToken cancellationToken);
|
||||
ValueTask WriteErrorAsync(string code, string message, bool retryable, CancellationToken cancellationToken);
|
||||
}
|
||||
+336
@@ -0,0 +1,336 @@
|
||||
using System.Security.Cryptography;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using SatelliteProvider.Common.Configs;
|
||||
using SatelliteProvider.Common.Interfaces;
|
||||
using SatelliteProvider.Common.Utils;
|
||||
using SatelliteProvider.DataAccess.Models;
|
||||
using SatelliteProvider.DataAccess.Repositories;
|
||||
|
||||
namespace SatelliteProvider.Services.RouteManagement.TileProvision;
|
||||
|
||||
public sealed class RouteTileDeliveryOrchestrator
|
||||
{
|
||||
private readonly RouteTileExpander _expander;
|
||||
private readonly ITileRepository _tileRepository;
|
||||
private readonly ITileService _tileService;
|
||||
private readonly MapConfig _mapConfig;
|
||||
private readonly ProcessingConfig _processingConfig;
|
||||
private readonly TileProvisionConfig _provisionConfig;
|
||||
private readonly ILogger<RouteTileDeliveryOrchestrator> _logger;
|
||||
|
||||
public RouteTileDeliveryOrchestrator(
|
||||
RouteTileExpander expander,
|
||||
ITileRepository tileRepository,
|
||||
ITileService tileService,
|
||||
IOptions<MapConfig> mapConfig,
|
||||
IOptions<ProcessingConfig> processingConfig,
|
||||
IOptions<TileProvisionConfig> provisionConfig,
|
||||
ILogger<RouteTileDeliveryOrchestrator> logger)
|
||||
{
|
||||
_expander = expander;
|
||||
_tileRepository = tileRepository;
|
||||
_tileService = tileService;
|
||||
_mapConfig = mapConfig.Value;
|
||||
_processingConfig = processingConfig.Value;
|
||||
_provisionConfig = provisionConfig.Value;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task DeliverAsync(
|
||||
RouteTileDeliveryJob job,
|
||||
IRouteTileDeliverySink sink,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ValidateJob(job);
|
||||
|
||||
var jobStartedAt = DateTime.UtcNow;
|
||||
var candidates = _expander.Expand(
|
||||
job.Waypoints,
|
||||
job.RegionSizeMeters,
|
||||
job.Zoom,
|
||||
job.GeofenceVertices,
|
||||
job.IncludeGeofenceTiles);
|
||||
|
||||
var clientIndex = ClientTileCatalog.IndexByZxy(job.ClientTiles);
|
||||
var locationHashes = candidates
|
||||
.Select(c => Uuidv5.LocationHashForTile(c.Z, c.X, c.Y))
|
||||
.Distinct()
|
||||
.ToArray();
|
||||
var dbTiles = await _tileRepository.GetTilesByLocationHashesAsync(locationHashes);
|
||||
|
||||
var workItems = new List<TileWorkItem>(candidates.Count);
|
||||
var skippedByClient = 0u;
|
||||
|
||||
foreach (var candidate in candidates)
|
||||
{
|
||||
var hash = Uuidv5.LocationHashForTile(candidate.Z, candidate.X, candidate.Y);
|
||||
dbTiles.TryGetValue(hash, out var dbTile);
|
||||
var prospect = BuildServerProspect(candidate, dbTile, jobStartedAt);
|
||||
clientIndex.TryGetValue((candidate.Z, candidate.X, candidate.Y), out var clientTile);
|
||||
|
||||
if (ClientTileCatalog.ShouldSkipForClient(clientTile, prospect))
|
||||
{
|
||||
skippedByClient++;
|
||||
continue;
|
||||
}
|
||||
|
||||
workItems.Add(new TileWorkItem(candidate, dbTile));
|
||||
}
|
||||
|
||||
var totalCandidates = (uint)candidates.Count;
|
||||
var toDeliver = (uint)workItems.Count;
|
||||
await sink.WriteManifestAsync(totalCandidates, skippedByClient, toDeliver, cancellationToken);
|
||||
|
||||
if (toDeliver == 0)
|
||||
{
|
||||
await sink.WriteCompleteAsync(0, skippedByClient, 0, cancellationToken);
|
||||
return;
|
||||
}
|
||||
|
||||
var delivered = 0u;
|
||||
var batchSeq = 0u;
|
||||
var pendingBatch = new List<PreparedTileDelivery>(_provisionConfig.MaxTilesPerBatch);
|
||||
|
||||
async Task FlushBatchAsync()
|
||||
{
|
||||
if (pendingBatch.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
pendingBatch.Sort((a, b) => a.Candidate.RoutePriority.CompareTo(b.Candidate.RoutePriority));
|
||||
await sink.WriteBatchAsync(batchSeq++, pendingBatch.ToList(), cancellationToken);
|
||||
delivered += (uint)pendingBatch.Count;
|
||||
pendingBatch.Clear();
|
||||
}
|
||||
|
||||
var cachedItems = workItems.Where(w => w.DbTile is not null).ToList();
|
||||
var missingItems = workItems.Where(w => w.DbTile is null).ToList();
|
||||
|
||||
foreach (var item in cachedItems.OrderBy(w => w.Candidate.RoutePriority))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var prepared = await TryPrepareFromDatabaseAsync(item.Candidate, item.DbTile!, cancellationToken);
|
||||
if (prepared is null)
|
||||
{
|
||||
missingItems.Add(item);
|
||||
continue;
|
||||
}
|
||||
|
||||
pendingBatch.Add(prepared);
|
||||
if (pendingBatch.Count >= _provisionConfig.MaxTilesPerBatch)
|
||||
{
|
||||
await FlushBatchAsync();
|
||||
}
|
||||
}
|
||||
|
||||
await FlushBatchAsync();
|
||||
|
||||
if (missingItems.Count > 0)
|
||||
{
|
||||
var downloadChannel = System.Threading.Channels.Channel.CreateBounded<PreparedTileDelivery>(
|
||||
new System.Threading.Channels.BoundedChannelOptions(_provisionConfig.MaxTilesPerBatch * 2)
|
||||
{
|
||||
FullMode = System.Threading.Channels.BoundedChannelFullMode.Wait,
|
||||
SingleReader = true,
|
||||
SingleWriter = false,
|
||||
});
|
||||
|
||||
var inFlight = 0;
|
||||
var lastProgressAt = DateTime.UtcNow;
|
||||
|
||||
async Task EmitProgressIfDueAsync()
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
if ((now - lastProgressAt).TotalSeconds < _provisionConfig.ProgressEmitIntervalSeconds)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
lastProgressAt = now;
|
||||
await sink.WriteProgressAsync(delivered, toDeliver, (uint)Volatile.Read(ref inFlight), cancellationToken);
|
||||
}
|
||||
|
||||
var downloadLimiter = new SemaphoreSlim(
|
||||
_processingConfig.MaxConcurrentDownloads,
|
||||
_processingConfig.MaxConcurrentDownloads);
|
||||
|
||||
try
|
||||
{
|
||||
var producer = Task.Run(async () =>
|
||||
{
|
||||
try
|
||||
{
|
||||
var tasks = missingItems
|
||||
.OrderBy(w => w.Candidate.RoutePriority)
|
||||
.Select(async item =>
|
||||
{
|
||||
await downloadLimiter.WaitAsync(cancellationToken);
|
||||
Interlocked.Increment(ref inFlight);
|
||||
try
|
||||
{
|
||||
var prepared = await TryDownloadAndPrepareAsync(item.Candidate, cancellationToken);
|
||||
if (prepared is not null)
|
||||
{
|
||||
await downloadChannel.Writer.WriteAsync(prepared, cancellationToken);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
Interlocked.Decrement(ref inFlight);
|
||||
downloadLimiter.Release();
|
||||
}
|
||||
});
|
||||
|
||||
await Task.WhenAll(tasks);
|
||||
}
|
||||
finally
|
||||
{
|
||||
downloadChannel.Writer.Complete();
|
||||
}
|
||||
}, cancellationToken);
|
||||
|
||||
await foreach (var prepared in downloadChannel.Reader.ReadAllAsync(cancellationToken))
|
||||
{
|
||||
await EmitProgressIfDueAsync();
|
||||
pendingBatch.Add(prepared);
|
||||
if (pendingBatch.Count >= _provisionConfig.MaxTilesPerBatch)
|
||||
{
|
||||
await FlushBatchAsync();
|
||||
}
|
||||
}
|
||||
|
||||
await producer;
|
||||
}
|
||||
finally
|
||||
{
|
||||
downloadLimiter.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
await FlushBatchAsync();
|
||||
var serverFiltered = toDeliver - delivered;
|
||||
await sink.WriteCompleteAsync(delivered, skippedByClient, serverFiltered, cancellationToken);
|
||||
}
|
||||
|
||||
private void ValidateJob(RouteTileDeliveryJob job)
|
||||
{
|
||||
if (job.RouteId == Guid.Empty)
|
||||
{
|
||||
throw new ArgumentException("route_id must be a non-empty UUID", nameof(job));
|
||||
}
|
||||
|
||||
if (job.Waypoints.Count < 2)
|
||||
{
|
||||
throw new ArgumentException("Route must have at least 2 waypoints", nameof(job));
|
||||
}
|
||||
|
||||
if (job.RegionSizeMeters <= 0)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(job), "region_size_meters must be positive");
|
||||
}
|
||||
|
||||
if (!_mapConfig.AllowedZoomLevels.Contains(job.Zoom))
|
||||
{
|
||||
throw new ArgumentException(
|
||||
$"zoom {job.Zoom} is not allowed. Allowed: {string.Join(", ", _mapConfig.AllowedZoomLevels)}",
|
||||
nameof(job));
|
||||
}
|
||||
}
|
||||
|
||||
private ServerTileProspect BuildServerProspect(
|
||||
RouteTileCandidate candidate,
|
||||
TileEntity? dbTile,
|
||||
DateTime jobStartedAtUtc)
|
||||
{
|
||||
var center = GeoUtils.TileToWorldPos(candidate.X, candidate.Y, candidate.Z);
|
||||
var resolution = TileResolutionHelper.ResolutionMetersPerPixel(
|
||||
candidate.Z,
|
||||
center.Lat,
|
||||
_mapConfig.TileSizePixels);
|
||||
|
||||
if (dbTile is null)
|
||||
{
|
||||
return new ServerTileProspect(resolution, jobStartedAtUtc, null);
|
||||
}
|
||||
|
||||
return new ServerTileProspect(
|
||||
dbTile.TileSizePixels > 0 ? dbTile.TileSizeMeters / dbTile.TileSizePixels : resolution,
|
||||
dbTile.CapturedAt,
|
||||
dbTile.ContentSha256);
|
||||
}
|
||||
|
||||
private async Task<PreparedTileDelivery?> TryPrepareFromDatabaseAsync(
|
||||
RouteTileCandidate candidate,
|
||||
TileEntity dbTile,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!File.Exists(dbTile.FilePath))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var jpeg = await File.ReadAllBytesAsync(dbTile.FilePath, cancellationToken);
|
||||
var hash = dbTile.ContentSha256 is { Length: 32 } existingHash
|
||||
? existingHash
|
||||
: SHA256.HashData(jpeg);
|
||||
var resolution = dbTile.TileSizePixels > 0
|
||||
? dbTile.TileSizeMeters / dbTile.TileSizePixels
|
||||
: TileResolutionHelper.ResolutionMetersPerPixel(
|
||||
candidate.Z,
|
||||
dbTile.Latitude,
|
||||
_mapConfig.TileSizePixels);
|
||||
|
||||
return new PreparedTileDelivery(
|
||||
candidate,
|
||||
jpeg,
|
||||
hash,
|
||||
resolution,
|
||||
dbTile.CapturedAt,
|
||||
dbTile.Source);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to read cached tile ({Z}, {X}, {Y})", candidate.Z, candidate.X, candidate.Y);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<PreparedTileDelivery?> TryDownloadAndPrepareAsync(
|
||||
RouteTileCandidate candidate,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var center = GeoUtils.TileToWorldPos(candidate.X, candidate.Y, candidate.Z);
|
||||
await _tileService.DownloadAndStoreSingleTileAsync(
|
||||
center.Lat,
|
||||
center.Lon,
|
||||
candidate.Z,
|
||||
cancellationToken);
|
||||
|
||||
var dbTile = await _tileRepository.GetByTileCoordinatesAsync(candidate.Z, candidate.X, candidate.Y);
|
||||
if (dbTile is null || !File.Exists(dbTile.FilePath))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return await TryPrepareFromDatabaseAsync(candidate, dbTile, cancellationToken);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
ex,
|
||||
"Failed to download tile ({Z}, {X}, {Y}) for route delivery",
|
||||
candidate.Z,
|
||||
candidate.X,
|
||||
candidate.Y);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record TileWorkItem(RouteTileCandidate Candidate, TileEntity? DbTile);
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
using SatelliteProvider.Common.Configs;
|
||||
using SatelliteProvider.Common.DTO;
|
||||
using SatelliteProvider.Common.Utils;
|
||||
|
||||
namespace SatelliteProvider.Services.RouteManagement.TileProvision;
|
||||
|
||||
public sealed class RouteTileExpander
|
||||
{
|
||||
private readonly RoutePointGraphBuilder _pointGraphBuilder;
|
||||
private readonly GeofenceGridCalculator _geofenceGridCalculator;
|
||||
|
||||
public RouteTileExpander(
|
||||
RoutePointGraphBuilder pointGraphBuilder,
|
||||
GeofenceGridCalculator geofenceGridCalculator)
|
||||
{
|
||||
_pointGraphBuilder = pointGraphBuilder;
|
||||
_geofenceGridCalculator = geofenceGridCalculator;
|
||||
}
|
||||
|
||||
public IReadOnlyList<RouteTileCandidate> Expand(
|
||||
IReadOnlyList<(double Lat, double Lon)> waypoints,
|
||||
double regionSizeMeters,
|
||||
int zoom,
|
||||
IReadOnlyList<IReadOnlyList<(double Lat, double Lon)>> geofenceVertices,
|
||||
bool includeGeofenceTiles)
|
||||
{
|
||||
if (waypoints.Count < 2)
|
||||
{
|
||||
throw new ArgumentException("Route must have at least 2 waypoints", nameof(waypoints));
|
||||
}
|
||||
|
||||
if (regionSizeMeters <= 0)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(regionSizeMeters), "Region size must be positive");
|
||||
}
|
||||
|
||||
var routePoints = waypoints
|
||||
.Select(w => new RoutePoint { Latitude = w.Lat, Longitude = w.Lon })
|
||||
.ToList();
|
||||
var graph = _pointGraphBuilder.Build(routePoints);
|
||||
|
||||
var tiles = new Dictionary<(int Z, int X, int Y), uint>();
|
||||
|
||||
for (var priority = 0; priority < graph.Points.Count; priority++)
|
||||
{
|
||||
var point = graph.Points[priority];
|
||||
AddCorridorTiles(
|
||||
tiles,
|
||||
new GeoPoint(point.Latitude, point.Longitude),
|
||||
regionSizeMeters,
|
||||
zoom,
|
||||
(uint)priority);
|
||||
}
|
||||
|
||||
if (includeGeofenceTiles)
|
||||
{
|
||||
var geofencePriority = (uint)graph.Points.Count;
|
||||
foreach (var vertices in geofenceVertices)
|
||||
{
|
||||
if (vertices.Count < 3)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var minLat = vertices.Min(v => v.Lat);
|
||||
var maxLat = vertices.Max(v => v.Lat);
|
||||
var minLon = vertices.Min(v => v.Lon);
|
||||
var maxLon = vertices.Max(v => v.Lon);
|
||||
|
||||
var northWest = new GeoPoint(maxLat, minLon);
|
||||
var southEast = new GeoPoint(minLat, maxLon);
|
||||
var centers = _geofenceGridCalculator.GenerateRegions(northWest, southEast, regionSizeMeters);
|
||||
|
||||
foreach (var center in centers)
|
||||
{
|
||||
AddCorridorTiles(tiles, center, regionSizeMeters, zoom, geofencePriority);
|
||||
}
|
||||
|
||||
geofencePriority++;
|
||||
}
|
||||
}
|
||||
|
||||
return tiles
|
||||
.OrderBy(t => t.Value)
|
||||
.ThenBy(t => t.Key.Y)
|
||||
.ThenBy(t => t.Key.X)
|
||||
.Select(t => new RouteTileCandidate(t.Key.Z, t.Key.X, t.Key.Y, t.Value))
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static void AddCorridorTiles(
|
||||
Dictionary<(int Z, int X, int Y), uint> tiles,
|
||||
GeoPoint center,
|
||||
double regionSizeMeters,
|
||||
int zoom,
|
||||
uint routePriority)
|
||||
{
|
||||
var radiusMeters = regionSizeMeters / 2.0;
|
||||
var (latMin, latMax, lonMin, lonMax) = GeoUtils.GetBoundingBox(center, radiusMeters);
|
||||
var (xMin, yMin) = GeoUtils.WorldToTilePos(new GeoPoint(latMax, lonMin), zoom);
|
||||
var (xMax, yMax) = GeoUtils.WorldToTilePos(new GeoPoint(latMin, lonMax), zoom);
|
||||
|
||||
for (var y = yMin; y <= yMax; y++)
|
||||
{
|
||||
for (var x = xMin; x <= xMax; x++)
|
||||
{
|
||||
var key = (zoom, x, y);
|
||||
if (!tiles.TryGetValue(key, out var existing) || routePriority < existing)
|
||||
{
|
||||
tiles[key] = routePriority;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
namespace SatelliteProvider.Services.RouteManagement.TileProvision;
|
||||
|
||||
public sealed record ServerTileProspect(
|
||||
double ResolutionMetersPerPx,
|
||||
DateTime CapturedAtUtc,
|
||||
byte[]? ContentSha256);
|
||||
@@ -0,0 +1,19 @@
|
||||
using SatelliteProvider.Common.Utils;
|
||||
|
||||
namespace SatelliteProvider.Services.RouteManagement.TileProvision;
|
||||
|
||||
public static class TileResolutionHelper
|
||||
{
|
||||
public static double ResolutionMetersPerPixel(int zoomLevel, double latitude, int tileSizePixels)
|
||||
{
|
||||
var latRad = latitude * Math.PI / 180.0;
|
||||
var metersPerPixel = (GeoUtils.EarthEquatorialCircumferenceMeters * Math.Cos(latRad))
|
||||
/ (Math.Pow(2, zoomLevel) * tileSizePixels);
|
||||
return metersPerPixel;
|
||||
}
|
||||
|
||||
public static double TileSizeMeters(int zoomLevel, double latitude, int tileSizePixels)
|
||||
{
|
||||
return ResolutionMetersPerPixel(zoomLevel, latitude, tileSizePixels) * tileSizePixels;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
using System.Security.Cryptography;
|
||||
using FluentAssertions;
|
||||
using SatelliteProvider.Services.RouteManagement.TileProvision;
|
||||
|
||||
namespace SatelliteProvider.Tests;
|
||||
|
||||
public class ClientTileCatalogTests
|
||||
{
|
||||
private static readonly byte[] ServerHash = SHA256.HashData("server-tile"u8.ToArray());
|
||||
private static readonly byte[] ClientHash = SHA256.HashData("client-tile"u8.ToArray());
|
||||
|
||||
[Fact]
|
||||
public void ShouldSkipForClient_HashMatch_Skips()
|
||||
{
|
||||
var client = new ClientTileSnapshot(18, 1, 2, 1.0, DateTime.UtcNow.AddDays(-1), ServerHash);
|
||||
var server = new ServerTileProspect(2.0, DateTime.UtcNow, ServerHash);
|
||||
|
||||
ClientTileCatalog.ShouldSkipForClient(client, server).Should().BeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ShouldSkipForClient_MetadataSufficient_Skips()
|
||||
{
|
||||
var client = new ClientTileSnapshot(18, 1, 2, 0.5, DateTime.UtcNow, null);
|
||||
var server = new ServerTileProspect(1.0, DateTime.UtcNow.AddHours(-1), null);
|
||||
|
||||
ClientTileCatalog.ShouldSkipForClient(client, server).Should().BeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ShouldSkipForClient_WorseResolution_DoesNotSkip()
|
||||
{
|
||||
var client = new ClientTileSnapshot(18, 1, 2, 2.0, DateTime.UtcNow, null);
|
||||
var server = new ServerTileProspect(1.0, DateTime.UtcNow, null);
|
||||
|
||||
ClientTileCatalog.ShouldSkipForClient(client, server).Should().BeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ShouldSkipForClient_OlderCapture_DoesNotSkip()
|
||||
{
|
||||
var client = new ClientTileSnapshot(18, 1, 2, 0.5, DateTime.UtcNow.AddDays(-2), null);
|
||||
var server = new ServerTileProspect(1.0, DateTime.UtcNow, null);
|
||||
|
||||
ClientTileCatalog.ShouldSkipForClient(client, server).Should().BeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ShouldSkipForClient_NoClientRecord_DoesNotSkip()
|
||||
{
|
||||
var server = new ServerTileProspect(1.0, DateTime.UtcNow, ServerHash);
|
||||
|
||||
ClientTileCatalog.ShouldSkipForClient(null, server).Should().BeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ShouldSkipForClient_DifferentHashWithInsufficientMetadata_DoesNotSkip()
|
||||
{
|
||||
var client = new ClientTileSnapshot(18, 1, 2, 2.0, DateTime.UtcNow.AddDays(-2), ClientHash);
|
||||
var server = new ServerTileProspect(1.0, DateTime.UtcNow, ServerHash);
|
||||
|
||||
ClientTileCatalog.ShouldSkipForClient(client, server).Should().BeFalse();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Moq;
|
||||
using SatelliteProvider.Common.Configs;
|
||||
using SatelliteProvider.Common.Enums;
|
||||
using SatelliteProvider.Common.Interfaces;
|
||||
using SatelliteProvider.Common.Utils;
|
||||
using SatelliteProvider.DataAccess.Models;
|
||||
using SatelliteProvider.DataAccess.Repositories;
|
||||
using SatelliteProvider.Services.RouteManagement;
|
||||
using SatelliteProvider.Services.RouteManagement.TileProvision;
|
||||
|
||||
namespace SatelliteProvider.Tests;
|
||||
|
||||
public class RouteTileDeliveryOrchestratorTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task DeliverAsync_AllTilesSkippedByClient_EmitsManifestAndCompleteWithZeroDelivered()
|
||||
{
|
||||
var expander = new RouteTileExpander(
|
||||
new RoutePointGraphBuilder(Options.Create(new ProcessingConfig { MaxRoutePointSpacingMeters = 200 })),
|
||||
new GeofenceGridCalculator());
|
||||
|
||||
var tileRepo = new Mock<ITileRepository>();
|
||||
tileRepo
|
||||
.Setup(r => r.GetTilesByLocationHashesAsync(It.IsAny<IReadOnlyList<Guid>>()))
|
||||
.ReturnsAsync(new Dictionary<Guid, TileEntity>());
|
||||
|
||||
var tileService = new Mock<ITileService>();
|
||||
var orchestrator = new RouteTileDeliveryOrchestrator(
|
||||
expander,
|
||||
tileRepo.Object,
|
||||
tileService.Object,
|
||||
Options.Create(new MapConfig { TileSizePixels = 256, AllowedZoomLevels = [18] }),
|
||||
Options.Create(new ProcessingConfig { MaxConcurrentDownloads = 2 }),
|
||||
Options.Create(new TileProvisionConfig { MaxTilesPerBatch = 100 }),
|
||||
NullLogger<RouteTileDeliveryOrchestrator>.Instance);
|
||||
|
||||
var waypoints = new List<(double Lat, double Lon)> { (47.0, 37.0), (47.001, 37.001) };
|
||||
var candidates = expander.Expand(waypoints, 400, 18, [], false);
|
||||
candidates.Should().NotBeEmpty();
|
||||
|
||||
var clientTiles = candidates.Select(c =>
|
||||
{
|
||||
var center = GeoUtils.TileToWorldPos(c.X, c.Y, c.Z);
|
||||
var resolution = TileResolutionHelper.ResolutionMetersPerPixel(c.Z, center.Lat, 256);
|
||||
return new ClientTileSnapshot(c.Z, c.X, c.Y, resolution, DateTime.UtcNow.AddHours(1), null);
|
||||
}).ToList();
|
||||
|
||||
var sink = new RecordingSink();
|
||||
var job = new RouteTileDeliveryJob(
|
||||
Guid.NewGuid(),
|
||||
waypoints,
|
||||
400,
|
||||
18,
|
||||
[],
|
||||
false,
|
||||
clientTiles);
|
||||
|
||||
await orchestrator.DeliverAsync(job, sink, CancellationToken.None);
|
||||
|
||||
sink.Manifest.Should().NotBeNull();
|
||||
sink.Manifest!.Value.Total.Should().Be((uint)candidates.Count);
|
||||
sink.Manifest.Value.Skipped.Should().Be((uint)candidates.Count);
|
||||
sink.Manifest.Value.ToDeliver.Should().Be(0);
|
||||
sink.Complete.Should().NotBeNull();
|
||||
sink.Complete!.Value.Delivered.Should().Be(0);
|
||||
sink.Complete.Value.SkippedClient.Should().Be((uint)candidates.Count);
|
||||
sink.Batches.Should().BeEmpty();
|
||||
tileService.Verify(
|
||||
s => s.DownloadAndStoreSingleTileAsync(It.IsAny<double>(), It.IsAny<double>(), It.IsAny<int>(), It.IsAny<CancellationToken>()),
|
||||
Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeliverAsync_CachedTileOnDisk_EmitsBatchWithoutDownload()
|
||||
{
|
||||
var tilesDir = Path.Combine(Path.GetTempPath(), "sp-grpc-test-" + Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(tilesDir);
|
||||
try
|
||||
{
|
||||
var expander = new RouteTileExpander(
|
||||
new RoutePointGraphBuilder(Options.Create(new ProcessingConfig { MaxRoutePointSpacingMeters = 200 })),
|
||||
new GeofenceGridCalculator());
|
||||
|
||||
var waypoints = new List<(double Lat, double Lon)> { (47.0, 37.0), (47.001, 37.001) };
|
||||
var candidates = expander.Expand(waypoints, 400, 18, [], false);
|
||||
var candidate = candidates[0];
|
||||
|
||||
var jpeg = new byte[] { 0xFF, 0xD8, 0xFF, 0xD9 };
|
||||
var tilePath = Path.Combine(tilesDir, "tile.jpg");
|
||||
await File.WriteAllBytesAsync(tilePath, jpeg);
|
||||
var hash = System.Security.Cryptography.SHA256.HashData(jpeg);
|
||||
var center = GeoUtils.TileToWorldPos(candidate.X, candidate.Y, candidate.Z);
|
||||
var locationHash = Uuidv5.LocationHashForTile(candidate.Z, candidate.X, candidate.Y);
|
||||
|
||||
var entity = new TileEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
TileZoom = candidate.Z,
|
||||
TileX = candidate.X,
|
||||
TileY = candidate.Y,
|
||||
Latitude = center.Lat,
|
||||
Longitude = center.Lon,
|
||||
TileSizeMeters = 100,
|
||||
TileSizePixels = 256,
|
||||
ImageType = "jpg",
|
||||
FilePath = tilePath,
|
||||
Source = TileSourceConverter.GoogleMapsWireValue,
|
||||
CapturedAt = DateTime.UtcNow,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
UpdatedAt = DateTime.UtcNow,
|
||||
LocationHash = locationHash,
|
||||
ContentSha256 = hash,
|
||||
};
|
||||
|
||||
var tileRepo = new Mock<ITileRepository>();
|
||||
tileRepo
|
||||
.Setup(r => r.GetTilesByLocationHashesAsync(It.IsAny<IReadOnlyList<Guid>>()))
|
||||
.ReturnsAsync((IReadOnlyList<Guid> hashes) =>
|
||||
{
|
||||
var dict = new Dictionary<Guid, TileEntity>();
|
||||
if (hashes.Contains(locationHash))
|
||||
{
|
||||
dict[locationHash] = entity;
|
||||
}
|
||||
|
||||
return dict;
|
||||
});
|
||||
|
||||
var tileService = new Mock<ITileService>();
|
||||
var orchestrator = new RouteTileDeliveryOrchestrator(
|
||||
expander,
|
||||
tileRepo.Object,
|
||||
tileService.Object,
|
||||
Options.Create(new MapConfig { TileSizePixels = 256, AllowedZoomLevels = [18] }),
|
||||
Options.Create(new ProcessingConfig { MaxConcurrentDownloads = 2 }),
|
||||
Options.Create(new TileProvisionConfig { MaxTilesPerBatch = 100 }),
|
||||
NullLogger<RouteTileDeliveryOrchestrator>.Instance);
|
||||
|
||||
var sink = new RecordingSink();
|
||||
var job = new RouteTileDeliveryJob(Guid.NewGuid(), waypoints, 400, 18, [], false, []);
|
||||
|
||||
await orchestrator.DeliverAsync(job, sink, CancellationToken.None);
|
||||
|
||||
sink.Manifest.Should().NotBeNull();
|
||||
sink.Manifest!.Value.ToDeliver.Should().BeGreaterThan(0);
|
||||
sink.Batches.Should().NotBeEmpty();
|
||||
sink.Batches.SelectMany(b => b).Should().Contain(t => t.Candidate.X == candidate.X && t.Candidate.Y == candidate.Y);
|
||||
sink.Complete.Should().NotBeNull();
|
||||
sink.Complete!.Value.Delivered.Should().BeGreaterThan(0u);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (Directory.Exists(tilesDir))
|
||||
{
|
||||
Directory.Delete(tilesDir, recursive: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class RecordingSink : IRouteTileDeliverySink
|
||||
{
|
||||
public (uint Total, uint Skipped, uint ToDeliver)? Manifest { get; private set; }
|
||||
public List<IReadOnlyList<PreparedTileDelivery>> Batches { get; } = new();
|
||||
public (uint Delivered, uint SkippedClient, uint SkippedServer)? Complete { get; private set; }
|
||||
|
||||
public ValueTask WriteManifestAsync(uint totalCandidates, uint skippedByClient, uint toDeliver, CancellationToken cancellationToken)
|
||||
{
|
||||
Manifest = (totalCandidates, skippedByClient, toDeliver);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask WriteBatchAsync(uint batchSeq, IReadOnlyList<PreparedTileDelivery> tiles, CancellationToken cancellationToken)
|
||||
{
|
||||
Batches.Add(tiles);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask WriteProgressAsync(uint delivered, uint total, uint downloading, CancellationToken cancellationToken)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask WriteCompleteAsync(uint delivered, uint skippedClient, uint skippedServerFilter, CancellationToken cancellationToken)
|
||||
{
|
||||
Complete = (delivered, skippedClient, skippedServerFilter);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask WriteErrorAsync(string code, string message, bool retryable, CancellationToken cancellationToken)
|
||||
=> ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using SatelliteProvider.Common.Configs;
|
||||
using SatelliteProvider.Services.RouteManagement;
|
||||
using SatelliteProvider.Services.RouteManagement.TileProvision;
|
||||
|
||||
namespace SatelliteProvider.Tests;
|
||||
|
||||
public class RouteTileExpanderTests
|
||||
{
|
||||
private readonly RouteTileExpander _expander = new(
|
||||
new RoutePointGraphBuilder(Options.Create(new ProcessingConfig { MaxRoutePointSpacingMeters = 200 })),
|
||||
new GeofenceGridCalculator());
|
||||
|
||||
[Fact]
|
||||
public void Expand_ShortRoute_ProducesTilesForEachWaypointCorridor()
|
||||
{
|
||||
var waypoints = new List<(double Lat, double Lon)>
|
||||
{
|
||||
(47.0, 37.0),
|
||||
(47.001, 37.001),
|
||||
};
|
||||
|
||||
var tiles = _expander.Expand(waypoints, regionSizeMeters: 400, zoom: 18, [], includeGeofenceTiles: false);
|
||||
|
||||
tiles.Should().NotBeEmpty();
|
||||
tiles.Select(t => (t.Z, t.X, t.Y)).Should().OnlyHaveUniqueItems();
|
||||
tiles.Should().OnlyContain(t => t.Z == 18);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Expand_LongSegment_AddsIntermediateCorridors()
|
||||
{
|
||||
var waypoints = new List<(double Lat, double Lon)>
|
||||
{
|
||||
(47.0, 37.0),
|
||||
(47.01, 37.0),
|
||||
};
|
||||
|
||||
var tiles = _expander.Expand(waypoints, regionSizeMeters: 400, zoom: 18, [], includeGeofenceTiles: false);
|
||||
|
||||
tiles.Should().NotBeEmpty();
|
||||
tiles.Select(t => t.RoutePriority).Max().Should().BeGreaterThan(1u);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Expand_WithGeofence_IncludesGeofenceTilesWhenRequested()
|
||||
{
|
||||
var waypoints = new List<(double Lat, double Lon)>
|
||||
{
|
||||
(47.0, 37.0),
|
||||
(47.001, 37.001),
|
||||
};
|
||||
|
||||
var geofence = new List<(double Lat, double Lon)>
|
||||
{
|
||||
(47.002, 37.002),
|
||||
(47.003, 37.002),
|
||||
(47.003, 37.003),
|
||||
(47.002, 37.003),
|
||||
};
|
||||
|
||||
var without = _expander.Expand(waypoints, 400, 18, [], false);
|
||||
var with = _expander.Expand(waypoints, 400, 18, [geofence], true);
|
||||
|
||||
with.Count.Should().BeGreaterThan(without.Count);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package satellite.v1;
|
||||
|
||||
import "google/protobuf/timestamp.proto";
|
||||
|
||||
option csharp_namespace = "Satellite.V1";
|
||||
|
||||
service RouteTileDelivery {
|
||||
rpc DeliverRouteTiles(DeliverRouteTilesRequest) returns (stream RouteTileEvent);
|
||||
}
|
||||
|
||||
message DeliverRouteTilesRequest {
|
||||
RouteSpec route = 1;
|
||||
repeated ClientTileRecord client_tiles = 2;
|
||||
}
|
||||
|
||||
message RouteSpec {
|
||||
string route_id = 1;
|
||||
repeated Waypoint waypoints = 2;
|
||||
double region_size_meters = 3;
|
||||
int32 zoom = 4;
|
||||
repeated GeofencePolygon geofences = 5;
|
||||
bool include_geofence_tiles = 6;
|
||||
}
|
||||
|
||||
message Waypoint {
|
||||
double lat = 1;
|
||||
double lon = 2;
|
||||
}
|
||||
|
||||
message GeofencePolygon {
|
||||
repeated Waypoint vertices = 1;
|
||||
}
|
||||
|
||||
message ClientTileRecord {
|
||||
int32 z = 1;
|
||||
int32 x = 2;
|
||||
int32 y = 3;
|
||||
double resolution_m_per_px = 4;
|
||||
google.protobuf.Timestamp captured_at = 5;
|
||||
optional string source = 6;
|
||||
bytes content_sha256 = 7;
|
||||
}
|
||||
|
||||
message RouteTileEvent {
|
||||
oneof payload {
|
||||
RouteManifest manifest = 1;
|
||||
TileBatch batch = 2;
|
||||
ProgressUpdate progress = 3;
|
||||
DeliveryComplete complete = 4;
|
||||
DeliveryError error = 5;
|
||||
}
|
||||
}
|
||||
|
||||
message RouteManifest {
|
||||
uint32 total_candidates = 1;
|
||||
uint32 skipped_by_client = 2;
|
||||
uint32 to_deliver = 3;
|
||||
}
|
||||
|
||||
message TileBatch {
|
||||
uint32 batch_seq = 1;
|
||||
repeated TilePayload tiles = 2;
|
||||
}
|
||||
|
||||
message TilePayload {
|
||||
int32 z = 1;
|
||||
int32 x = 2;
|
||||
int32 y = 3;
|
||||
double resolution_m_per_px = 4;
|
||||
google.protobuf.Timestamp captured_at = 5;
|
||||
string source = 6;
|
||||
bytes jpeg = 7;
|
||||
bytes content_sha256 = 8;
|
||||
uint32 route_priority = 9;
|
||||
}
|
||||
|
||||
message ProgressUpdate {
|
||||
uint32 delivered = 1;
|
||||
uint32 total = 2;
|
||||
uint32 downloading = 3;
|
||||
}
|
||||
|
||||
message DeliveryComplete {
|
||||
uint32 delivered = 1;
|
||||
uint32 skipped_client = 2;
|
||||
uint32 skipped_server_filter = 3;
|
||||
}
|
||||
|
||||
message DeliveryError {
|
||||
string code = 1;
|
||||
string message = 2;
|
||||
bool retryable = 3;
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
# Contract: RouteTileDelivery (gRPC)
|
||||
|
||||
**Component**: c11_tilemanager (consumer), satellite-provider (producer)
|
||||
**Epic**: AZ-976
|
||||
**ADR**: ADR-013 (architecture.md)
|
||||
**Proto**: `tile_provision.proto` — `package satellite.v1`
|
||||
**Version**: 0.3.0
|
||||
**Status**: proposed
|
||||
**Last Updated**: 2026-06-19
|
||||
|
||||
## Purpose
|
||||
|
||||
Operator-side **pre-flight cache provisioning**. Client sends route + onboard tile catalog once; server streams `RouteTileEvent` messages until `DeliveryComplete` or `DeliveryError`.
|
||||
|
||||
satellite-provider does **not** receive `flight_id` — that is a C6 bookkeeping concern on the gps-denied side only (`route_id` is the wire correlation id).
|
||||
|
||||
C11/C12 on the **operator workstation** only. ADR-004: airborne image must not import stubs or open this channel.
|
||||
|
||||
## RPC
|
||||
|
||||
```protobuf
|
||||
service RouteTileDelivery {
|
||||
rpc DeliverRouteTiles(DeliverRouteTilesRequest) returns (stream RouteTileEvent);
|
||||
}
|
||||
```
|
||||
|
||||
| Concern | Rule |
|
||||
|---------|------|
|
||||
| Auth | gRPC metadata `authorization: Bearer <JWT>` |
|
||||
| TLS | Required in production; `SATELLITE_PROVIDER_TLS_INSECURE=1` dev knob |
|
||||
| Idempotency | `RouteSpec.route_id` (UUID string) |
|
||||
| Resume | Client persists last acked `batch_seq` per `route_id` locally (not on wire) |
|
||||
|
||||
## Request
|
||||
|
||||
### `DeliverRouteTilesRequest`
|
||||
|
||||
| Field | Description |
|
||||
|-------|-------------|
|
||||
| `route` | Corridor geometry + single zoom |
|
||||
| `client_tiles` | Onboard inventory snapshot (route intersection only) |
|
||||
|
||||
### `RouteSpec`
|
||||
|
||||
| Field | Maps from gps-denied |
|
||||
|-------|----------------------|
|
||||
| `route_id` | Client-generated UUID per provision job |
|
||||
| `waypoints` | `replay_input.tlog_route.RouteSpec.waypoints` |
|
||||
| `region_size_meters` | `RouteSpec.suggested_region_size_meters` |
|
||||
| `zoom` | Single slippy zoom level (confirmed sufficient) |
|
||||
| `geofences` | Optional inclusion polygons |
|
||||
| `include_geofence_tiles` | Union geofence tiles with corridor grid |
|
||||
|
||||
### `ClientTileRecord`
|
||||
|
||||
Canonical key: **`(z, x, y)`**. `source` is informational only — **not** used in skip logic.
|
||||
|
||||
| Field | C6 mapping |
|
||||
|-------|------------|
|
||||
| `resolution_m_per_px` | RESTRICT-SAT-4 (lower = better) |
|
||||
| `captured_at` | `TileMetadata.capture_timestamp` |
|
||||
| `content_sha256` | `TileMetadata.content_sha256_hex` (raw 32 bytes) |
|
||||
|
||||
## Server skip rule (client catalog)
|
||||
|
||||
For each server candidate tile, **omit from stream** when `client_tiles` has matching `(z,x,y)` and **any** of:
|
||||
|
||||
1. `client.content_sha256` is non-empty and **equals** server payload hash → skip (byte-identical)
|
||||
2. `client.resolution_m_per_px <= server.resolution_m_per_px` **and** `client.captured_at >= server.captured_at` → skip (metadata-sufficient)
|
||||
|
||||
`source` is **not** compared.
|
||||
|
||||
`RouteManifest.skipped_by_client` counts tiles removed by this rule.
|
||||
|
||||
## Sector — not on this wire
|
||||
|
||||
**Sector** (`active_conflict` vs `stable_rear`) controls **how stale a tile may be before C6 rejects it on write** (AC-NEW-6 freshness). It is an operator decision about the geographic area, not something satellite-provider needs to deliver tiles.
|
||||
|
||||
| Layer | Who applies sector |
|
||||
|-------|-------------------|
|
||||
| satellite-provider | Does not need sector — streams tiles by route geometry |
|
||||
| C11 client write | Reads sector from **C11/C12 config** (same as today) when calling C6 freshness gate |
|
||||
|
||||
No `SectorClass` field on the gRPC request.
|
||||
|
||||
## Response stream: `RouteTileEvent`
|
||||
|
||||
Typical sequence:
|
||||
|
||||
1. **`RouteManifest`** — `total_candidates`, `skipped_by_client`, `to_deliver`
|
||||
2. **`TileBatch`** — monotonic `batch_seq`; on-disk hits first, then freshly fetched
|
||||
3. **`ProgressUpdate`** — optional
|
||||
4. **`DeliveryComplete`** or **`DeliveryError`**
|
||||
|
||||
### `DeliveryComplete` counters
|
||||
|
||||
| Field | Meaning |
|
||||
|-------|---------|
|
||||
| `delivered` | Tiles actually sent in `TileBatch` streams |
|
||||
| `skipped_client` | Same as manifest `skipped_by_client` (echo for client verify) |
|
||||
| `skipped_server_filter` | Tiles SP required but **did not send** after client dedup — see below |
|
||||
|
||||
#### `skipped_server_filter` — what counts
|
||||
|
||||
Tiles that entered the post-client-dedup work queue but never appeared in a batch:
|
||||
|
||||
| Reason | Example |
|
||||
|--------|---------|
|
||||
| **Fetch failed** | External imagery provider 404/timeout after retries |
|
||||
| **Below SP min resolution** | SP refuses to store/serve below its configured floor |
|
||||
| **Geometry clip** | Tile dropped after server-side corridor/geofence validation |
|
||||
| **Operational cap** | Job hit max-tiles / rate limit (if SP enforces) |
|
||||
|
||||
Tiles skipped by the **client catalog rule** are **not** included here (they are `skipped_client`).
|
||||
|
||||
If SP has no server-side filters in v1, `skipped_server_filter` may be **0**; the field is reserved for observability.
|
||||
|
||||
### `TilePayload`
|
||||
|
||||
| Field | Notes |
|
||||
|-------|-------|
|
||||
| `content_sha256` | 32-byte SHA-256 of `jpeg`; matches C6 DB invariant |
|
||||
| `route_priority` | Lower = earlier along route |
|
||||
|
||||
## Client write path (gps-denied)
|
||||
|
||||
`RouteTileDeliveryClient` (C11):
|
||||
|
||||
- Assigns C6 `flight_id` from operator context locally (not from SP)
|
||||
- Applies RESTRICT-SAT-4, **sector-based freshness**, AZ-308 budget, download journal
|
||||
- Resumes via persisted `route_id` + `batch_seq`
|
||||
|
||||
## Migration
|
||||
|
||||
REST `route_client` + `HttpTileDownloader` remain fallback until AZ-979 benchmark.
|
||||
|
||||
## Change log
|
||||
|
||||
| Version | Date | Change |
|
||||
|---------|------|--------|
|
||||
| 0.3.0 | 2026-06-19 | `ClientTileRecord.content_sha256`; sequential field nums on `TilePayload`; sector/flight_id off wire; skip rule + `skipped_server_filter` defined |
|
||||
| 0.2.0 | 2026-06-19 | `satellite.v1.RouteTileDelivery` + `RouteTileEvent` oneof |
|
||||
| 0.1.0 | 2026-06-19 | Initial draft (superseded) |
|
||||
@@ -17,6 +17,7 @@ services:
|
||||
retries: 5
|
||||
|
||||
api:
|
||||
platform: linux/amd64
|
||||
build:
|
||||
context: .
|
||||
dockerfile: SatelliteProvider.Api/Dockerfile
|
||||
|
||||
Reference in New Issue
Block a user