mirror of
https://github.com/azaion/satellite-provider.git
synced 2026-06-21 08:51:13 +00:00
[AZ-488] UAV tile batch upload + 5-rule quality gate
Replaces the 501 stub at POST /api/satellite/upload with a multipart
batch endpoint that ingests UAV-captured tiles, runs each item through
a 5-rule quality gate, and persists accepted tiles via the AZ-484
multi-source storage path with source='uav'.
Quality gate (in fixed order, first failure wins): JPEG format
(content-type + magic), size band 5 KiB-5 MiB, exact 256x256
dimensions, captured-at age (no future >30 s skew, no older than
7 days), luminance variance on 32x32 downsample. Closed reject-reason
enumeration in v1.0.0 contract.
Authorization: custom PermissionsRequirement / PermissionsAuthorization
Handler that reads the JWT `permissions` claim (tolerates both
repeated-string and JSON-array shapes). Endpoint protected by
RequiresGpsPermission policy; 401 without token, 403 without GPS perm.
Persistence: file-first to ./tiles/uav/{z}/{x}/{y}.jpg, then
ITileRepository.InsertAsync UPSERT (per-source UPSERT contract from
AZ-484). Per-item failures reported in response without aborting the
batch. Kestrel MaxRequestBodySize and FormOptions limits set to
MaxBatchSize x MaxBytes (default 100 x 5 MiB = 500 MiB).
New frozen contract: _docs/02_document/contracts/api/uav-tile-upload.md
v1.0.0. PT-08 NFR added to performance-tests.md as Deferred (harness
work tracked in PT-07 leftover, per AZ-488 § Risk 4).
Tests: 11 quality-gate unit tests, 5 handler unit tests, 3 file-path
unit tests, 12 permission-handler unit tests, 7 integration tests
(AC-1..AC-6, AC-8). All 253 unit tests + smoke integration suite
green.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
+1
@@ -12,6 +12,7 @@
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.10" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="9.0.10" />
|
||||
<PackageReference Include="Newtonsoft.Json" Version="13.0.4" />
|
||||
<PackageReference Include="SixLabors.ImageSharp" Version="3.1.11" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@@ -15,6 +15,8 @@ public static class TileDownloaderServiceCollectionExtensions
|
||||
});
|
||||
services.AddSingleton<ISatelliteDownloader, GoogleMapsDownloaderV2>();
|
||||
services.AddSingleton<ITileService, TileService>();
|
||||
services.AddSingleton<IUavTileQualityGate, UavTileQualityGate>();
|
||||
services.AddSingleton<IUavTileUploadHandler, UavTileUploadHandler>();
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
using SatelliteProvider.Common.Configs;
|
||||
using SatelliteProvider.Common.DTO;
|
||||
using SixLabors.ImageSharp;
|
||||
using SixLabors.ImageSharp.PixelFormats;
|
||||
using SixLabors.ImageSharp.Processing;
|
||||
|
||||
namespace SatelliteProvider.Services.TileDownloader;
|
||||
|
||||
public interface IUavTileQualityGate
|
||||
{
|
||||
UavTileQualityResult Validate(ReadOnlyMemory<byte> imageBytes, string? contentType, UavTileMetadata metadata);
|
||||
}
|
||||
|
||||
public sealed record UavTileQualityResult(bool Accepted, string? Reason, string? Details)
|
||||
{
|
||||
public static UavTileQualityResult Pass() => new(true, null, null);
|
||||
public static UavTileQualityResult Fail(string reason, string? details = null) => new(false, reason, details);
|
||||
}
|
||||
|
||||
public sealed class UavTileQualityGate : IUavTileQualityGate
|
||||
{
|
||||
private static readonly byte[] JpegMagicBytes = { 0xFF, 0xD8, 0xFF };
|
||||
|
||||
private readonly UavQualityConfig _qualityConfig;
|
||||
private readonly MapConfig _mapConfig;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
public UavTileQualityGate(
|
||||
IOptions<UavQualityConfig> qualityConfig,
|
||||
IOptions<MapConfig> mapConfig,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(qualityConfig);
|
||||
ArgumentNullException.ThrowIfNull(mapConfig);
|
||||
|
||||
_qualityConfig = qualityConfig.Value;
|
||||
_mapConfig = mapConfig.Value;
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public UavTileQualityResult Validate(ReadOnlyMemory<byte> imageBytes, string? contentType, UavTileMetadata metadata)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(metadata);
|
||||
|
||||
// Rule 1 (Format): content-type AND magic bytes both indicate JPEG.
|
||||
if (!HasJpegContentType(contentType) || !HasJpegMagicBytes(imageBytes.Span))
|
||||
{
|
||||
return UavTileQualityResult.Fail(UavTileRejectReasons.InvalidFormat);
|
||||
}
|
||||
|
||||
// Rule 2 (Size band): configurable lower/upper bounds.
|
||||
if (imageBytes.Length < _qualityConfig.MinBytes || imageBytes.Length > _qualityConfig.MaxBytes)
|
||||
{
|
||||
return UavTileQualityResult.Fail(UavTileRejectReasons.SizeOutOfBand);
|
||||
}
|
||||
|
||||
// Rule 3 (Dimensions): strict equality with MapConfig.TileSizePixels.
|
||||
// ImageSharp.Image.Identify reads only the JPEG header — no full decode cost.
|
||||
ImageInfo? info;
|
||||
try
|
||||
{
|
||||
using var identifyStream = new ReadOnlyMemoryStream(imageBytes);
|
||||
info = Image.Identify(identifyStream);
|
||||
}
|
||||
catch (UnknownImageFormatException)
|
||||
{
|
||||
return UavTileQualityResult.Fail(UavTileRejectReasons.InvalidFormat);
|
||||
}
|
||||
catch (InvalidImageContentException)
|
||||
{
|
||||
return UavTileQualityResult.Fail(UavTileRejectReasons.InvalidFormat);
|
||||
}
|
||||
|
||||
if (info is null || info.Width != _mapConfig.TileSizePixels || info.Height != _mapConfig.TileSizePixels)
|
||||
{
|
||||
return UavTileQualityResult.Fail(UavTileRejectReasons.WrongDimensions);
|
||||
}
|
||||
|
||||
// Rule 4 (Captured-at age): forbid future timestamps beyond clock skew and
|
||||
// reject anything older than the configured max age.
|
||||
var now = _timeProvider.GetUtcNow().UtcDateTime;
|
||||
var capturedAt = metadata.CapturedAt.Kind == DateTimeKind.Utc
|
||||
? metadata.CapturedAt
|
||||
: metadata.CapturedAt.ToUniversalTime();
|
||||
|
||||
if (capturedAt > now.AddSeconds(_qualityConfig.CapturedAtFutureSkewSeconds))
|
||||
{
|
||||
return UavTileQualityResult.Fail(UavTileRejectReasons.CapturedAtFuture);
|
||||
}
|
||||
|
||||
if (capturedAt < now.AddDays(-_qualityConfig.MaxAgeDays))
|
||||
{
|
||||
return UavTileQualityResult.Fail(UavTileRejectReasons.CapturedAtTooOld);
|
||||
}
|
||||
|
||||
// Rule 5 (Blank/uniform): pixel-luminance variance on a downsampled image.
|
||||
// Runs last because it requires the full decode pass.
|
||||
try
|
||||
{
|
||||
var variance = ComputeLuminanceVariance(imageBytes);
|
||||
if (variance < _qualityConfig.MinLuminanceVariance)
|
||||
{
|
||||
return UavTileQualityResult.Fail(UavTileRejectReasons.ImageTooUniform);
|
||||
}
|
||||
}
|
||||
catch (UnknownImageFormatException)
|
||||
{
|
||||
return UavTileQualityResult.Fail(UavTileRejectReasons.InvalidFormat);
|
||||
}
|
||||
catch (InvalidImageContentException)
|
||||
{
|
||||
return UavTileQualityResult.Fail(UavTileRejectReasons.InvalidFormat);
|
||||
}
|
||||
|
||||
return UavTileQualityResult.Pass();
|
||||
}
|
||||
|
||||
private static bool HasJpegContentType(string? contentType)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(contentType))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Allow trailing parameters such as "image/jpeg; charset=binary".
|
||||
var separator = contentType.IndexOf(';');
|
||||
var mediaType = separator >= 0 ? contentType[..separator].Trim() : contentType.Trim();
|
||||
return string.Equals(mediaType, "image/jpeg", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static bool HasJpegMagicBytes(ReadOnlySpan<byte> bytes)
|
||||
{
|
||||
if (bytes.Length < JpegMagicBytes.Length)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
for (var i = 0; i < JpegMagicBytes.Length; i++)
|
||||
{
|
||||
if (bytes[i] != JpegMagicBytes[i])
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private double ComputeLuminanceVariance(ReadOnlyMemory<byte> imageBytes)
|
||||
{
|
||||
using var stream = new ReadOnlyMemoryStream(imageBytes);
|
||||
using var image = Image.Load<L8>(stream);
|
||||
|
||||
var sampleSize = Math.Clamp(_qualityConfig.LuminanceSampleSize, 4, _mapConfig.TileSizePixels);
|
||||
if (image.Width != sampleSize || image.Height != sampleSize)
|
||||
{
|
||||
image.Mutate(ctx => ctx.Resize(sampleSize, sampleSize));
|
||||
}
|
||||
|
||||
double mean = 0.0;
|
||||
double m2 = 0.0;
|
||||
long n = 0;
|
||||
|
||||
image.ProcessPixelRows(accessor =>
|
||||
{
|
||||
for (var y = 0; y < accessor.Height; y++)
|
||||
{
|
||||
var row = accessor.GetRowSpan(y);
|
||||
for (var x = 0; x < row.Length; x++)
|
||||
{
|
||||
n++;
|
||||
double value = row[x].PackedValue;
|
||||
var delta = value - mean;
|
||||
mean += delta / n;
|
||||
m2 += delta * (value - mean);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return n > 1 ? m2 / (n - 1) : 0.0;
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class ReadOnlyMemoryStream : Stream
|
||||
{
|
||||
private readonly ReadOnlyMemory<byte> _memory;
|
||||
private int _position;
|
||||
|
||||
public ReadOnlyMemoryStream(ReadOnlyMemory<byte> memory)
|
||||
{
|
||||
_memory = memory;
|
||||
}
|
||||
|
||||
public override bool CanRead => true;
|
||||
public override bool CanSeek => true;
|
||||
public override bool CanWrite => false;
|
||||
public override long Length => _memory.Length;
|
||||
|
||||
public override long Position
|
||||
{
|
||||
get => _position;
|
||||
set => _position = checked((int)value);
|
||||
}
|
||||
|
||||
public override void Flush() { }
|
||||
|
||||
public override int Read(byte[] buffer, int offset, int count)
|
||||
{
|
||||
var span = _memory.Span;
|
||||
var remaining = span.Length - _position;
|
||||
if (remaining <= 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var toCopy = Math.Min(remaining, count);
|
||||
span.Slice(_position, toCopy).CopyTo(buffer.AsSpan(offset, toCopy));
|
||||
_position += toCopy;
|
||||
return toCopy;
|
||||
}
|
||||
|
||||
public override long Seek(long offset, SeekOrigin origin)
|
||||
{
|
||||
_position = origin switch
|
||||
{
|
||||
SeekOrigin.Begin => checked((int)offset),
|
||||
SeekOrigin.Current => checked(_position + (int)offset),
|
||||
SeekOrigin.End => checked(_memory.Length + (int)offset),
|
||||
_ => throw new ArgumentOutOfRangeException(nameof(origin)),
|
||||
};
|
||||
return _position;
|
||||
}
|
||||
|
||||
public override void SetLength(long value) => throw new NotSupportedException();
|
||||
public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using SatelliteProvider.Common.Configs;
|
||||
using SatelliteProvider.Common.DTO;
|
||||
using SatelliteProvider.Common.Enums;
|
||||
using SatelliteProvider.Common.Utils;
|
||||
using SatelliteProvider.DataAccess.Models;
|
||||
using SatelliteProvider.DataAccess.Repositories;
|
||||
|
||||
namespace SatelliteProvider.Services.TileDownloader;
|
||||
|
||||
public interface IUavTileUploadHandler
|
||||
{
|
||||
Task<UavTileUploadHandlerResult> HandleAsync(string metadataJson, IReadOnlyList<UavUploadFile> files, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
public sealed record UavUploadFile(string FileName, string? ContentType, ReadOnlyMemory<byte> Content);
|
||||
|
||||
public sealed record UavTileUploadHandlerResult(bool EnvelopeRejected, string? EnvelopeError, UavTileBatchUploadResponse? Response);
|
||||
|
||||
public sealed class UavTileUploadHandler : IUavTileUploadHandler
|
||||
{
|
||||
private const string UavTileFileExtension = ".jpg";
|
||||
private const string UavTileSubdirectory = "uav";
|
||||
|
||||
private readonly IUavTileQualityGate _qualityGate;
|
||||
private readonly ITileRepository _tileRepository;
|
||||
private readonly StorageConfig _storageConfig;
|
||||
private readonly MapConfig _mapConfig;
|
||||
private readonly UavQualityConfig _qualityConfig;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<UavTileUploadHandler> _logger;
|
||||
|
||||
public UavTileUploadHandler(
|
||||
IUavTileQualityGate qualityGate,
|
||||
ITileRepository tileRepository,
|
||||
IOptions<StorageConfig> storageConfig,
|
||||
IOptions<MapConfig> mapConfig,
|
||||
IOptions<UavQualityConfig> qualityConfig,
|
||||
ILogger<UavTileUploadHandler> logger,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
_qualityGate = qualityGate ?? throw new ArgumentNullException(nameof(qualityGate));
|
||||
_tileRepository = tileRepository ?? throw new ArgumentNullException(nameof(tileRepository));
|
||||
_storageConfig = storageConfig?.Value ?? throw new ArgumentNullException(nameof(storageConfig));
|
||||
_mapConfig = mapConfig?.Value ?? throw new ArgumentNullException(nameof(mapConfig));
|
||||
_qualityConfig = qualityConfig?.Value ?? throw new ArgumentNullException(nameof(qualityConfig));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
private static readonly JsonSerializerOptions MetadataJsonOptions = new()
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
};
|
||||
|
||||
public async Task<UavTileUploadHandlerResult> HandleAsync(string metadataJson, IReadOnlyList<UavUploadFile> files, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(files);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(metadataJson))
|
||||
{
|
||||
return EnvelopeError("Request `metadata` field is required.");
|
||||
}
|
||||
|
||||
UavTileBatchMetadataPayload? payload;
|
||||
try
|
||||
{
|
||||
payload = JsonSerializer.Deserialize<UavTileBatchMetadataPayload>(metadataJson, MetadataJsonOptions);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
return EnvelopeError($"Invalid `metadata` JSON: {ex.Message}");
|
||||
}
|
||||
|
||||
if (payload is null || payload.Items.Count == 0)
|
||||
{
|
||||
return EnvelopeError("Request `metadata.items` is required and must contain at least one entry.");
|
||||
}
|
||||
|
||||
if (payload.Items.Count != files.Count)
|
||||
{
|
||||
return EnvelopeError($"Mismatched batch: metadata has {payload.Items.Count} entries but {files.Count} file(s) were uploaded.");
|
||||
}
|
||||
|
||||
if (payload.Items.Count > _qualityConfig.MaxBatchSize)
|
||||
{
|
||||
return EnvelopeError($"Batch size {payload.Items.Count} exceeds the configured maximum of {_qualityConfig.MaxBatchSize}.");
|
||||
}
|
||||
|
||||
var response = new UavTileBatchUploadResponse();
|
||||
for (var index = 0; index < payload.Items.Count; index++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var metadata = payload.Items[index];
|
||||
var file = files[index];
|
||||
|
||||
var gateResult = _qualityGate.Validate(file.Content, file.ContentType, metadata);
|
||||
if (!gateResult.Accepted)
|
||||
{
|
||||
response.Items.Add(new UavTileUploadResultItem
|
||||
{
|
||||
Index = index,
|
||||
Status = UavTileUploadStatus.Rejected,
|
||||
RejectReason = gateResult.Reason,
|
||||
RejectDetails = gateResult.Details,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var persistedId = await PersistAsync(metadata, file.Content, cancellationToken);
|
||||
response.Items.Add(new UavTileUploadResultItem
|
||||
{
|
||||
Index = index,
|
||||
Status = UavTileUploadStatus.Accepted,
|
||||
TileId = persistedId,
|
||||
});
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
_logger.LogError(ex, "UAV tile persistence failed at index {Index}", index);
|
||||
response.Items.Add(new UavTileUploadResultItem
|
||||
{
|
||||
Index = index,
|
||||
Status = UavTileUploadStatus.Rejected,
|
||||
RejectReason = UavTileRejectReasons.StorageFailure,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return new UavTileUploadHandlerResult(EnvelopeRejected: false, EnvelopeError: null, Response: response);
|
||||
}
|
||||
|
||||
private async Task<Guid> PersistAsync(UavTileMetadata metadata, ReadOnlyMemory<byte> imageBytes, CancellationToken cancellationToken)
|
||||
{
|
||||
var (tileX, tileY) = GeoUtils.WorldToTilePos(new GeoPoint(metadata.Latitude, metadata.Longitude), metadata.TileZoom);
|
||||
var filePath = BuildUavTileFilePath(_storageConfig, metadata.TileZoom, tileX, tileY);
|
||||
|
||||
var directory = Path.GetDirectoryName(filePath);
|
||||
if (!string.IsNullOrEmpty(directory))
|
||||
{
|
||||
Directory.CreateDirectory(directory);
|
||||
}
|
||||
|
||||
// File-first, row-second so a crash leaves an orphan file rather than a row
|
||||
// pointing at nothing (Risk 2 in the AZ-488 task spec).
|
||||
await File.WriteAllBytesAsync(filePath, imageBytes.ToArray(), cancellationToken);
|
||||
|
||||
var capturedAtUtc = metadata.CapturedAt.Kind == DateTimeKind.Utc
|
||||
? metadata.CapturedAt
|
||||
: metadata.CapturedAt.ToUniversalTime();
|
||||
var now = _timeProvider.GetUtcNow().UtcDateTime;
|
||||
|
||||
var entity = new TileEntity
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
TileZoom = metadata.TileZoom,
|
||||
TileX = tileX,
|
||||
TileY = tileY,
|
||||
Latitude = metadata.Latitude,
|
||||
Longitude = metadata.Longitude,
|
||||
TileSizeMeters = metadata.TileSizeMeters,
|
||||
TileSizePixels = _mapConfig.TileSizePixels,
|
||||
ImageType = "jpg",
|
||||
MapsVersion = null,
|
||||
Version = null,
|
||||
FilePath = filePath,
|
||||
Source = TileSourceConverter.ToWireValue(TileSource.Uav),
|
||||
CapturedAt = capturedAtUtc,
|
||||
CreatedAt = now,
|
||||
UpdatedAt = now,
|
||||
};
|
||||
|
||||
return await _tileRepository.InsertAsync(entity);
|
||||
}
|
||||
|
||||
public static string BuildUavTileFilePath(StorageConfig storageConfig, int tileZoom, int tileX, int tileY)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(storageConfig);
|
||||
return Path.Combine(
|
||||
storageConfig.TilesDirectory,
|
||||
UavTileSubdirectory,
|
||||
tileZoom.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
||||
tileX.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
||||
tileY.ToString(System.Globalization.CultureInfo.InvariantCulture) + UavTileFileExtension);
|
||||
}
|
||||
|
||||
private static UavTileUploadHandlerResult EnvelopeError(string detail) =>
|
||||
new(EnvelopeRejected: true, EnvelopeError: detail, Response: null);
|
||||
}
|
||||
Reference in New Issue
Block a user