From bbb112940d8cac8c80577821f1b5cb6aaee45bd0 Mon Sep 17 00:00:00 2001 From: Anton Martynenko Date: Tue, 28 Oct 2025 15:56:16 +0100 Subject: [PATCH] first region implementation --- SatelliteProvider.Api/Program.cs | 84 +++++++ SatelliteProvider.Common/DTO/RegionRequest.cs | 11 + SatelliteProvider.Common/DTO/RegionStatus.cs | 14 ++ .../Interfaces/IRegionRequestQueue.cs | 11 + .../Interfaces/IRegionService.cs | 11 + SatelliteProvider.IntegrationTests/Program.cs | 139 ++++++++++++ .../RegionProcessingService.cs | 53 +++++ .../RegionRequestQueue.cs | 39 ++++ SatelliteProvider.Services/RegionService.cs | 213 ++++++++++++++++++ .../SatelliteProvider.Services.csproj | 1 + 10 files changed, 576 insertions(+) create mode 100644 SatelliteProvider.Common/DTO/RegionRequest.cs create mode 100644 SatelliteProvider.Common/DTO/RegionStatus.cs create mode 100644 SatelliteProvider.Common/Interfaces/IRegionRequestQueue.cs create mode 100644 SatelliteProvider.Common/Interfaces/IRegionService.cs create mode 100644 SatelliteProvider.Services/RegionProcessingService.cs create mode 100644 SatelliteProvider.Services/RegionRequestQueue.cs create mode 100644 SatelliteProvider.Services/RegionService.cs diff --git a/SatelliteProvider.Api/Program.cs b/SatelliteProvider.Api/Program.cs index 765c021..fd8145d 100644 --- a/SatelliteProvider.Api/Program.cs +++ b/SatelliteProvider.Api/Program.cs @@ -6,6 +6,7 @@ using SatelliteProvider.DataAccess; using SatelliteProvider.DataAccess.Models; using SatelliteProvider.DataAccess.Repositories; using SatelliteProvider.Common.Configs; +using SatelliteProvider.Common.DTO; using SatelliteProvider.Common.Interfaces; using SatelliteProvider.Services; @@ -26,6 +27,11 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +var processingConfig = builder.Configuration.GetSection("ProcessingConfig").Get() ?? new ProcessingConfig(); +builder.Services.AddSingleton(sp => new RegionRequestQueue(processingConfig.QueueCapacity)); +builder.Services.AddSingleton(); +builder.Services.AddHostedService(); + builder.Services.AddEndpointsApiExplorer(); builder.Services.AddSwaggerGen(c => { @@ -60,6 +66,12 @@ if (!migrator.RunMigrations()) throw new Exception("Database migration failed. Application cannot start."); } +var storageConfig = app.Configuration.GetSection("StorageConfig").Get() ?? new StorageConfig(); +Directory.CreateDirectory(storageConfig.TilesDirectory); +Directory.CreateDirectory(storageConfig.ReadyDirectory); +logger.LogInformation("Storage directories created: Tiles={TilesDir}, Ready={ReadyDir}", + storageConfig.TilesDirectory, storageConfig.ReadyDirectory); + if (app.Environment.IsDevelopment()) { app.UseSwagger(); @@ -82,6 +94,12 @@ app.MapPost("/api/satellite/upload", UploadImage) app.MapPost("/api/satellite/tiles/download", DownloadSingleTile) .WithOpenApi(op => new(op) { Summary = "TEMPORARY: Download single tile at specified coordinates" }); +app.MapPost("/api/satellite/request", RequestRegion) + .WithOpenApi(op => new(op) { Summary = "Request tiles for a region" }); + +app.MapGet("/api/satellite/region/{id:guid}", GetRegionStatus) + .WithOpenApi(op => new(op) { Summary = "Get region status and file paths" }); + app.Run(); IResult GetSatelliteTilesByLatLon(double lat, double lon, double squareSideMeters) @@ -154,6 +172,54 @@ async Task DownloadSingleTile([FromBody] DownloadTileRequest request, G } } +async Task RequestRegion([FromBody] RequestRegionRequest request, IRegionService regionService, ILogger logger) +{ + try + { + if (request.SizeMeters < 100 || request.SizeMeters > 10000) + { + return Results.BadRequest(new { error = "Size must be between 100 and 10000 meters" }); + } + + logger.LogInformation("Region request received: ID={Id}, Lat={Lat}, Lon={Lon}, Size={Size}m, Zoom={Zoom}", + request.Id, request.Latitude, request.Longitude, request.SizeMeters, request.ZoomLevel); + + var status = await regionService.RequestRegionAsync( + request.Id, + request.Latitude, + request.Longitude, + request.SizeMeters, + request.ZoomLevel); + + return Results.Ok(status); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to request region"); + return Results.Problem(detail: ex.Message, statusCode: 500); + } +} + +async Task GetRegionStatus(Guid id, IRegionService regionService, ILogger logger) +{ + try + { + var status = await regionService.GetRegionStatusAsync(id); + + if (status == null) + { + return Results.NotFound(new { error = $"Region {id} not found" }); + } + + return Results.Ok(status); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to get region status"); + return Results.Problem(detail: ex.Message, statusCode: 500); + } +} + public record GetSatelliteTilesResponse { public List Tiles { get; set; } = new(); @@ -228,6 +294,24 @@ public record DownloadTileResponse public DateTime UpdatedAt { get; set; } } +public record RequestRegionRequest +{ + [Required] + public Guid Id { get; set; } + + [Required] + public double Latitude { get; set; } + + [Required] + public double Longitude { get; set; } + + [Required] + public double SizeMeters { get; set; } + + [Required] + public int ZoomLevel { get; set; } = 18; +} + public class ParameterDescriptionFilter : IOperationFilter { public void Apply(OpenApiOperation operation, OperationFilterContext context) diff --git a/SatelliteProvider.Common/DTO/RegionRequest.cs b/SatelliteProvider.Common/DTO/RegionRequest.cs new file mode 100644 index 0000000..920a68d --- /dev/null +++ b/SatelliteProvider.Common/DTO/RegionRequest.cs @@ -0,0 +1,11 @@ +namespace SatelliteProvider.Common.DTO; + +public class RegionRequest +{ + public Guid Id { get; set; } + public double Latitude { get; set; } + public double Longitude { get; set; } + public double SizeMeters { get; set; } + public int ZoomLevel { get; set; } +} + diff --git a/SatelliteProvider.Common/DTO/RegionStatus.cs b/SatelliteProvider.Common/DTO/RegionStatus.cs new file mode 100644 index 0000000..e7e831e --- /dev/null +++ b/SatelliteProvider.Common/DTO/RegionStatus.cs @@ -0,0 +1,14 @@ +namespace SatelliteProvider.Common.DTO; + +public class RegionStatus +{ + public Guid Id { get; set; } + public string Status { get; set; } = string.Empty; + public string? CsvFilePath { get; set; } + public string? SummaryFilePath { get; set; } + public int TilesDownloaded { get; set; } + public int TilesReused { get; set; } + public DateTime CreatedAt { get; set; } + public DateTime UpdatedAt { get; set; } +} + diff --git a/SatelliteProvider.Common/Interfaces/IRegionRequestQueue.cs b/SatelliteProvider.Common/Interfaces/IRegionRequestQueue.cs new file mode 100644 index 0000000..e64df71 --- /dev/null +++ b/SatelliteProvider.Common/Interfaces/IRegionRequestQueue.cs @@ -0,0 +1,11 @@ +using SatelliteProvider.Common.DTO; + +namespace SatelliteProvider.Common.Interfaces; + +public interface IRegionRequestQueue +{ + ValueTask EnqueueAsync(RegionRequest request, CancellationToken cancellationToken = default); + ValueTask DequeueAsync(CancellationToken cancellationToken = default); + int Count { get; } +} + diff --git a/SatelliteProvider.Common/Interfaces/IRegionService.cs b/SatelliteProvider.Common/Interfaces/IRegionService.cs new file mode 100644 index 0000000..8d72469 --- /dev/null +++ b/SatelliteProvider.Common/Interfaces/IRegionService.cs @@ -0,0 +1,11 @@ +using SatelliteProvider.Common.DTO; + +namespace SatelliteProvider.Common.Interfaces; + +public interface IRegionService +{ + Task RequestRegionAsync(Guid id, double latitude, double longitude, double sizeMeters, int zoomLevel); + Task GetRegionStatusAsync(Guid id); + Task ProcessRegionAsync(Guid id, CancellationToken cancellationToken = default); +} + diff --git a/SatelliteProvider.IntegrationTests/Program.cs b/SatelliteProvider.IntegrationTests/Program.cs index d8127fc..685dbb5 100644 --- a/SatelliteProvider.IntegrationTests/Program.cs +++ b/SatelliteProvider.IntegrationTests/Program.cs @@ -33,6 +33,8 @@ class Program Console.WriteLine(); await RunSingleTileDownloadTest(httpClient); + + await RunRegionProcessingTest(httpClient); Console.WriteLine(); Console.WriteLine("========================="); @@ -164,6 +166,122 @@ class Program Console.WriteLine(); Console.WriteLine("Single Tile Download Test: PASSED"); } + + static async Task RunRegionProcessingTest(HttpClient httpClient) + { + Console.WriteLine(); + Console.WriteLine("Test: Region Processing at Coordinates 47.461747, 37.647063"); + Console.WriteLine("------------------------------------------------------------------"); + + const double latitude = 47.461747; + const double longitude = 37.647063; + const double sizeMeters = 200; + const int zoomLevel = 18; + + var regionId = Guid.NewGuid(); + + Console.WriteLine($"Requesting region: ID={regionId}"); + Console.WriteLine($" Coordinates: ({latitude}, {longitude})"); + Console.WriteLine($" Size: {sizeMeters}m"); + Console.WriteLine($" Zoom Level: {zoomLevel}"); + Console.WriteLine(); + + var requestRegion = new RequestRegionRequest + { + Id = regionId, + Latitude = latitude, + Longitude = longitude, + SizeMeters = sizeMeters, + ZoomLevel = zoomLevel + }; + + var requestResponse = await httpClient.PostAsJsonAsync("/api/satellite/request", requestRegion); + + if (!requestResponse.IsSuccessStatusCode) + { + var errorContent = await requestResponse.Content.ReadAsStringAsync(); + throw new Exception($"Region request failed with status {requestResponse.StatusCode}: {errorContent}"); + } + + var initialStatus = await requestResponse.Content.ReadFromJsonAsync(JsonOptions); + + if (initialStatus == null) + { + throw new Exception("No status returned from region request"); + } + + Console.WriteLine($"✓ Region queued successfully"); + Console.WriteLine($" Initial Status: {initialStatus.Status}"); + Console.WriteLine(); + + Console.WriteLine("Polling for region status updates..."); + RegionStatusResponse? finalStatus = null; + int maxAttempts = 30; + + for (int i = 0; i < maxAttempts; i++) + { + await Task.Delay(1000); + + var statusResponse = await httpClient.GetAsync($"/api/satellite/region/{regionId}"); + + if (!statusResponse.IsSuccessStatusCode) + { + var errorContent = await statusResponse.Content.ReadAsStringAsync(); + throw new Exception($"Status check failed with status {statusResponse.StatusCode}: {errorContent}"); + } + + var status = await statusResponse.Content.ReadFromJsonAsync(JsonOptions); + + if (status == null) + { + throw new Exception("No status returned"); + } + + Console.WriteLine($" Attempt {i + 1}: Status = {status.Status}"); + + if (status.Status == "completed" || status.Status == "failed") + { + finalStatus = status; + break; + } + } + + if (finalStatus == null) + { + throw new Exception("Region processing did not complete in time"); + } + + Console.WriteLine(); + Console.WriteLine("Region Processing Results:"); + Console.WriteLine($" Status: {finalStatus.Status}"); + Console.WriteLine($" Tiles Downloaded: {finalStatus.TilesDownloaded}"); + Console.WriteLine($" Tiles Reused: {finalStatus.TilesReused}"); + Console.WriteLine($" CSV File: {finalStatus.CsvFilePath}"); + Console.WriteLine($" Summary File: {finalStatus.SummaryFilePath}"); + Console.WriteLine($" Created At: {finalStatus.CreatedAt:yyyy-MM-dd HH:mm:ss}"); + Console.WriteLine($" Updated At: {finalStatus.UpdatedAt:yyyy-MM-dd HH:mm:ss}"); + + if (finalStatus.Status != "completed") + { + throw new Exception($"Expected status 'completed', got '{finalStatus.Status}'"); + } + + if (string.IsNullOrEmpty(finalStatus.CsvFilePath)) + { + throw new Exception("CSV file path is empty"); + } + + if (string.IsNullOrEmpty(finalStatus.SummaryFilePath)) + { + throw new Exception("Summary file path is empty"); + } + + Console.WriteLine(); + Console.WriteLine("✓ Region processed successfully"); + Console.WriteLine("✓ CSV and summary files created"); + Console.WriteLine(); + Console.WriteLine("Region Processing Test: PASSED"); + } } public record DownloadTileRequest @@ -187,3 +305,24 @@ public record DownloadTileResponse public DateTime CreatedAt { get; set; } public DateTime UpdatedAt { get; set; } } + +public record RequestRegionRequest +{ + public Guid Id { get; set; } + public double Latitude { get; set; } + public double Longitude { get; set; } + public double SizeMeters { get; set; } + public int ZoomLevel { get; set; } +} + +public record RegionStatusResponse +{ + public Guid Id { get; set; } + public string Status { get; set; } = string.Empty; + public string? CsvFilePath { get; set; } + public string? SummaryFilePath { get; set; } + public int TilesDownloaded { get; set; } + public int TilesReused { get; set; } + public DateTime CreatedAt { get; set; } + public DateTime UpdatedAt { get; set; } +} diff --git a/SatelliteProvider.Services/RegionProcessingService.cs b/SatelliteProvider.Services/RegionProcessingService.cs new file mode 100644 index 0000000..c2f37b9 --- /dev/null +++ b/SatelliteProvider.Services/RegionProcessingService.cs @@ -0,0 +1,53 @@ +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using SatelliteProvider.Common.Interfaces; + +namespace SatelliteProvider.Services; + +public class RegionProcessingService : BackgroundService +{ + private readonly IRegionRequestQueue _queue; + private readonly IRegionService _regionService; + private readonly ILogger _logger; + + public RegionProcessingService( + IRegionRequestQueue queue, + IRegionService regionService, + ILogger logger) + { + _queue = queue; + _regionService = regionService; + _logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _logger.LogInformation("Region Processing Service started"); + + while (!stoppingToken.IsCancellationRequested) + { + try + { + var request = await _queue.DequeueAsync(stoppingToken); + + if (request != null) + { + _logger.LogInformation("Dequeued region request {RegionId}", request.Id); + + await _regionService.ProcessRegionAsync(request.Id, stoppingToken); + } + } + catch (OperationCanceledException) + { + break; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error processing region request"); + } + } + + _logger.LogInformation("Region Processing Service stopped"); + } +} + diff --git a/SatelliteProvider.Services/RegionRequestQueue.cs b/SatelliteProvider.Services/RegionRequestQueue.cs new file mode 100644 index 0000000..827d426 --- /dev/null +++ b/SatelliteProvider.Services/RegionRequestQueue.cs @@ -0,0 +1,39 @@ +using System.Threading.Channels; +using SatelliteProvider.Common.DTO; +using SatelliteProvider.Common.Interfaces; + +namespace SatelliteProvider.Services; + +public class RegionRequestQueue : IRegionRequestQueue +{ + private readonly Channel _queue; + + public RegionRequestQueue(int capacity) + { + var options = new BoundedChannelOptions(capacity) + { + FullMode = BoundedChannelFullMode.Wait + }; + _queue = Channel.CreateBounded(options); + } + + public async ValueTask EnqueueAsync(RegionRequest request, CancellationToken cancellationToken = default) + { + await _queue.Writer.WriteAsync(request, cancellationToken); + } + + public async ValueTask DequeueAsync(CancellationToken cancellationToken = default) + { + if (await _queue.Reader.WaitToReadAsync(cancellationToken)) + { + if (_queue.Reader.TryRead(out var request)) + { + return request; + } + } + return null; + } + + public int Count => _queue.Reader.Count; +} + diff --git a/SatelliteProvider.Services/RegionService.cs b/SatelliteProvider.Services/RegionService.cs new file mode 100644 index 0000000..bf13c2a --- /dev/null +++ b/SatelliteProvider.Services/RegionService.cs @@ -0,0 +1,213 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using SatelliteProvider.Common.Configs; +using SatelliteProvider.Common.DTO; +using SatelliteProvider.Common.Interfaces; +using SatelliteProvider.DataAccess.Models; +using SatelliteProvider.DataAccess.Repositories; + +namespace SatelliteProvider.Services; + +public class RegionService : IRegionService +{ + private readonly IRegionRepository _regionRepository; + private readonly IRegionRequestQueue _queue; + private readonly ITileService _tileService; + private readonly StorageConfig _storageConfig; + private readonly ILogger _logger; + + public RegionService( + IRegionRepository regionRepository, + IRegionRequestQueue queue, + ITileService tileService, + IOptions storageConfig, + ILogger logger) + { + _regionRepository = regionRepository; + _queue = queue; + _tileService = tileService; + _storageConfig = storageConfig.Value; + _logger = logger; + } + + public async Task RequestRegionAsync(Guid id, double latitude, double longitude, double sizeMeters, int zoomLevel) + { + var now = DateTime.UtcNow; + var region = new RegionEntity + { + Id = id, + Latitude = latitude, + Longitude = longitude, + SizeMeters = sizeMeters, + ZoomLevel = zoomLevel, + Status = "queued", + TilesDownloaded = 0, + TilesReused = 0, + CreatedAt = now, + UpdatedAt = now + }; + + await _regionRepository.InsertAsync(region); + + var request = new RegionRequest + { + Id = id, + Latitude = latitude, + Longitude = longitude, + SizeMeters = sizeMeters, + ZoomLevel = zoomLevel + }; + + await _queue.EnqueueAsync(request); + + _logger.LogInformation("Region {RegionId} queued for processing", id); + + return MapToStatus(region); + } + + public async Task GetRegionStatusAsync(Guid id) + { + var region = await _regionRepository.GetByIdAsync(id); + return region != null ? MapToStatus(region) : null; + } + + public async Task ProcessRegionAsync(Guid id, CancellationToken cancellationToken = default) + { + _logger.LogInformation("Processing region {RegionId}", id); + var startTime = DateTime.UtcNow; + + var region = await _regionRepository.GetByIdAsync(id); + if (region == null) + { + _logger.LogWarning("Region {RegionId} not found", id); + return; + } + + region.Status = "processing"; + region.UpdatedAt = DateTime.UtcNow; + await _regionRepository.UpdateAsync(region); + + try + { + _logger.LogInformation("Downloading tiles for region {RegionId} at ({Lat}, {Lon}) size {Size}m zoom {Zoom}", + id, region.Latitude, region.Longitude, region.SizeMeters, region.ZoomLevel); + + var processingStartTime = DateTime.UtcNow; + + _logger.LogInformation("Checking for existing tiles in region {RegionId}", id); + var tilesBeforeDownload = await _tileService.GetTilesByRegionAsync( + region.Latitude, + region.Longitude, + region.SizeMeters, + region.ZoomLevel); + var existingTileIds = new HashSet(tilesBeforeDownload.Select(t => t.Id)); + _logger.LogInformation("Found {Count} existing tiles for region {RegionId}", existingTileIds.Count, id); + + _logger.LogInformation("Starting tile download for region {RegionId}", id); + var tiles = await _tileService.DownloadAndStoreTilesAsync( + region.Latitude, + region.Longitude, + region.SizeMeters, + region.ZoomLevel, + cancellationToken); + + var tilesDownloaded = tiles.Count(t => !existingTileIds.Contains(t.Id)); + var tilesReused = tiles.Count(t => existingTileIds.Contains(t.Id)); + + _logger.LogInformation("Region {RegionId}: Downloaded {Downloaded} tiles, Reused {Reused} tiles", + id, tilesDownloaded, tilesReused); + + var readyDir = _storageConfig.ReadyDirectory; + Directory.CreateDirectory(readyDir); + + var csvPath = Path.Combine(readyDir, $"region_{id}_ready.csv"); + var summaryPath = Path.Combine(readyDir, $"region_{id}_summary.txt"); + + await GenerateCsvFileAsync(csvPath, tiles, cancellationToken); + await GenerateSummaryFileAsync(summaryPath, id, region, tiles, tilesDownloaded, tilesReused, processingStartTime, cancellationToken); + + region.Status = "completed"; + region.CsvFilePath = csvPath; + region.SummaryFilePath = summaryPath; + region.TilesDownloaded = tilesDownloaded; + region.TilesReused = tilesReused; + region.UpdatedAt = DateTime.UtcNow; + await _regionRepository.UpdateAsync(region); + + var duration = (DateTime.UtcNow - startTime).TotalSeconds; + _logger.LogInformation("Region {RegionId} processing completed in {Duration:F2}s", id, duration); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to process region {RegionId}: {Message}", id, ex.Message); + region.Status = "failed"; + region.UpdatedAt = DateTime.UtcNow; + await _regionRepository.UpdateAsync(region); + } + } + + private async Task GenerateCsvFileAsync(string filePath, List tiles, CancellationToken cancellationToken) + { + var orderedTiles = tiles.OrderByDescending(t => t.Latitude).ThenBy(t => t.Longitude).ToList(); + + using var writer = new StreamWriter(filePath); + await writer.WriteLineAsync("latitude,longitude,file_path"); + + foreach (var tile in orderedTiles) + { + await writer.WriteLineAsync($"{tile.Latitude:F6},{tile.Longitude:F6},{tile.FilePath}"); + } + } + + private async Task GenerateSummaryFileAsync( + string filePath, + Guid regionId, + RegionEntity region, + List tiles, + int tilesDownloaded, + int tilesReused, + DateTime startTime, + CancellationToken cancellationToken) + { + var endTime = DateTime.UtcNow; + var processingTime = (endTime - startTime).TotalSeconds; + + var summary = new System.Text.StringBuilder(); + summary.AppendLine("Region Processing Summary"); + summary.AppendLine("========================"); + summary.AppendLine($"Region ID: {regionId}"); + summary.AppendLine($"Center: {region.Latitude:F6}, {region.Longitude:F6}"); + summary.AppendLine($"Size: {region.SizeMeters:F0} meters"); + summary.AppendLine($"Zoom Level: {region.ZoomLevel}"); + summary.AppendLine(); + summary.AppendLine("Processing Statistics:"); + summary.AppendLine($"- Tiles Downloaded: {tilesDownloaded}"); + summary.AppendLine($"- Tiles Reused from Cache: {tilesReused}"); + summary.AppendLine($"- Total Tiles: {tiles.Count}"); + summary.AppendLine($"- Processing Time: {processingTime:F2} seconds"); + summary.AppendLine($"- Started: {startTime:yyyy-MM-dd HH:mm:ss} UTC"); + summary.AppendLine($"- Completed: {endTime:yyyy-MM-dd HH:mm:ss} UTC"); + summary.AppendLine(); + summary.AppendLine("Files Created:"); + summary.AppendLine($"- CSV: {Path.GetFileName(filePath).Replace("_summary.txt", "_ready.csv")}"); + summary.AppendLine($"- Summary: {Path.GetFileName(filePath)}"); + + await File.WriteAllTextAsync(filePath, summary.ToString(), cancellationToken); + } + + private static RegionStatus MapToStatus(RegionEntity region) + { + return new RegionStatus + { + Id = region.Id, + Status = region.Status, + CsvFilePath = region.CsvFilePath, + SummaryFilePath = region.SummaryFilePath, + TilesDownloaded = region.TilesDownloaded, + TilesReused = region.TilesReused, + CreatedAt = region.CreatedAt, + UpdatedAt = region.UpdatedAt + }; + } +} + diff --git a/SatelliteProvider.Services/SatelliteProvider.Services.csproj b/SatelliteProvider.Services/SatelliteProvider.Services.csproj index b7512db..82dda4b 100644 --- a/SatelliteProvider.Services/SatelliteProvider.Services.csproj +++ b/SatelliteProvider.Services/SatelliteProvider.Services.csproj @@ -7,6 +7,7 @@ +