From f13f3eea6b3c831d5ae091ae6c793cbb7650d24c Mon Sep 17 00:00:00 2001 From: Anton Martynenko Date: Wed, 29 Oct 2025 15:07:20 +0100 Subject: [PATCH] improve retries --- SatelliteProvider.Api/Program.cs | 1 - .../RegionTests.cs | 8 +- .../GoogleMapsDownloader.cs | 140 ------------------ .../GoogleMapsDownloaderV2.cs | 83 +++++++++-- SatelliteProvider.Services/RegionService.cs | 125 ++++++++++++++-- 5 files changed, 186 insertions(+), 171 deletions(-) delete mode 100644 SatelliteProvider.Services/GoogleMapsDownloader.cs diff --git a/SatelliteProvider.Api/Program.cs b/SatelliteProvider.Api/Program.cs index fc9d0c4..3765936 100644 --- a/SatelliteProvider.Api/Program.cs +++ b/SatelliteProvider.Api/Program.cs @@ -27,7 +27,6 @@ builder.Services.AddSingleton(sp => new TileRepository(connecti builder.Services.AddSingleton(sp => new RegionRepository(connectionString)); builder.Services.AddHttpClient(); -builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); diff --git a/SatelliteProvider.IntegrationTests/RegionTests.cs b/SatelliteProvider.IntegrationTests/RegionTests.cs index 165371f..e78ca2a 100644 --- a/SatelliteProvider.IntegrationTests/RegionTests.cs +++ b/SatelliteProvider.IntegrationTests/RegionTests.cs @@ -106,11 +106,11 @@ public static class RegionTests Console.WriteLine("Polling for region status updates..."); RegionStatusResponse? finalStatus = null; - int maxAttempts = 30; + int maxAttempts = 120; for (int i = 0; i < maxAttempts; i++) { - await Task.Delay(1000); + await Task.Delay(2000); var statusResponse = await httpClient.GetAsync($"/api/satellite/region/{regionId}"); @@ -127,7 +127,7 @@ public static class RegionTests throw new Exception("No status returned"); } - Console.WriteLine($" Attempt {i + 1}: Status = {status.Status}"); + Console.WriteLine($" Attempt {i + 1}/{maxAttempts}: Status = {status.Status}"); if (status.Status == "completed" || status.Status == "failed") { @@ -138,7 +138,7 @@ public static class RegionTests if (finalStatus == null) { - throw new Exception("Region processing did not complete in time"); + throw new Exception($"Region processing did not complete in time (waited {maxAttempts * 2} seconds)"); } Console.WriteLine(); diff --git a/SatelliteProvider.Services/GoogleMapsDownloader.cs b/SatelliteProvider.Services/GoogleMapsDownloader.cs deleted file mode 100644 index e496573..0000000 --- a/SatelliteProvider.Services/GoogleMapsDownloader.cs +++ /dev/null @@ -1,140 +0,0 @@ -using System.Collections.Concurrent; -using System.Net.Http.Json; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Newtonsoft.Json; -using SatelliteProvider.Common.Configs; -using SatelliteProvider.Common.DTO; -using SatelliteProvider.Common.Interfaces; -using SatelliteProvider.Common.Utils; -using SixLabors.ImageSharp; - -namespace SatelliteProvider.Services; - -public record DownloadedTileInfo(int X, int Y, int ZoomLevel, double Latitude, double Longitude, string FilePath, double TileSizeMeters); - -public class GoogleMapsDownloader(ILogger logger, IOptions mapConfig, IOptions storageConfig, IHttpClientFactory httpClientFactory) - : ISatelliteDownloader -{ - private const string TILE_URL_TEMPLATE = "https://mt{0}.google.com/vt/lyrs=s&x={1}&y={2}&z={3}&token={4}"; - private const string USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.0.0 Safari/537.36"; - private const int NUM_SERVERS = 4; - private const int TILE_SIZE_PIXELS = 256; - private readonly string _apiKey = mapConfig.Value.ApiKey; - private readonly string _tilesDirectory = storageConfig.Value.TilesDirectory; - - private record SessionResponse(string Session); - - private async Task GetSessionToken() - { - var url = $"https://tile.googleapis.com/v1/createSession?key={_apiKey}"; - using var httpClient = httpClientFactory.CreateClient(); - try - { - var str = JsonConvert.SerializeObject(new { mapType = "satellite" }); - var response = await httpClient.PostAsync(url, new StringContent(str)); - response.EnsureSuccessStatusCode(); - var sessionResponse = await response.Content.ReadFromJsonAsync(); - return sessionResponse?.Session; - } - catch (Exception e) - { - logger.LogError(e, "Failed to get session token"); - throw; - } - } - - public async Task GetTiles(GeoPoint centerGeoPoint, double radiusM, int zoomLevel, CancellationToken token = default) - { - await GetTilesWithMetadataAsync(centerGeoPoint, radiusM, zoomLevel, token); - } - - public async Task> GetTilesWithMetadataAsync(GeoPoint centerGeoPoint, double radiusM, int zoomLevel, CancellationToken token = default) - { - var (latMin, latMax, lonMin, lonMax) = GeoUtils.GetBoundingBox(centerGeoPoint, radiusM); - - var (xMin, yMin) = GeoUtils.WorldToTilePos(new GeoPoint(latMax, lonMin), zoomLevel); - var (xMax, yMax) = GeoUtils.WorldToTilePos(new GeoPoint(latMin, lonMax), zoomLevel); - - var tilesToDownload = new ConcurrentQueue(); - var downloadedTiles = new ConcurrentBag(); - var server = 0; - var sessionToken = await GetSessionToken(); - - for (var y = yMin; y <= yMax + 1; y++) - for (var x = xMin; x <= xMax + 1; x++) - { - token.ThrowIfCancellationRequested(); - var url = string.Format(TILE_URL_TEMPLATE, server, x, y, zoomLevel, sessionToken); - - tilesToDownload.Enqueue(new SatTile(x, y, zoomLevel, url)); - server = (server + 1) % NUM_SERVERS; - } - - var downloadTasks = new List(); - - for (int i = 0; i < NUM_SERVERS; i++) - { - downloadTasks.Add(Task.Run(() => DownloadTilesWorker(tilesToDownload, downloadedTiles, zoomLevel, token), token)); - } - - await Task.WhenAll(downloadTasks); - return downloadedTiles.ToList(); - } - - private async Task DownloadTilesWorker(ConcurrentQueue tilesToDownload, ConcurrentBag downloadedTiles, int zoomLevel, CancellationToken token) - { - using var httpClient = httpClientFactory.CreateClient(); - - while (tilesToDownload.TryDequeue(out var tileInfo)) - { - if (token.IsCancellationRequested) break; - try - { - Directory.CreateDirectory(_tilesDirectory); - - var timestamp = DateTime.UtcNow.ToString("yyyyMMddHHmmss"); - var fileName = $"tile_{tileInfo.Zoom}_{tileInfo.X}_{tileInfo.Y}_{timestamp}.jpg"; - var filePath = Path.Combine(_tilesDirectory, fileName); - - httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(USER_AGENT); - var response = await httpClient.GetAsync(tileInfo.Url, token); - response.EnsureSuccessStatusCode(); - var tileData = await response.Content.ReadAsByteArrayAsync(token); - using var tileImage = Image.Load(tileData); - await tileImage.SaveAsync(filePath, token); - - var tileCenter = GeoUtils.TileToWorldPos(tileInfo.X, tileInfo.Y, tileInfo.Zoom); - var tileSizeMeters = CalculateTileSizeInMeters(tileInfo.Zoom, tileCenter.Lat); - - var downloadedTile = new DownloadedTileInfo( - tileInfo.X, - tileInfo.Y, - tileInfo.Zoom, - tileCenter.Lat, - tileCenter.Lon, - filePath, - tileSizeMeters - ); - - downloadedTiles.Add(downloadedTile); - } - catch (HttpRequestException requestException) - { - logger.LogError(requestException, "Failed to download tile. Url: {Url}", tileInfo.Url); - } - catch (Exception e) - { - logger.LogError(e, "Failed to download tile"); - } - } - } - - private static double CalculateTileSizeInMeters(int zoomLevel, double latitude) - { - const double EARTH_CIRCUMFERENCE_METERS = 40075016.686; - var latRad = latitude * Math.PI / 180.0; - var metersPerPixel = (EARTH_CIRCUMFERENCE_METERS * Math.Cos(latRad)) / (Math.Pow(2, zoomLevel) * TILE_SIZE_PIXELS); - return metersPerPixel * TILE_SIZE_PIXELS; - } -} \ No newline at end of file diff --git a/SatelliteProvider.Services/GoogleMapsDownloaderV2.cs b/SatelliteProvider.Services/GoogleMapsDownloaderV2.cs index e8f0d0a..e1fd5c3 100644 --- a/SatelliteProvider.Services/GoogleMapsDownloaderV2.cs +++ b/SatelliteProvider.Services/GoogleMapsDownloaderV2.cs @@ -1,3 +1,4 @@ +using System.Net; using System.Net.Http.Json; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -10,11 +11,18 @@ namespace SatelliteProvider.Services; public record DownloadedTileInfoV2(int X, int Y, int ZoomLevel, double CenterLatitude, double CenterLongitude, string FilePath, double TileSizeMeters); +public class RateLimitException : Exception +{ + public RateLimitException(string message) : base(message) { } +} + public class GoogleMapsDownloaderV2 { private const string TILE_URL_TEMPLATE = "https://mt{0}.google.com/vt/lyrs=s&x={1}&y={2}&z={3}&token={4}"; private const string USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.0.0 Safari/537.36"; private const int TILE_SIZE_PIXELS = 256; + private const int MAX_RETRY_DELAY_SECONDS = 30; + private const int BASE_RETRY_DELAY_SECONDS = 1; private static readonly int[] ALLOWED_ZOOM_LEVELS = { 15, 16, 17, 18, 19 }; private readonly ILogger _logger; @@ -78,13 +86,17 @@ public class GoogleMapsDownloaderV2 var fileName = $"tile_{zoomLevel}_{tileX}_{tileY}_{timestamp}.jpg"; var filePath = Path.Combine(_tilesDirectory, fileName); - using var httpClient = _httpClientFactory.CreateClient(); - httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(USER_AGENT); + var imageBytes = await ExecuteWithRetryAsync(async () => + { + using var httpClient = _httpClientFactory.CreateClient(); + httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(USER_AGENT); - var response = await httpClient.GetAsync(url, token); - response.EnsureSuccessStatusCode(); + var response = await httpClient.GetAsync(url, token); + response.EnsureSuccessStatusCode(); + + return await response.Content.ReadAsByteArrayAsync(token); + }, cancellationToken: token); - var imageBytes = await response.Content.ReadAsByteArrayAsync(token); await File.WriteAllBytesAsync(filePath, imageBytes, token); var tileCenter = GeoUtils.TileToWorldPos(tileX, tileY, zoomLevel); @@ -111,6 +123,48 @@ public class GoogleMapsDownloaderV2 return metersPerPixel * TILE_SIZE_PIXELS; } + private async Task ExecuteWithRetryAsync(Func> action, int maxRetries = 5, CancellationToken cancellationToken = default) + { + int attempt = 0; + int delay = BASE_RETRY_DELAY_SECONDS; + + while (attempt < maxRetries) + { + try + { + return await action(); + } + catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.TooManyRequests || ex.StatusCode == (HttpStatusCode)429) + { + attempt++; + if (attempt >= maxRetries) + { + _logger.LogError("Rate limit exceeded after {Attempts} attempts", maxRetries); + throw new RateLimitException($"Rate limit exceeded after {maxRetries} attempts"); + } + + delay = Math.Min(delay * 2, MAX_RETRY_DELAY_SECONDS); + _logger.LogWarning("Rate limited. Waiting {Delay}s before retry {Attempt}/{Max}", delay, attempt, maxRetries); + await Task.Delay(TimeSpan.FromSeconds(delay), cancellationToken); + } + catch (HttpRequestException ex) when (ex.StatusCode >= HttpStatusCode.InternalServerError) + { + attempt++; + if (attempt >= maxRetries) + { + _logger.LogError(ex, "Server error after {Attempts} attempts", maxRetries); + throw; + } + + delay = Math.Min(delay * 2, MAX_RETRY_DELAY_SECONDS); + _logger.LogWarning("Server error. Waiting {Delay}s before retry {Attempt}/{Max}", delay, attempt, maxRetries); + await Task.Delay(TimeSpan.FromSeconds(delay), cancellationToken); + } + } + + throw new InvalidOperationException("Retry logic failed unexpectedly"); + } + public async Task> GetTilesWithMetadataAsync( GeoPoint centerGeoPoint, double radiusM, @@ -169,13 +223,17 @@ public class GoogleMapsDownloaderV2 var fileName = $"tile_{zoomLevel}_{x}_{y}_{timestamp}.jpg"; var filePath = Path.Combine(_tilesDirectory, fileName); - using var httpClient = _httpClientFactory.CreateClient(); - httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(USER_AGENT); + var imageBytes = await ExecuteWithRetryAsync(async () => + { + using var httpClient = _httpClientFactory.CreateClient(); + httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(USER_AGENT); - var response = await httpClient.GetAsync(url, token); - response.EnsureSuccessStatusCode(); + var response = await httpClient.GetAsync(url, token); + response.EnsureSuccessStatusCode(); + + return await response.Content.ReadAsByteArrayAsync(token); + }, cancellationToken: token); - var imageBytes = await response.Content.ReadAsByteArrayAsync(token); await File.WriteAllBytesAsync(filePath, imageBytes, token); _logger.LogInformation("Downloaded tile ({X}, {Y}) to {FilePath}, center=({Lat:F6}, {Lon:F6}), size={Size:F2}m", @@ -184,6 +242,11 @@ public class GoogleMapsDownloaderV2 downloadedTiles.Add(new DownloadedTileInfoV2( x, y, zoomLevel, tileCenter.Lat, tileCenter.Lon, filePath, tileSizeMeters)); } + catch (RateLimitException ex) + { + _logger.LogError(ex, "Rate limit exceeded for tile ({X}, {Y})", x, y); + throw; + } catch (Exception ex) { _logger.LogError(ex, "Failed to download tile ({X}, {Y})", x, y); diff --git a/SatelliteProvider.Services/RegionService.cs b/SatelliteProvider.Services/RegionService.cs index d15ed33..7b4401f 100644 --- a/SatelliteProvider.Services/RegionService.cs +++ b/SatelliteProvider.Services/RegionService.cs @@ -91,6 +91,14 @@ public class RegionService : IRegionService region.UpdatedAt = DateTime.UtcNow; await _regionRepository.UpdateAsync(region); + using var timeoutCts = new CancellationTokenSource(TimeSpan.FromMinutes(5)); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCts.Token); + + string? errorMessage = null; + List? tiles = null; + int tilesDownloaded = 0; + int tilesReused = 0; + try { _logger.LogInformation("Downloading tiles for region {RegionId} at ({Lat}, {Lon}) size {Size}m zoom {Zoom}", @@ -108,15 +116,15 @@ public class RegionService : IRegionService _logger.LogInformation("Found {Count} existing tiles for region {RegionId}", existingTileIds.Count, id); _logger.LogInformation("Starting tile download for region {RegionId}", id); - var tiles = await _tileService.DownloadAndStoreTilesAsync( + tiles = await _tileService.DownloadAndStoreTilesAsync( region.Latitude, region.Longitude, region.SizeMeters, region.ZoomLevel, - cancellationToken); + linkedCts.Token); - var tilesDownloaded = tiles.Count(t => !existingTileIds.Contains(t.Id)); - var tilesReused = tiles.Count(t => existingTileIds.Contains(t.Id)); + tilesDownloaded = tiles.Count(t => !existingTileIds.Contains(t.Id)); + tilesReused = tiles.Count(t => existingTileIds.Contains(t.Id)); _logger.LogInformation("Region {RegionId}: Downloaded {Downloaded} tiles, Reused {Reused} tiles", id, tilesDownloaded, tilesReused); @@ -128,12 +136,12 @@ public class RegionService : IRegionService var summaryPath = Path.Combine(readyDir, $"region_{id}_summary.txt"); var stitchedImagePath = Path.Combine(readyDir, $"region_{id}_stitched.jpg"); - await GenerateCsvFileAsync(csvPath, tiles, cancellationToken); + await GenerateCsvFileAsync(csvPath, tiles, linkedCts.Token); _logger.LogInformation("Stitching tiles for region {RegionId}", id); - await StitchTilesAsync(tiles, region.Latitude, region.Longitude, region.ZoomLevel, stitchedImagePath, cancellationToken); + await StitchTilesAsync(tiles, region.Latitude, region.Longitude, region.ZoomLevel, stitchedImagePath, linkedCts.Token); - await GenerateSummaryFileAsync(summaryPath, id, region, tiles, tilesDownloaded, tilesReused, stitchedImagePath, processingStartTime, cancellationToken); + await GenerateSummaryFileAsync(summaryPath, id, region, tiles, tilesDownloaded, tilesReused, stitchedImagePath, processingStartTime, linkedCts.Token, errorMessage); region.Status = "completed"; region.CsvFilePath = csvPath; @@ -146,15 +154,72 @@ public class RegionService : IRegionService var duration = (DateTime.UtcNow - startTime).TotalSeconds; _logger.LogInformation("Region {RegionId} processing completed in {Duration:F2}s", id, duration); } + catch (OperationCanceledException) when (timeoutCts.IsCancellationRequested) + { + errorMessage = "Processing timed out after 5 minutes. Unable to download tiles within the time limit."; + _logger.LogError("Region {RegionId} processing timed out after 5 minutes", id); + await HandleProcessingFailureAsync(id, region, startTime, tiles, tilesDownloaded, tilesReused, errorMessage); + } + catch (RateLimitException ex) + { + errorMessage = $"Rate limit exceeded: {ex.Message}. Google Maps API rate limit was reached and retries were exhausted."; + _logger.LogError(ex, "Rate limit exceeded for region {RegionId}", id); + await HandleProcessingFailureAsync(id, region, startTime, tiles, tilesDownloaded, tilesReused, errorMessage); + } + catch (HttpRequestException ex) + { + errorMessage = $"Network error: {ex.Message}. Failed to download tiles from Google Maps."; + _logger.LogError(ex, "Network error processing region {RegionId}", id); + await HandleProcessingFailureAsync(id, region, startTime, tiles, tilesDownloaded, tilesReused, errorMessage); + } catch (Exception ex) { + errorMessage = $"Unexpected error: {ex.Message}"; _logger.LogError(ex, "Failed to process region {RegionId}: {Message}", id, ex.Message); - region.Status = "failed"; - region.UpdatedAt = DateTime.UtcNow; - await _regionRepository.UpdateAsync(region); + await HandleProcessingFailureAsync(id, region, startTime, tiles, tilesDownloaded, tilesReused, errorMessage); } } + private async Task HandleProcessingFailureAsync( + Guid id, + RegionEntity region, + DateTime startTime, + List? tiles, + int tilesDownloaded, + int tilesReused, + string errorMessage) + { + region.Status = "failed"; + region.UpdatedAt = DateTime.UtcNow; + + try + { + var readyDir = _storageConfig.ReadyDirectory; + Directory.CreateDirectory(readyDir); + + var summaryPath = Path.Combine(readyDir, $"region_{id}_summary.txt"); + region.SummaryFilePath = summaryPath; + + await GenerateSummaryFileAsync( + summaryPath, + id, + region, + tiles ?? new List(), + tilesDownloaded, + tilesReused, + null, + startTime, + CancellationToken.None, + errorMessage); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to generate error summary for region {RegionId}", id); + } + + await _regionRepository.UpdateAsync(region); + } + private async Task StitchTilesAsync( List tiles, double centerLatitude, @@ -265,9 +330,10 @@ public class RegionService : IRegionService List tiles, int tilesDownloaded, int tilesReused, - string stitchedImagePath, + string? stitchedImagePath, DateTime startTime, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + string? errorMessage = null) { var endTime = DateTime.UtcNow; var processingTime = (endTime - startTime).TotalSeconds; @@ -279,19 +345,46 @@ public class RegionService : IRegionService summary.AppendLine($"Center: {region.Latitude:F6}, {region.Longitude:F6}"); summary.AppendLine($"Size: {region.SizeMeters:F0} meters"); summary.AppendLine($"Zoom Level: {region.ZoomLevel}"); + summary.AppendLine($"Status: {region.Status}"); summary.AppendLine(); + + if (!string.IsNullOrEmpty(errorMessage)) + { + summary.AppendLine("ERROR:"); + summary.AppendLine(errorMessage); + summary.AppendLine(); + } + summary.AppendLine("Processing Statistics:"); summary.AppendLine($"- Tiles Downloaded: {tilesDownloaded}"); summary.AppendLine($"- Tiles Reused from Cache: {tilesReused}"); summary.AppendLine($"- Total Tiles: {tiles.Count}"); summary.AppendLine($"- Processing Time: {processingTime:F2} seconds"); summary.AppendLine($"- Started: {startTime:yyyy-MM-dd HH:mm:ss} UTC"); - summary.AppendLine($"- Completed: {endTime:yyyy-MM-dd HH:mm:ss} UTC"); + + if (region.Status == "completed") + { + summary.AppendLine($"- Completed: {endTime:yyyy-MM-dd HH:mm:ss} UTC"); + } + else + { + summary.AppendLine($"- Failed: {endTime:yyyy-MM-dd HH:mm:ss} UTC"); + } + summary.AppendLine(); summary.AppendLine("Files Created:"); - summary.AppendLine($"- CSV: {Path.GetFileName(filePath).Replace("_summary.txt", "_ready.csv")}"); - summary.AppendLine($"- Stitched Image: {Path.GetFileName(stitchedImagePath)}"); - summary.AppendLine($"- Stitched Image Path: {stitchedImagePath}"); + + if (tiles.Count > 0) + { + summary.AppendLine($"- CSV: {Path.GetFileName(filePath).Replace("_summary.txt", "_ready.csv")}"); + } + + if (!string.IsNullOrEmpty(stitchedImagePath)) + { + summary.AppendLine($"- Stitched Image: {Path.GetFileName(stitchedImagePath)}"); + summary.AppendLine($"- Stitched Image Path: {stitchedImagePath}"); + } + summary.AppendLine($"- Summary: {Path.GetFileName(filePath)}"); await File.WriteAllTextAsync(filePath, summary.ToString(), cancellationToken);