improve retries

SatelliteProvider.Api/Program.cs

@@ -27,7 +27,6 @@ builder.Services.AddSingleton<ITileRepository>(sp => new TileRepository(connecti
 builder.Services.AddSingleton<IRegionRepository>(sp => new RegionRepository(connectionString));
 
 builder.Services.AddHttpClient();
-builder.Services.AddSingleton<GoogleMapsDownloader>();
 builder.Services.AddSingleton<GoogleMapsDownloaderV2>();
 builder.Services.AddSingleton<ITileService, TileService>();
 

SatelliteProvider.IntegrationTests/RegionTests.cs

@@ -106,11 +106,11 @@ public static class RegionTests
 
         Console.WriteLine("Polling for region status updates...");
         RegionStatusResponse? finalStatus = null;
-        int maxAttempts = 30;
+        int maxAttempts = 120;
         
         for (int i = 0; i < maxAttempts; i++)
         {
-            await Task.Delay(1000);
+            await Task.Delay(2000);
             
             var statusResponse = await httpClient.GetAsync($"/api/satellite/region/{regionId}");
             
@@ -127,7 +127,7 @@ public static class RegionTests
                 throw new Exception("No status returned");
             }
 
-            Console.WriteLine($"  Attempt {i + 1}: Status = {status.Status}");
+            Console.WriteLine($"  Attempt {i + 1}/{maxAttempts}: Status = {status.Status}");
 
             if (status.Status == "completed" || status.Status == "failed")
             {
@@ -138,7 +138,7 @@ public static class RegionTests
 
         if (finalStatus == null)
         {
-            throw new Exception("Region processing did not complete in time");
+            throw new Exception($"Region processing did not complete in time (waited {maxAttempts * 2} seconds)");
         }
 
         Console.WriteLine();

SatelliteProvider.Services/GoogleMapsDownloader.cs

@@ -1,140 +0,0 @@
-using System.Collections.Concurrent;
-using System.Net.Http.Json;
-using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Options;
-using Newtonsoft.Json;
-using SatelliteProvider.Common.Configs;
-using SatelliteProvider.Common.DTO;
-using SatelliteProvider.Common.Interfaces;
-using SatelliteProvider.Common.Utils;
-using SixLabors.ImageSharp;
-
-namespace SatelliteProvider.Services;
-
-public record DownloadedTileInfo(int X, int Y, int ZoomLevel, double Latitude, double Longitude, string FilePath, double TileSizeMeters);
-
-public class GoogleMapsDownloader(ILogger<GoogleMapsDownloader> logger, IOptions<MapConfig> mapConfig, IOptions<StorageConfig> storageConfig, IHttpClientFactory httpClientFactory)
-    : ISatelliteDownloader
-{
-    private const string TILE_URL_TEMPLATE = "https://mt{0}.google.com/vt/lyrs=s&x={1}&y={2}&z={3}&token={4}";
-    private const string USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.0.0 Safari/537.36";
-    private const int NUM_SERVERS = 4;
-    private const int TILE_SIZE_PIXELS = 256;
-    private readonly string _apiKey = mapConfig.Value.ApiKey;
-    private readonly string _tilesDirectory = storageConfig.Value.TilesDirectory;
-
-    private record SessionResponse(string Session);
-
-    private async Task<string?> GetSessionToken()
-    {
-        var url = $"https://tile.googleapis.com/v1/createSession?key={_apiKey}";
-        using var httpClient = httpClientFactory.CreateClient();
-        try
-        {
-            var str = JsonConvert.SerializeObject(new { mapType = "satellite" });
-            var response = await httpClient.PostAsync(url, new StringContent(str));
-            response.EnsureSuccessStatusCode();
-            var sessionResponse = await response.Content.ReadFromJsonAsync<SessionResponse>();
-            return sessionResponse?.Session;
-        }
-        catch (Exception e)
-        {
-            logger.LogError(e, "Failed to get session token");
-            throw;
-        }
-    }
-
-    public async Task GetTiles(GeoPoint centerGeoPoint, double radiusM, int zoomLevel, CancellationToken token = default)
-    {
-        await GetTilesWithMetadataAsync(centerGeoPoint, radiusM, zoomLevel, token);
-    }
-
-    public async Task<List<DownloadedTileInfo>> GetTilesWithMetadataAsync(GeoPoint centerGeoPoint, double radiusM, int zoomLevel, CancellationToken token = default)
-    {
-        var (latMin, latMax, lonMin, lonMax) = GeoUtils.GetBoundingBox(centerGeoPoint, radiusM);
-
-        var (xMin, yMin) = GeoUtils.WorldToTilePos(new GeoPoint(latMax, lonMin), zoomLevel);
-        var (xMax, yMax) = GeoUtils.WorldToTilePos(new GeoPoint(latMin, lonMax), zoomLevel);
-
-        var tilesToDownload = new ConcurrentQueue<SatTile>();
-        var downloadedTiles = new ConcurrentBag<DownloadedTileInfo>();
-        var server = 0;
-        var sessionToken = await GetSessionToken();
-
-        for (var y = yMin; y <= yMax + 1; y++)
-            for (var x = xMin; x <= xMax + 1; x++)
-            {
-                token.ThrowIfCancellationRequested();
-                var url = string.Format(TILE_URL_TEMPLATE, server, x, y, zoomLevel, sessionToken);
-
-                tilesToDownload.Enqueue(new SatTile(x, y, zoomLevel, url));
-                server = (server + 1) % NUM_SERVERS;
-            }
-
-        var downloadTasks = new List<Task>();
-
-        for (int i = 0; i < NUM_SERVERS; i++)
-        {
-            downloadTasks.Add(Task.Run(() => DownloadTilesWorker(tilesToDownload, downloadedTiles, zoomLevel, token), token));
-        }
-
-        await Task.WhenAll(downloadTasks);
-        return downloadedTiles.ToList();
-    }
-
-    private async Task DownloadTilesWorker(ConcurrentQueue<SatTile> tilesToDownload, ConcurrentBag<DownloadedTileInfo> downloadedTiles, int zoomLevel, CancellationToken token)
-    {
-        using var httpClient = httpClientFactory.CreateClient();
-
-        while (tilesToDownload.TryDequeue(out var tileInfo))
-        {
-            if (token.IsCancellationRequested) break;
-            try
-            {
-                Directory.CreateDirectory(_tilesDirectory);
-                
-                var timestamp = DateTime.UtcNow.ToString("yyyyMMddHHmmss");
-                var fileName = $"tile_{tileInfo.Zoom}_{tileInfo.X}_{tileInfo.Y}_{timestamp}.jpg";
-                var filePath = Path.Combine(_tilesDirectory, fileName);
-                
-                httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(USER_AGENT);
-                var response = await httpClient.GetAsync(tileInfo.Url, token);
-                response.EnsureSuccessStatusCode();
-                var tileData = await response.Content.ReadAsByteArrayAsync(token);
-                using var tileImage = Image.Load(tileData);
-                await tileImage.SaveAsync(filePath, token);
-                
-                var tileCenter = GeoUtils.TileToWorldPos(tileInfo.X, tileInfo.Y, tileInfo.Zoom);
-                var tileSizeMeters = CalculateTileSizeInMeters(tileInfo.Zoom, tileCenter.Lat);
-                
-                var downloadedTile = new DownloadedTileInfo(
-                    tileInfo.X, 
-                    tileInfo.Y, 
-                    tileInfo.Zoom, 
-                    tileCenter.Lat, 
-                    tileCenter.Lon, 
-                    filePath, 
-                    tileSizeMeters
-                );
-                
-                downloadedTiles.Add(downloadedTile);
-            }
-            catch (HttpRequestException requestException)
-            {
-                logger.LogError(requestException, "Failed to download tile. Url: {Url}", tileInfo.Url);
-            }
-            catch (Exception e)
-            {
-                logger.LogError(e, "Failed to download tile");
-            }
-        }
-    }
-
-    private static double CalculateTileSizeInMeters(int zoomLevel, double latitude)
-    {
-        const double EARTH_CIRCUMFERENCE_METERS = 40075016.686;
-        var latRad = latitude * Math.PI / 180.0;
-        var metersPerPixel = (EARTH_CIRCUMFERENCE_METERS * Math.Cos(latRad)) / (Math.Pow(2, zoomLevel) * TILE_SIZE_PIXELS);
-        return metersPerPixel * TILE_SIZE_PIXELS;
-    }
-}

SatelliteProvider.Services/GoogleMapsDownloaderV2.cs

@@ -1,3 +1,4 @@
+using System.Net;
 using System.Net.Http.Json;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
@@ -10,11 +11,18 @@ namespace SatelliteProvider.Services;
 
 public record DownloadedTileInfoV2(int X, int Y, int ZoomLevel, double CenterLatitude, double CenterLongitude, string FilePath, double TileSizeMeters);
 
+public class RateLimitException : Exception
+{
+    public RateLimitException(string message) : base(message) { }
+}
+
 public class GoogleMapsDownloaderV2
 {
     private const string TILE_URL_TEMPLATE = "https://mt{0}.google.com/vt/lyrs=s&x={1}&y={2}&z={3}&token={4}";
     private const string USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.0.0 Safari/537.36";
     private const int TILE_SIZE_PIXELS = 256;
+    private const int MAX_RETRY_DELAY_SECONDS = 30;
+    private const int BASE_RETRY_DELAY_SECONDS = 1;
     private static readonly int[] ALLOWED_ZOOM_LEVELS = { 15, 16, 17, 18, 19 };
     
     private readonly ILogger<GoogleMapsDownloaderV2> _logger;
@@ -78,13 +86,17 @@ public class GoogleMapsDownloaderV2
         var fileName = $"tile_{zoomLevel}_{tileX}_{tileY}_{timestamp}.jpg";
         var filePath = Path.Combine(_tilesDirectory, fileName);
 
+        var imageBytes = await ExecuteWithRetryAsync(async () =>
+        {
             using var httpClient = _httpClientFactory.CreateClient();
             httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(USER_AGENT);
 
             var response = await httpClient.GetAsync(url, token);
             response.EnsureSuccessStatusCode();
 
-        var imageBytes = await response.Content.ReadAsByteArrayAsync(token);
+            return await response.Content.ReadAsByteArrayAsync(token);
+        }, cancellationToken: token);
+
         await File.WriteAllBytesAsync(filePath, imageBytes, token);
 
         var tileCenter = GeoUtils.TileToWorldPos(tileX, tileY, zoomLevel);
@@ -111,6 +123,48 @@ public class GoogleMapsDownloaderV2
         return metersPerPixel * TILE_SIZE_PIXELS;
     }
 
+    private async Task<T> ExecuteWithRetryAsync<T>(Func<Task<T>> action, int maxRetries = 5, CancellationToken cancellationToken = default)
+    {
+        int attempt = 0;
+        int delay = BASE_RETRY_DELAY_SECONDS;
+
+        while (attempt < maxRetries)
+        {
+            try
+            {
+                return await action();
+            }
+            catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.TooManyRequests || ex.StatusCode == (HttpStatusCode)429)
+            {
+                attempt++;
+                if (attempt >= maxRetries)
+                {
+                    _logger.LogError("Rate limit exceeded after {Attempts} attempts", maxRetries);
+                    throw new RateLimitException($"Rate limit exceeded after {maxRetries} attempts");
+                }
+
+                delay = Math.Min(delay * 2, MAX_RETRY_DELAY_SECONDS);
+                _logger.LogWarning("Rate limited. Waiting {Delay}s before retry {Attempt}/{Max}", delay, attempt, maxRetries);
+                await Task.Delay(TimeSpan.FromSeconds(delay), cancellationToken);
+            }
+            catch (HttpRequestException ex) when (ex.StatusCode >= HttpStatusCode.InternalServerError)
+            {
+                attempt++;
+                if (attempt >= maxRetries)
+                {
+                    _logger.LogError(ex, "Server error after {Attempts} attempts", maxRetries);
+                    throw;
+                }
+
+                delay = Math.Min(delay * 2, MAX_RETRY_DELAY_SECONDS);
+                _logger.LogWarning("Server error. Waiting {Delay}s before retry {Attempt}/{Max}", delay, attempt, maxRetries);
+                await Task.Delay(TimeSpan.FromSeconds(delay), cancellationToken);
+            }
+        }
+
+        throw new InvalidOperationException("Retry logic failed unexpectedly");
+    }
+
     public async Task<List<DownloadedTileInfoV2>> GetTilesWithMetadataAsync(
         GeoPoint centerGeoPoint, 
         double radiusM, 
@@ -169,13 +223,17 @@ public class GoogleMapsDownloaderV2
                     var fileName = $"tile_{zoomLevel}_{x}_{y}_{timestamp}.jpg";
                     var filePath = Path.Combine(_tilesDirectory, fileName);
 
+                    var imageBytes = await ExecuteWithRetryAsync(async () =>
+                    {
                         using var httpClient = _httpClientFactory.CreateClient();
                         httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(USER_AGENT);
 
                         var response = await httpClient.GetAsync(url, token);
                         response.EnsureSuccessStatusCode();
 
-                    var imageBytes = await response.Content.ReadAsByteArrayAsync(token);
+                        return await response.Content.ReadAsByteArrayAsync(token);
+                    }, cancellationToken: token);
+
                     await File.WriteAllBytesAsync(filePath, imageBytes, token);
 
                     _logger.LogInformation("Downloaded tile ({X}, {Y}) to {FilePath}, center=({Lat:F6}, {Lon:F6}), size={Size:F2}m", 
@@ -184,6 +242,11 @@ public class GoogleMapsDownloaderV2
                     downloadedTiles.Add(new DownloadedTileInfoV2(
                         x, y, zoomLevel, tileCenter.Lat, tileCenter.Lon, filePath, tileSizeMeters));
                 }
+                catch (RateLimitException ex)
+                {
+                    _logger.LogError(ex, "Rate limit exceeded for tile ({X}, {Y})", x, y);
+                    throw;
+                }
                 catch (Exception ex)
                 {
                     _logger.LogError(ex, "Failed to download tile ({X}, {Y})", x, y);

SatelliteProvider.Services/RegionService.cs

@@ -91,6 +91,14 @@ public class RegionService : IRegionService
         region.UpdatedAt = DateTime.UtcNow;
         await _regionRepository.UpdateAsync(region);
 
+        using var timeoutCts = new CancellationTokenSource(TimeSpan.FromMinutes(5));
+        using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCts.Token);
+
+        string? errorMessage = null;
+        List<TileMetadata>? tiles = null;
+        int tilesDownloaded = 0;
+        int tilesReused = 0;
+
         try
         {
             _logger.LogInformation("Downloading tiles for region {RegionId} at ({Lat}, {Lon}) size {Size}m zoom {Zoom}",
@@ -108,15 +116,15 @@ public class RegionService : IRegionService
             _logger.LogInformation("Found {Count} existing tiles for region {RegionId}", existingTileIds.Count, id);
 
             _logger.LogInformation("Starting tile download for region {RegionId}", id);
-            var tiles = await _tileService.DownloadAndStoreTilesAsync(
+            tiles = await _tileService.DownloadAndStoreTilesAsync(
                 region.Latitude,
                 region.Longitude,
                 region.SizeMeters,
                 region.ZoomLevel,
-                cancellationToken);
+                linkedCts.Token);
 
-            var tilesDownloaded = tiles.Count(t => !existingTileIds.Contains(t.Id));
+            tilesDownloaded = tiles.Count(t => !existingTileIds.Contains(t.Id));
-            var tilesReused = tiles.Count(t => existingTileIds.Contains(t.Id));
+            tilesReused = tiles.Count(t => existingTileIds.Contains(t.Id));
 
             _logger.LogInformation("Region {RegionId}: Downloaded {Downloaded} tiles, Reused {Reused} tiles", 
                 id, tilesDownloaded, tilesReused);
@@ -128,12 +136,12 @@ public class RegionService : IRegionService
             var summaryPath = Path.Combine(readyDir, $"region_{id}_summary.txt");
             var stitchedImagePath = Path.Combine(readyDir, $"region_{id}_stitched.jpg");
 
-            await GenerateCsvFileAsync(csvPath, tiles, cancellationToken);
+            await GenerateCsvFileAsync(csvPath, tiles, linkedCts.Token);
             
             _logger.LogInformation("Stitching tiles for region {RegionId}", id);
-            await StitchTilesAsync(tiles, region.Latitude, region.Longitude, region.ZoomLevel, stitchedImagePath, cancellationToken);
+            await StitchTilesAsync(tiles, region.Latitude, region.Longitude, region.ZoomLevel, stitchedImagePath, linkedCts.Token);
             
-            await GenerateSummaryFileAsync(summaryPath, id, region, tiles, tilesDownloaded, tilesReused, stitchedImagePath, processingStartTime, cancellationToken);
+            await GenerateSummaryFileAsync(summaryPath, id, region, tiles, tilesDownloaded, tilesReused, stitchedImagePath, processingStartTime, linkedCts.Token, errorMessage);
 
             region.Status = "completed";
             region.CsvFilePath = csvPath;
@@ -146,13 +154,70 @@ public class RegionService : IRegionService
             var duration = (DateTime.UtcNow - startTime).TotalSeconds;
             _logger.LogInformation("Region {RegionId} processing completed in {Duration:F2}s", id, duration);
         }
+        catch (OperationCanceledException) when (timeoutCts.IsCancellationRequested)
+        {
+            errorMessage = "Processing timed out after 5 minutes. Unable to download tiles within the time limit.";
+            _logger.LogError("Region {RegionId} processing timed out after 5 minutes", id);
+            await HandleProcessingFailureAsync(id, region, startTime, tiles, tilesDownloaded, tilesReused, errorMessage);
+        }
+        catch (RateLimitException ex)
+        {
+            errorMessage = $"Rate limit exceeded: {ex.Message}. Google Maps API rate limit was reached and retries were exhausted.";
+            _logger.LogError(ex, "Rate limit exceeded for region {RegionId}", id);
+            await HandleProcessingFailureAsync(id, region, startTime, tiles, tilesDownloaded, tilesReused, errorMessage);
+        }
+        catch (HttpRequestException ex)
+        {
+            errorMessage = $"Network error: {ex.Message}. Failed to download tiles from Google Maps.";
+            _logger.LogError(ex, "Network error processing region {RegionId}", id);
+            await HandleProcessingFailureAsync(id, region, startTime, tiles, tilesDownloaded, tilesReused, errorMessage);
+        }
         catch (Exception ex)
         {
+            errorMessage = $"Unexpected error: {ex.Message}";
             _logger.LogError(ex, "Failed to process region {RegionId}: {Message}", id, ex.Message);
+            await HandleProcessingFailureAsync(id, region, startTime, tiles, tilesDownloaded, tilesReused, errorMessage);
+        }
+    }
+
+    private async Task HandleProcessingFailureAsync(
+        Guid id, 
+        RegionEntity region, 
+        DateTime startTime,
+        List<TileMetadata>? tiles,
+        int tilesDownloaded,
+        int tilesReused,
+        string errorMessage)
+    {
         region.Status = "failed";
         region.UpdatedAt = DateTime.UtcNow;
-            await _regionRepository.UpdateAsync(region);
+        
+        try
+        {
+            var readyDir = _storageConfig.ReadyDirectory;
+            Directory.CreateDirectory(readyDir);
+
+            var summaryPath = Path.Combine(readyDir, $"region_{id}_summary.txt");
+            region.SummaryFilePath = summaryPath;
+
+            await GenerateSummaryFileAsync(
+                summaryPath, 
+                id, 
+                region, 
+                tiles ?? new List<TileMetadata>(), 
+                tilesDownloaded, 
+                tilesReused, 
+                null,
+                startTime, 
+                CancellationToken.None,
+                errorMessage);
         }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Failed to generate error summary for region {RegionId}", id);
+        }
+
+        await _regionRepository.UpdateAsync(region);
     }
 
     private async Task<string> StitchTilesAsync(
@@ -265,9 +330,10 @@ public class RegionService : IRegionService
         List<TileMetadata> tiles,
         int tilesDownloaded,
         int tilesReused,
-        string stitchedImagePath,
+        string? stitchedImagePath,
         DateTime startTime,
-        CancellationToken cancellationToken)
+        CancellationToken cancellationToken,
+        string? errorMessage = null)
     {
         var endTime = DateTime.UtcNow;
         var processingTime = (endTime - startTime).TotalSeconds;
@@ -279,19 +345,46 @@ public class RegionService : IRegionService
         summary.AppendLine($"Center: {region.Latitude:F6}, {region.Longitude:F6}");
         summary.AppendLine($"Size: {region.SizeMeters:F0} meters");
         summary.AppendLine($"Zoom Level: {region.ZoomLevel}");
+        summary.AppendLine($"Status: {region.Status}");
         summary.AppendLine();
+        
+        if (!string.IsNullOrEmpty(errorMessage))
+        {
+            summary.AppendLine("ERROR:");
+            summary.AppendLine(errorMessage);
+            summary.AppendLine();
+        }
+        
         summary.AppendLine("Processing Statistics:");
         summary.AppendLine($"- Tiles Downloaded: {tilesDownloaded}");
         summary.AppendLine($"- Tiles Reused from Cache: {tilesReused}");
         summary.AppendLine($"- Total Tiles: {tiles.Count}");
         summary.AppendLine($"- Processing Time: {processingTime:F2} seconds");
         summary.AppendLine($"- Started: {startTime:yyyy-MM-dd HH:mm:ss} UTC");
+        
+        if (region.Status == "completed")
+        {
             summary.AppendLine($"- Completed: {endTime:yyyy-MM-dd HH:mm:ss} UTC");
+        }
+        else
+        {
+            summary.AppendLine($"- Failed: {endTime:yyyy-MM-dd HH:mm:ss} UTC");
+        }
+        
         summary.AppendLine();
         summary.AppendLine("Files Created:");
+        
+        if (tiles.Count > 0)
+        {
             summary.AppendLine($"- CSV: {Path.GetFileName(filePath).Replace("_summary.txt", "_ready.csv")}");
+        }
+        
+        if (!string.IsNullOrEmpty(stitchedImagePath))
+        {
             summary.AppendLine($"- Stitched Image: {Path.GetFileName(stitchedImagePath)}");
             summary.AppendLine($"- Stitched Image Path: {stitchedImagePath}");
+        }
+        
         summary.AppendLine($"- Summary: {Path.GetFileName(filePath)}");
 
         await File.WriteAllTextAsync(filePath, summary.ToString(), cancellationToken);