azaion/satellite-provider
1
0
Fork 0
mirror of https://github.com/azaion/satellite-provider.git synced 2026-04-22 09:06:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

improve retries

Browse Source
This commit is contained in:
Anton Martynenko
2025-10-29 15:07:20 +01:00
parent 250d13c62e
commit f13f3eea6b
5 changed files with 186 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files
SatelliteProvider.Services/GoogleMapsDownloaderV2.cs
+73 -10
View File
@@ -1,3 +1,4 @@
using System.Net;
using System.Net.Http.Json;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -10,11 +11,18 @@ namespace SatelliteProvider.Services;
public record DownloadedTileInfoV2(int X, int Y, int ZoomLevel, double CenterLatitude, double CenterLongitude, string FilePath, double TileSizeMeters);
public class RateLimitException : Exception
{
public RateLimitException(string message) : base(message) { }
}
public class GoogleMapsDownloaderV2
{
private const string TILE_URL_TEMPLATE = "https://mt{0}.google.com/vt/lyrs=s&x={1}&y={2}&z={3}&token={4}";
private const string USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.0.0 Safari/537.36";
private const int TILE_SIZE_PIXELS = 256;
private const int MAX_RETRY_DELAY_SECONDS = 30;
private const int BASE_RETRY_DELAY_SECONDS = 1;
private static readonly int[] ALLOWED_ZOOM_LEVELS = { 15, 16, 17, 18, 19 };
private readonly ILogger<GoogleMapsDownloaderV2> _logger;
@@ -78,13 +86,17 @@ public class GoogleMapsDownloaderV2
var fileName = $"tile_{zoomLevel}_{tileX}_{tileY}_{timestamp}.jpg";
var filePath = Path.Combine(_tilesDirectory, fileName);
using var httpClient = _httpClientFactory.CreateClient();
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(USER_AGENT);
var imageBytes = await ExecuteWithRetryAsync(async () =>
{
using var httpClient = _httpClientFactory.CreateClient();
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(USER_AGENT);
var response = await httpClient.GetAsync(url, token);
response.EnsureSuccessStatusCode();
var response = await httpClient.GetAsync(url, token);
response.EnsureSuccessStatusCode();
return await response.Content.ReadAsByteArrayAsync(token);
}, cancellationToken: token);
var imageBytes = await response.Content.ReadAsByteArrayAsync(token);
await File.WriteAllBytesAsync(filePath, imageBytes, token);
var tileCenter = GeoUtils.TileToWorldPos(tileX, tileY, zoomLevel);
@@ -111,6 +123,48 @@ public class GoogleMapsDownloaderV2
return metersPerPixel * TILE_SIZE_PIXELS;
}
private async Task<T> ExecuteWithRetryAsync<T>(Func<Task<T>> action, int maxRetries = 5, CancellationToken cancellationToken = default)
{
int attempt = 0;
int delay = BASE_RETRY_DELAY_SECONDS;
while (attempt < maxRetries)
{
try
{
return await action();
}
catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.TooManyRequests || ex.StatusCode == (HttpStatusCode)429)
{
attempt++;
if (attempt >= maxRetries)
{
_logger.LogError("Rate limit exceeded after {Attempts} attempts", maxRetries);
throw new RateLimitException($"Rate limit exceeded after {maxRetries} attempts");
}
delay = Math.Min(delay * 2, MAX_RETRY_DELAY_SECONDS);
_logger.LogWarning("Rate limited. Waiting {Delay}s before retry {Attempt}/{Max}", delay, attempt, maxRetries);
await Task.Delay(TimeSpan.FromSeconds(delay), cancellationToken);
}
catch (HttpRequestException ex) when (ex.StatusCode >= HttpStatusCode.InternalServerError)
{
attempt++;
if (attempt >= maxRetries)
{
_logger.LogError(ex, "Server error after {Attempts} attempts", maxRetries);
throw;
}
delay = Math.Min(delay * 2, MAX_RETRY_DELAY_SECONDS);
_logger.LogWarning("Server error. Waiting {Delay}s before retry {Attempt}/{Max}", delay, attempt, maxRetries);
await Task.Delay(TimeSpan.FromSeconds(delay), cancellationToken);
}
}
throw new InvalidOperationException("Retry logic failed unexpectedly");
}
public async Task<List<DownloadedTileInfoV2>> GetTilesWithMetadataAsync(
GeoPoint centerGeoPoint,
double radiusM,
@@ -169,13 +223,17 @@ public class GoogleMapsDownloaderV2
var fileName = $"tile_{zoomLevel}_{x}_{y}_{timestamp}.jpg";
var filePath = Path.Combine(_tilesDirectory, fileName);
using var httpClient = _httpClientFactory.CreateClient();
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(USER_AGENT);
var imageBytes = await ExecuteWithRetryAsync(async () =>
{
using var httpClient = _httpClientFactory.CreateClient();
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(USER_AGENT);
var response = await httpClient.GetAsync(url, token);
response.EnsureSuccessStatusCode();
var response = await httpClient.GetAsync(url, token);
response.EnsureSuccessStatusCode();
return await response.Content.ReadAsByteArrayAsync(token);
}, cancellationToken: token);
var imageBytes = await response.Content.ReadAsByteArrayAsync(token);
await File.WriteAllBytesAsync(filePath, imageBytes, token);
_logger.LogInformation("Downloaded tile ({X}, {Y}) to {FilePath}, center=({Lat:F6}, {Lon:F6}), size={Size:F2}m",
@@ -184,6 +242,11 @@ public class GoogleMapsDownloaderV2
downloadedTiles.Add(new DownloadedTileInfoV2(
x, y, zoomLevel, tileCenter.Lat, tileCenter.Lon, filePath, tileSizeMeters));
}
catch (RateLimitException ex)
{
_logger.LogError(ex, "Rate limit exceeded for tile ({X}, {Y})", x, y);
throw;
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to download tile ({X}, {Y})", x, y);