using System.Net; using System.Net.Http.Json; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Newtonsoft.Json; using SatelliteProvider.Common.Configs; using SatelliteProvider.Common.DTO; using SatelliteProvider.Common.Utils; namespace SatelliteProvider.Services; public record DownloadedTileInfoV2(int X, int Y, int ZoomLevel, double CenterLatitude, double CenterLongitude, string FilePath, double TileSizeMeters); public class RateLimitException : Exception { public RateLimitException(string message) : base(message) { } } public class GoogleMapsDownloaderV2 { private const string TILE_URL_TEMPLATE = "https://mt{0}.google.com/vt/lyrs=s&x={1}&y={2}&z={3}&token={4}"; private const string USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.0.0 Safari/537.36"; private const int TILE_SIZE_PIXELS = 256; private const int MAX_RETRY_DELAY_SECONDS = 30; private const int BASE_RETRY_DELAY_SECONDS = 1; private static readonly int[] ALLOWED_ZOOM_LEVELS = { 15, 16, 17, 18, 19 }; private readonly ILogger _logger; private readonly string _apiKey; private readonly StorageConfig _storageConfig; private readonly ProcessingConfig _processingConfig; private readonly IHttpClientFactory _httpClientFactory; private readonly SemaphoreSlim _downloadSemaphore; private static readonly System.Collections.Concurrent.ConcurrentDictionary> _activeDownloads = new(); public GoogleMapsDownloaderV2( ILogger logger, IOptions mapConfig, IOptions storageConfig, IOptions processingConfig, IHttpClientFactory httpClientFactory) { _logger = logger; _apiKey = mapConfig.Value.ApiKey; _storageConfig = storageConfig.Value; _processingConfig = processingConfig.Value; _httpClientFactory = httpClientFactory; _downloadSemaphore = new SemaphoreSlim(_processingConfig.MaxConcurrentDownloads, _processingConfig.MaxConcurrentDownloads); } private record SessionResponse(string Session); private async Task GetSessionToken() { var url = $"https://tile.googleapis.com/v1/createSession?key={_apiKey}"; using var httpClient = _httpClientFactory.CreateClient(); try { var str = JsonConvert.SerializeObject(new { mapType = "satellite" }); var response = await httpClient.PostAsync(url, new StringContent(str)); if (!response.IsSuccessStatusCode) { var errorBody = await response.Content.ReadAsStringAsync(); _logger.LogError("Failed to get session token. Status: {StatusCode}, Response: {Response}", response.StatusCode, errorBody); } response.EnsureSuccessStatusCode(); var sessionResponse = await response.Content.ReadFromJsonAsync(); return sessionResponse?.Session; } catch (TaskCanceledException ex) { _logger.LogError(ex, "Session token request cancelled or timed out"); throw; } catch (HttpRequestException ex) { _logger.LogError(ex, "HTTP request failed while getting session token. StatusCode: {StatusCode}", ex.StatusCode); throw; } catch (Exception e) { _logger.LogError(e, "Unexpected error getting session token"); throw; } } public async Task DownloadSingleTileAsync(double latitude, double longitude, int zoomLevel, CancellationToken token = default) { if (!ALLOWED_ZOOM_LEVELS.Contains(zoomLevel)) { throw new ArgumentException($"Zoom level {zoomLevel} is not allowed. Allowed zoom levels are: {string.Join(", ", ALLOWED_ZOOM_LEVELS)}", nameof(zoomLevel)); } var geoPoint = new GeoPoint(latitude, longitude); var (tileX, tileY) = GeoUtils.WorldToTilePos(geoPoint, zoomLevel); var sessionToken = await GetSessionToken(); var server = 0; var url = string.Format(TILE_URL_TEMPLATE, server, tileX, tileY, zoomLevel, sessionToken); var timestamp = DateTime.UtcNow.ToString("yyyyMMddHHmmss"); var subdirectory = _storageConfig.GetTileSubdirectoryPath(zoomLevel, tileX, tileY); Directory.CreateDirectory(subdirectory); var filePath = _storageConfig.GetTileFilePath(zoomLevel, tileX, tileY, timestamp); var imageBytes = await ExecuteWithRetryAsync(async () => { using var httpClient = _httpClientFactory.CreateClient(); httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(USER_AGENT); var response = await httpClient.GetAsync(url, token); if (!response.IsSuccessStatusCode) { var errorBody = await response.Content.ReadAsStringAsync(token); _logger.LogError("Single tile download failed. Tile: ({X}, {Y}), Status: {StatusCode}, Response: {Response}", tileX, tileY, response.StatusCode, errorBody); } response.EnsureSuccessStatusCode(); return await response.Content.ReadAsByteArrayAsync(token); }, cancellationToken: token); await File.WriteAllBytesAsync(filePath, imageBytes, token); var tileCenter = GeoUtils.TileToWorldPos(tileX, tileY, zoomLevel); var tileSizeMeters = CalculateTileSizeInMeters(zoomLevel, tileCenter.Lat); return new DownloadedTileInfoV2( tileX, tileY, zoomLevel, tileCenter.Lat, tileCenter.Lon, filePath, tileSizeMeters ); } private static double CalculateTileSizeInMeters(int zoomLevel, double latitude) { const double EARTH_CIRCUMFERENCE_METERS = 40075016.686; var latRad = latitude * Math.PI / 180.0; var metersPerPixel = (EARTH_CIRCUMFERENCE_METERS * Math.Cos(latRad)) / (Math.Pow(2, zoomLevel) * TILE_SIZE_PIXELS); return metersPerPixel * TILE_SIZE_PIXELS; } private async Task ExecuteWithRetryAsync(Func> action, int maxRetries = 5, CancellationToken cancellationToken = default) { int attempt = 0; int delay = BASE_RETRY_DELAY_SECONDS; Exception? lastException = null; while (attempt < maxRetries) { try { return await action(); } catch (TaskCanceledException ex) { _logger.LogError(ex, "Request was cancelled (timeout or explicit cancellation). Attempt {Attempt}/{Max}", attempt + 1, maxRetries); throw; } catch (OperationCanceledException ex) { _logger.LogError(ex, "Operation was cancelled. Attempt {Attempt}/{Max}", attempt + 1, maxRetries); throw; } catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.TooManyRequests || ex.StatusCode == (HttpStatusCode)429) { attempt++; lastException = ex; if (attempt >= maxRetries) { _logger.LogError(ex, "Rate limit (429) exceeded after {Attempts} attempts. This indicates Google Maps API throttling.", maxRetries); throw new RateLimitException($"Rate limit exceeded after {maxRetries} attempts. Google Maps API is throttling requests."); } delay = Math.Min(delay * 2, MAX_RETRY_DELAY_SECONDS); _logger.LogWarning("Rate limited (429 Too Many Requests). Waiting {Delay}s before retry {Attempt}/{Max}", delay, attempt, maxRetries); await Task.Delay(TimeSpan.FromSeconds(delay), cancellationToken); } catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.Forbidden) { _logger.LogError(ex, "Access forbidden (403). Check API key validity and permissions."); throw; } catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.Unauthorized) { _logger.LogError(ex, "Unauthorized (401). API key is invalid or missing."); throw; } catch (HttpRequestException ex) when (ex.StatusCode >= HttpStatusCode.InternalServerError) { attempt++; lastException = ex; if (attempt >= maxRetries) { _logger.LogError(ex, "Server error ({StatusCode}) after {Attempts} attempts", ex.StatusCode, maxRetries); throw; } delay = Math.Min(delay * 2, MAX_RETRY_DELAY_SECONDS); _logger.LogWarning("Server error ({StatusCode}). Waiting {Delay}s before retry {Attempt}/{Max}", ex.StatusCode, delay, attempt, maxRetries); await Task.Delay(TimeSpan.FromSeconds(delay), cancellationToken); } catch (HttpRequestException ex) { _logger.LogError(ex, "HTTP request failed with status {StatusCode}. Message: {Message}", ex.StatusCode, ex.Message); throw; } } if (lastException != null) { throw new InvalidOperationException($"Retry logic exhausted after {maxRetries} attempts", lastException); } throw new InvalidOperationException("Retry logic failed unexpectedly"); } public async Task> GetTilesWithMetadataAsync( GeoPoint centerGeoPoint, double radiusM, int zoomLevel, IEnumerable existingTiles, CancellationToken token = default) { if (!ALLOWED_ZOOM_LEVELS.Contains(zoomLevel)) { throw new ArgumentException($"Zoom level {zoomLevel} is not allowed. Allowed zoom levels are: {string.Join(", ", ALLOWED_ZOOM_LEVELS)}", nameof(zoomLevel)); } var (latMin, latMax, lonMin, lonMax) = GeoUtils.GetBoundingBox(centerGeoPoint, radiusM); var (xMin, yMin) = GeoUtils.WorldToTilePos(new GeoPoint(latMax, lonMin), zoomLevel); var (xMax, yMax) = GeoUtils.WorldToTilePos(new GeoPoint(latMin, lonMax), zoomLevel); var tilesToDownload = new List<(int x, int y, GeoPoint center, double tileSizeMeters)>(); int skippedCount = 0; for (var y = yMin; y <= yMax; y++) { for (var x = xMin; x <= xMax; x++) { var tileCenter = GeoUtils.TileToWorldPos(x, y, zoomLevel); var existingTile = existingTiles.FirstOrDefault(t => Math.Abs(t.Latitude - tileCenter.Lat) < 0.0001 && Math.Abs(t.Longitude - tileCenter.Lon) < 0.0001 && t.ZoomLevel == zoomLevel); if (existingTile != null) { skippedCount++; continue; } var tileSizeMeters = CalculateTileSizeInMeters(zoomLevel, tileCenter.Lat); tilesToDownload.Add((x, y, tileCenter, tileSizeMeters)); } } if (tilesToDownload.Count == 0) { return new List(); } var sessionToken = await GetSessionToken(); var downloadTasks = new List>(); int sessionTokenUsageCount = 0; for (int i = 0; i < tilesToDownload.Count; i++) { var tileInfo = tilesToDownload[i]; if (sessionTokenUsageCount >= _processingConfig.SessionTokenReuseCount) { sessionToken = await GetSessionToken(); sessionTokenUsageCount = 0; } var currentToken = sessionToken; var tileIndex = i; sessionTokenUsageCount++; var downloadTask = DownloadTileAsync( tileInfo.x, tileInfo.y, tileInfo.center, tileInfo.tileSizeMeters, zoomLevel, currentToken, tileIndex, tilesToDownload.Count, token); downloadTasks.Add(downloadTask); } var results = await Task.WhenAll(downloadTasks); var downloadedTiles = results.Where(r => r != null).Cast().ToList(); return downloadedTiles; } private async Task DownloadTileAsync( int x, int y, GeoPoint tileCenter, double tileSizeMeters, int zoomLevel, string? sessionToken, int tileIndex, int totalTiles, CancellationToken token) { var tileKey = $"{zoomLevel}_{x}_{y}"; var downloadTask = _activeDownloads.GetOrAdd(tileKey, _ => PerformDownloadAsync( x, y, tileCenter, tileSizeMeters, zoomLevel, sessionToken, tileIndex, totalTiles, token)); try { return await downloadTask; } finally { _activeDownloads.TryRemove(tileKey, out _); } } private async Task PerformDownloadAsync( int x, int y, GeoPoint tileCenter, double tileSizeMeters, int zoomLevel, string? sessionToken, int tileIndex, int totalTiles, CancellationToken token) { await _downloadSemaphore.WaitAsync(token); try { if (_processingConfig.DelayBetweenRequestsMs > 0) { await Task.Delay(_processingConfig.DelayBetweenRequestsMs, token); } var server = (x + y) % 4; var url = string.Format(TILE_URL_TEMPLATE, server, x, y, zoomLevel, sessionToken); var timestamp = DateTime.UtcNow.ToString("yyyyMMddHHmmss"); var subdirectory = _storageConfig.GetTileSubdirectoryPath(zoomLevel, x, y); Directory.CreateDirectory(subdirectory); var filePath = _storageConfig.GetTileFilePath(zoomLevel, x, y, timestamp); var imageBytes = await ExecuteWithRetryAsync(async () => { using var httpClient = _httpClientFactory.CreateClient(); httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(USER_AGENT); var response = await httpClient.GetAsync(url, token); if (!response.IsSuccessStatusCode) { var errorBody = await response.Content.ReadAsStringAsync(token); _logger.LogError("Tile download failed. Tile: ({X}, {Y}), Status: {StatusCode}, Response: {Response}", x, y, response.StatusCode, errorBody); } response.EnsureSuccessStatusCode(); return await response.Content.ReadAsByteArrayAsync(token); }, cancellationToken: token); await File.WriteAllBytesAsync(filePath, imageBytes, token); return new DownloadedTileInfoV2( x, y, zoomLevel, tileCenter.Lat, tileCenter.Lon, filePath, tileSizeMeters); } catch (TaskCanceledException ex) { _logger.LogError(ex, "Tile download cancelled for ({X}, {Y})", x, y); throw; } catch (OperationCanceledException ex) { _logger.LogError(ex, "Tile download operation cancelled for ({X}, {Y})", x, y); throw; } catch (RateLimitException ex) { _logger.LogError(ex, "Rate limit exceeded for tile ({X}, {Y})", x, y); throw; } catch (HttpRequestException ex) { _logger.LogError(ex, "HTTP request failed for tile ({X}, {Y}). StatusCode: {StatusCode}", x, y, ex.StatusCode); throw; } catch (Exception ex) { _logger.LogError(ex, "Unexpected error downloading tile ({X}, {Y})", x, y); throw; } finally { _downloadSemaphore.Release(); } } }