satellite-provider/SatelliteProvider.Services/GoogleMapsDownloaderV2.cs

using System.Net;
using System.Net.Http.Json;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Newtonsoft.Json;
using SatelliteProvider.Common.Configs;
using SatelliteProvider.Common.DTO;
using SatelliteProvider.Common.Utils;

namespace SatelliteProvider.Services;

public record DownloadedTileInfoV2(int X, int Y, int ZoomLevel, double CenterLatitude, double CenterLongitude, string FilePath, double TileSizeMeters);

public class RateLimitException : Exception
{
    public RateLimitException(string message) : base(message) { }
}

public class GoogleMapsDownloaderV2
{
    private const string TILE_URL_TEMPLATE = "https://mt{0}.google.com/vt/lyrs=s&x={1}&y={2}&z={3}&token={4}";
    private const string USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.0.0 Safari/537.36";
    private const int TILE_SIZE_PIXELS = 256;
    private const int MAX_RETRY_DELAY_SECONDS = 30;
    private const int BASE_RETRY_DELAY_SECONDS = 1;
    private static readonly int[] ALLOWED_ZOOM_LEVELS = { 15, 16, 17, 18, 19 };
    
    private readonly ILogger<GoogleMapsDownloaderV2> _logger;
    private readonly string _apiKey;
    private readonly StorageConfig _storageConfig;
    private readonly ProcessingConfig _processingConfig;
    private readonly IHttpClientFactory _httpClientFactory;
    private readonly SemaphoreSlim _downloadSemaphore;
    private static readonly System.Collections.Concurrent.ConcurrentDictionary<string, Task<DownloadedTileInfoV2>> _activeDownloads = new();

    public GoogleMapsDownloaderV2(
        ILogger<GoogleMapsDownloaderV2> logger, 
        IOptions<MapConfig> mapConfig, 
        IOptions<StorageConfig> storageConfig, 
        IOptions<ProcessingConfig> processingConfig,
        IHttpClientFactory httpClientFactory)
    {
        _logger = logger;
        _apiKey = mapConfig.Value.ApiKey;
        _storageConfig = storageConfig.Value;
        _processingConfig = processingConfig.Value;
        _httpClientFactory = httpClientFactory;
        _downloadSemaphore = new SemaphoreSlim(_processingConfig.MaxConcurrentDownloads, _processingConfig.MaxConcurrentDownloads);
    }

    private record SessionResponse(string Session);

    private async Task<string?> GetSessionToken()
    {
        var url = $"https://tile.googleapis.com/v1/createSession?key={_apiKey}";
        using var httpClient = _httpClientFactory.CreateClient();
        try
        {
            var str = JsonConvert.SerializeObject(new { mapType = "satellite" });
            var response = await httpClient.PostAsync(url, new StringContent(str));
            
            if (!response.IsSuccessStatusCode)
            {
                var errorBody = await response.Content.ReadAsStringAsync();
                _logger.LogError("Failed to get session token. Status: {StatusCode}, Response: {Response}", 
                    response.StatusCode, errorBody);
            }
            
            response.EnsureSuccessStatusCode();
            var sessionResponse = await response.Content.ReadFromJsonAsync<SessionResponse>();
            return sessionResponse?.Session;
        }
        catch (TaskCanceledException ex)
        {
            _logger.LogError(ex, "Session token request cancelled or timed out");
            throw;
        }
        catch (HttpRequestException ex)
        {
            _logger.LogError(ex, "HTTP request failed while getting session token. StatusCode: {StatusCode}", ex.StatusCode);
            throw;
        }
        catch (Exception e)
        {
            _logger.LogError(e, "Unexpected error getting session token");
            throw;
        }
    }

    public async Task<DownloadedTileInfoV2> DownloadSingleTileAsync(double latitude, double longitude, int zoomLevel, CancellationToken token = default)
    {
        if (!ALLOWED_ZOOM_LEVELS.Contains(zoomLevel))
        {
            throw new ArgumentException($"Zoom level {zoomLevel} is not allowed. Allowed zoom levels are: {string.Join(", ", ALLOWED_ZOOM_LEVELS)}", nameof(zoomLevel));
        }

        var geoPoint = new GeoPoint(latitude, longitude);
        var (tileX, tileY) = GeoUtils.WorldToTilePos(geoPoint, zoomLevel);

        var sessionToken = await GetSessionToken();
        var server = 0;
        var url = string.Format(TILE_URL_TEMPLATE, server, tileX, tileY, zoomLevel, sessionToken);

        var timestamp = DateTime.UtcNow.ToString("yyyyMMddHHmmss");
        var subdirectory = _storageConfig.GetTileSubdirectoryPath(zoomLevel, tileX, tileY);
        Directory.CreateDirectory(subdirectory);
        
        var filePath = _storageConfig.GetTileFilePath(zoomLevel, tileX, tileY, timestamp);

        var imageBytes = await ExecuteWithRetryAsync(async () =>
        {
            using var httpClient = _httpClientFactory.CreateClient();
            httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(USER_AGENT);

            var response = await httpClient.GetAsync(url, token);
            
            if (!response.IsSuccessStatusCode)
            {
                var errorBody = await response.Content.ReadAsStringAsync(token);
                _logger.LogError("Single tile download failed. Tile: ({X}, {Y}), Status: {StatusCode}, Response: {Response}", 
                    tileX, tileY, response.StatusCode, errorBody);
            }
            
            response.EnsureSuccessStatusCode();

            return await response.Content.ReadAsByteArrayAsync(token);
        }, cancellationToken: token);

        await File.WriteAllBytesAsync(filePath, imageBytes, token);

        var tileCenter = GeoUtils.TileToWorldPos(tileX, tileY, zoomLevel);
        var tileSizeMeters = CalculateTileSizeInMeters(zoomLevel, tileCenter.Lat);

        return new DownloadedTileInfoV2(
            tileX,
            tileY,
            zoomLevel,
            tileCenter.Lat,
            tileCenter.Lon,
            filePath,
            tileSizeMeters
        );
    }

    private static double CalculateTileSizeInMeters(int zoomLevel, double latitude)
    {
        const double EARTH_CIRCUMFERENCE_METERS = 40075016.686;
        var latRad = latitude * Math.PI / 180.0;
        var metersPerPixel = (EARTH_CIRCUMFERENCE_METERS * Math.Cos(latRad)) / (Math.Pow(2, zoomLevel) * TILE_SIZE_PIXELS);
        return metersPerPixel * TILE_SIZE_PIXELS;
    }

    private async Task<T> ExecuteWithRetryAsync<T>(Func<Task<T>> action, int maxRetries = 5, CancellationToken cancellationToken = default)
    {
        int attempt = 0;
        int delay = BASE_RETRY_DELAY_SECONDS;
        Exception? lastException = null;

        while (attempt < maxRetries)
        {
            try
            {
                return await action();
            }
            catch (TaskCanceledException ex)
            {
                _logger.LogError(ex, "Request was cancelled (timeout or explicit cancellation). Attempt {Attempt}/{Max}", attempt + 1, maxRetries);
                throw;
            }
            catch (OperationCanceledException ex)
            {
                _logger.LogError(ex, "Operation was cancelled. Attempt {Attempt}/{Max}", attempt + 1, maxRetries);
                throw;
            }
            catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.TooManyRequests || ex.StatusCode == (HttpStatusCode)429)
            {
                attempt++;
                lastException = ex;
                
                if (attempt >= maxRetries)
                {
                    _logger.LogError(ex, "Rate limit (429) exceeded after {Attempts} attempts. This indicates Google Maps API throttling.", maxRetries);
                    throw new RateLimitException($"Rate limit exceeded after {maxRetries} attempts. Google Maps API is throttling requests.");
                }

                delay = Math.Min(delay * 2, MAX_RETRY_DELAY_SECONDS);
                _logger.LogWarning("Rate limited (429 Too Many Requests). Waiting {Delay}s before retry {Attempt}/{Max}", delay, attempt, maxRetries);
                await Task.Delay(TimeSpan.FromSeconds(delay), cancellationToken);
            }
            catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.Forbidden)
            {
                _logger.LogError(ex, "Access forbidden (403). Check API key validity and permissions.");
                throw;
            }
            catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.Unauthorized)
            {
                _logger.LogError(ex, "Unauthorized (401). API key is invalid or missing.");
                throw;
            }
            catch (HttpRequestException ex) when (ex.StatusCode >= HttpStatusCode.InternalServerError)
            {
                attempt++;
                lastException = ex;
                
                if (attempt >= maxRetries)
                {
                    _logger.LogError(ex, "Server error ({StatusCode}) after {Attempts} attempts", ex.StatusCode, maxRetries);
                    throw;
                }

                delay = Math.Min(delay * 2, MAX_RETRY_DELAY_SECONDS);
                _logger.LogWarning("Server error ({StatusCode}). Waiting {Delay}s before retry {Attempt}/{Max}", ex.StatusCode, delay, attempt, maxRetries);
                await Task.Delay(TimeSpan.FromSeconds(delay), cancellationToken);
            }
            catch (HttpRequestException ex)
            {
                _logger.LogError(ex, "HTTP request failed with status {StatusCode}. Message: {Message}", ex.StatusCode, ex.Message);
                throw;
            }
        }

        if (lastException != null)
        {
            throw new InvalidOperationException($"Retry logic exhausted after {maxRetries} attempts", lastException);
        }
        
        throw new InvalidOperationException("Retry logic failed unexpectedly");
    }

    public async Task<List<DownloadedTileInfoV2>> GetTilesWithMetadataAsync(
        GeoPoint centerGeoPoint, 
        double radiusM, 
        int zoomLevel, 
        IEnumerable<DataAccess.Models.TileEntity> existingTiles,
        CancellationToken token = default)
    {
        if (!ALLOWED_ZOOM_LEVELS.Contains(zoomLevel))
        {
            throw new ArgumentException($"Zoom level {zoomLevel} is not allowed. Allowed zoom levels are: {string.Join(", ", ALLOWED_ZOOM_LEVELS)}", nameof(zoomLevel));
        }

        var (latMin, latMax, lonMin, lonMax) = GeoUtils.GetBoundingBox(centerGeoPoint, radiusM);

        var (xMin, yMin) = GeoUtils.WorldToTilePos(new GeoPoint(latMax, lonMin), zoomLevel);
        var (xMax, yMax) = GeoUtils.WorldToTilePos(new GeoPoint(latMin, lonMax), zoomLevel);

        var tilesToDownload = new List<(int x, int y, GeoPoint center, double tileSizeMeters)>();
        int skippedCount = 0;

        for (var y = yMin; y <= yMax; y++)
        {
            for (var x = xMin; x <= xMax; x++)
            {
                var tileCenter = GeoUtils.TileToWorldPos(x, y, zoomLevel);
                
                var existingTile = existingTiles.FirstOrDefault(t =>
                    Math.Abs(t.Latitude - tileCenter.Lat) < 0.0001 &&
                    Math.Abs(t.Longitude - tileCenter.Lon) < 0.0001 &&
                    t.TileZoom == zoomLevel);

                if (existingTile != null)
                {
                    skippedCount++;
                    continue;
                }

                var tileSizeMeters = CalculateTileSizeInMeters(zoomLevel, tileCenter.Lat);
                tilesToDownload.Add((x, y, tileCenter, tileSizeMeters));
            }
        }

        if (tilesToDownload.Count == 0)
        {
            return new List<DownloadedTileInfoV2>();
        }

        var sessionToken = await GetSessionToken();
        
        var downloadTasks = new List<Task<DownloadedTileInfoV2?>>();
        int sessionTokenUsageCount = 0;

        for (int i = 0; i < tilesToDownload.Count; i++)
        {
            var tileInfo = tilesToDownload[i];
            
            if (sessionTokenUsageCount >= _processingConfig.SessionTokenReuseCount)
            {
                sessionToken = await GetSessionToken();
                sessionTokenUsageCount = 0;
            }

            var currentToken = sessionToken;
            var tileIndex = i;
            sessionTokenUsageCount++;

            var downloadTask = DownloadTileAsync(
                tileInfo.x, 
                tileInfo.y, 
                tileInfo.center, 
                tileInfo.tileSizeMeters, 
                zoomLevel, 
                currentToken,
                tileIndex,
                tilesToDownload.Count,
                token);

            downloadTasks.Add(downloadTask);
        }

        var results = await Task.WhenAll(downloadTasks);
        
        var downloadedTiles = results.Where(r => r != null).Cast<DownloadedTileInfoV2>().ToList();

        return downloadedTiles;
    }

    private async Task<DownloadedTileInfoV2?> DownloadTileAsync(
        int x, 
        int y, 
        GeoPoint tileCenter, 
        double tileSizeMeters, 
        int zoomLevel, 
        string? sessionToken,
        int tileIndex,
        int totalTiles,
        CancellationToken token)
    {
        var tileKey = $"{zoomLevel}_{x}_{y}";
        
        var downloadTask = _activeDownloads.GetOrAdd(tileKey, _ => PerformDownloadAsync(
            x, y, tileCenter, tileSizeMeters, zoomLevel, sessionToken, tileIndex, totalTiles, token));

        try
        {
            return await downloadTask;
        }
        finally
        {
            _activeDownloads.TryRemove(tileKey, out _);
        }
    }

    private async Task<DownloadedTileInfoV2> PerformDownloadAsync(
        int x, 
        int y, 
        GeoPoint tileCenter, 
        double tileSizeMeters, 
        int zoomLevel, 
        string? sessionToken,
        int tileIndex,
        int totalTiles,
        CancellationToken token)
    {
        await _downloadSemaphore.WaitAsync(token);

        try
        {
            if (_processingConfig.DelayBetweenRequestsMs > 0)
            {
                await Task.Delay(_processingConfig.DelayBetweenRequestsMs, token);
            }

            var server = (x + y) % 4;
            var url = string.Format(TILE_URL_TEMPLATE, server, x, y, zoomLevel, sessionToken);

            var timestamp = DateTime.UtcNow.ToString("yyyyMMddHHmmss");
            var subdirectory = _storageConfig.GetTileSubdirectoryPath(zoomLevel, x, y);
            Directory.CreateDirectory(subdirectory);
            
            var filePath = _storageConfig.GetTileFilePath(zoomLevel, x, y, timestamp);

            var imageBytes = await ExecuteWithRetryAsync(async () =>
            {
                using var httpClient = _httpClientFactory.CreateClient();
                httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(USER_AGENT);

                var response = await httpClient.GetAsync(url, token);
                
                if (!response.IsSuccessStatusCode)
                {
                    var errorBody = await response.Content.ReadAsStringAsync(token);
                    _logger.LogError("Tile download failed. Tile: ({X}, {Y}), Status: {StatusCode}, Response: {Response}", 
                        x, y, response.StatusCode, errorBody);
                }
                
                response.EnsureSuccessStatusCode();

                return await response.Content.ReadAsByteArrayAsync(token);
            }, cancellationToken: token);

            await File.WriteAllBytesAsync(filePath, imageBytes, token);

            return new DownloadedTileInfoV2(
                x, y, zoomLevel, tileCenter.Lat, tileCenter.Lon, filePath, tileSizeMeters);
        }
        catch (TaskCanceledException ex)
        {
            _logger.LogError(ex, "Tile download cancelled for ({X}, {Y})", x, y);
            throw;
        }
        catch (OperationCanceledException ex)
        {
            _logger.LogError(ex, "Tile download operation cancelled for ({X}, {Y})", x, y);
            throw;
        }
        catch (RateLimitException ex)
        {
            _logger.LogError(ex, "Rate limit exceeded for tile ({X}, {Y})", x, y);
            throw;
        }
        catch (HttpRequestException ex)
        {
            _logger.LogError(ex, "HTTP request failed for tile ({X}, {Y}). StatusCode: {StatusCode}", 
                x, y, ex.StatusCode);
            throw;
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Unexpected error downloading tile ({X}, {Y})", x, y);
            throw;
        }
        finally
        {
            _downloadSemaphore.Release();
        }
    }
}