mirror of
https://github.com/azaion/satellite-provider.git
synced 2026-06-22 15:41:13 +00:00
[AZ-312] [AZ-313] [AZ-314] Split Services into per-component csprojs
Phase B of architecture coupling refactor (epic AZ-309). Replaces the monolithic SatelliteProvider.Services with three per-component csprojs to add a compiler-enforced module boundary (resolves F4): - SatelliteProvider.Services.TileDownloader - SatelliteProvider.Services.RegionProcessing - SatelliteProvider.Services.RouteManagement DI registrations relocated into per-component AddTileDownloader / AddRegionProcessing / AddRouteManagement extension methods called from Program.cs. RateLimitException moved to Common/Exceptions/ to keep the three new csprojs as siblings (no Region->TileDownloader ProjectReference). Dockerfiles and consumer csprojs (Api, Tests) rewired to the new project paths. No DI lifetime or hosted-service order changes. Build: 0 warn, 0 err. Unit tests: 40/40. Smoke integration: green. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Oleksandr Bezdieniezhnykh
@@ -0,0 +1,423 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Newtonsoft.Json;
|
||||
using SatelliteProvider.Common.Configs;
|
||||
using SatelliteProvider.Common.DTO;
|
||||
using SatelliteProvider.Common.Exceptions;
|
||||
using SatelliteProvider.Common.Interfaces;
|
||||
using SatelliteProvider.Common.Utils;
|
||||
|
||||
namespace SatelliteProvider.Services.TileDownloader;
|
||||
|
||||
public class GoogleMapsDownloaderV2 : ISatelliteDownloader
|
||||
{
|
||||
private const string TILE_URL_TEMPLATE = "https://mt{0}.google.com/vt/lyrs=s&x={1}&y={2}&z={3}&token={4}";
|
||||
private const string USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.0.0 Safari/537.36";
|
||||
private const int TILE_SIZE_PIXELS = 256;
|
||||
private const int MAX_RETRY_DELAY_SECONDS = 30;
|
||||
private const int BASE_RETRY_DELAY_SECONDS = 1;
|
||||
private static readonly int[] ALLOWED_ZOOM_LEVELS = { 15, 16, 17, 18, 19 };
|
||||
|
||||
private readonly ILogger<GoogleMapsDownloaderV2> _logger;
|
||||
private readonly string _apiKey;
|
||||
private readonly StorageConfig _storageConfig;
|
||||
private readonly ProcessingConfig _processingConfig;
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly SemaphoreSlim _downloadSemaphore;
|
||||
private static readonly System.Collections.Concurrent.ConcurrentDictionary<string, Task<DownloadedTileInfoV2>> _activeDownloads = new();
|
||||
|
||||
public GoogleMapsDownloaderV2(
|
||||
ILogger<GoogleMapsDownloaderV2> logger,
|
||||
IOptions<MapConfig> mapConfig,
|
||||
IOptions<StorageConfig> storageConfig,
|
||||
IOptions<ProcessingConfig> processingConfig,
|
||||
IHttpClientFactory httpClientFactory)
|
||||
{
|
||||
_logger = logger;
|
||||
_apiKey = mapConfig.Value.ApiKey;
|
||||
_storageConfig = storageConfig.Value;
|
||||
_processingConfig = processingConfig.Value;
|
||||
_httpClientFactory = httpClientFactory;
|
||||
_downloadSemaphore = new SemaphoreSlim(_processingConfig.MaxConcurrentDownloads, _processingConfig.MaxConcurrentDownloads);
|
||||
}
|
||||
|
||||
private record SessionResponse(string Session);
|
||||
|
||||
private async Task<string?> GetSessionToken()
|
||||
{
|
||||
var url = $"https://tile.googleapis.com/v1/createSession?key={_apiKey}";
|
||||
using var httpClient = _httpClientFactory.CreateClient();
|
||||
try
|
||||
{
|
||||
var str = JsonConvert.SerializeObject(new { mapType = "satellite" });
|
||||
var response = await httpClient.PostAsync(url, new StringContent(str));
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
var errorBody = await response.Content.ReadAsStringAsync();
|
||||
_logger.LogError("Failed to get session token. Status: {StatusCode}, Response: {Response}",
|
||||
response.StatusCode, errorBody);
|
||||
}
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
var sessionResponse = await response.Content.ReadFromJsonAsync<SessionResponse>();
|
||||
return sessionResponse?.Session;
|
||||
}
|
||||
catch (TaskCanceledException ex)
|
||||
{
|
||||
_logger.LogError(ex, "Session token request cancelled or timed out");
|
||||
throw;
|
||||
}
|
||||
catch (HttpRequestException ex)
|
||||
{
|
||||
_logger.LogError(ex, "HTTP request failed while getting session token. StatusCode: {StatusCode}", ex.StatusCode);
|
||||
throw;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "Unexpected error getting session token");
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<DownloadedTileInfoV2> DownloadSingleTileAsync(double latitude, double longitude, int zoomLevel, CancellationToken token = default)
|
||||
{
|
||||
if (!ALLOWED_ZOOM_LEVELS.Contains(zoomLevel))
|
||||
{
|
||||
throw new ArgumentException($"Zoom level {zoomLevel} is not allowed. Allowed zoom levels are: {string.Join(", ", ALLOWED_ZOOM_LEVELS)}", nameof(zoomLevel));
|
||||
}
|
||||
|
||||
var geoPoint = new GeoPoint(latitude, longitude);
|
||||
var (tileX, tileY) = GeoUtils.WorldToTilePos(geoPoint, zoomLevel);
|
||||
|
||||
var sessionToken = await GetSessionToken();
|
||||
var server = 0;
|
||||
var url = string.Format(TILE_URL_TEMPLATE, server, tileX, tileY, zoomLevel, sessionToken);
|
||||
|
||||
var timestamp = DateTime.UtcNow.ToString("yyyyMMddHHmmss");
|
||||
var subdirectory = _storageConfig.GetTileSubdirectoryPath(zoomLevel, tileX, tileY);
|
||||
Directory.CreateDirectory(subdirectory);
|
||||
|
||||
var filePath = _storageConfig.GetTileFilePath(zoomLevel, tileX, tileY, timestamp);
|
||||
|
||||
var imageBytes = await ExecuteWithRetryAsync(async () =>
|
||||
{
|
||||
using var httpClient = _httpClientFactory.CreateClient();
|
||||
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(USER_AGENT);
|
||||
|
||||
var response = await httpClient.GetAsync(url, token);
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
var errorBody = await response.Content.ReadAsStringAsync(token);
|
||||
_logger.LogError("Single tile download failed. Tile: ({X}, {Y}), Status: {StatusCode}, Response: {Response}",
|
||||
tileX, tileY, response.StatusCode, errorBody);
|
||||
}
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
return await response.Content.ReadAsByteArrayAsync(token);
|
||||
}, cancellationToken: token);
|
||||
|
||||
await File.WriteAllBytesAsync(filePath, imageBytes, token);
|
||||
|
||||
var tileCenter = GeoUtils.TileToWorldPos(tileX, tileY, zoomLevel);
|
||||
var tileSizeMeters = CalculateTileSizeInMeters(zoomLevel, tileCenter.Lat);
|
||||
|
||||
return new DownloadedTileInfoV2(
|
||||
tileX,
|
||||
tileY,
|
||||
zoomLevel,
|
||||
tileCenter.Lat,
|
||||
tileCenter.Lon,
|
||||
filePath,
|
||||
tileSizeMeters
|
||||
);
|
||||
}
|
||||
|
||||
private static double CalculateTileSizeInMeters(int zoomLevel, double latitude)
|
||||
{
|
||||
const double EARTH_CIRCUMFERENCE_METERS = 40075016.686;
|
||||
var latRad = latitude * Math.PI / 180.0;
|
||||
var metersPerPixel = (EARTH_CIRCUMFERENCE_METERS * Math.Cos(latRad)) / (Math.Pow(2, zoomLevel) * TILE_SIZE_PIXELS);
|
||||
return metersPerPixel * TILE_SIZE_PIXELS;
|
||||
}
|
||||
|
||||
private async Task<T> ExecuteWithRetryAsync<T>(Func<Task<T>> action, int maxRetries = 5, CancellationToken cancellationToken = default)
|
||||
{
|
||||
int attempt = 0;
|
||||
int delay = BASE_RETRY_DELAY_SECONDS;
|
||||
Exception? lastException = null;
|
||||
|
||||
while (attempt < maxRetries)
|
||||
{
|
||||
try
|
||||
{
|
||||
return await action();
|
||||
}
|
||||
catch (TaskCanceledException ex)
|
||||
{
|
||||
_logger.LogError(ex, "Request was cancelled (timeout or explicit cancellation). Attempt {Attempt}/{Max}", attempt + 1, maxRetries);
|
||||
throw;
|
||||
}
|
||||
catch (OperationCanceledException ex)
|
||||
{
|
||||
_logger.LogError(ex, "Operation was cancelled. Attempt {Attempt}/{Max}", attempt + 1, maxRetries);
|
||||
throw;
|
||||
}
|
||||
catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.TooManyRequests || ex.StatusCode == (HttpStatusCode)429)
|
||||
{
|
||||
attempt++;
|
||||
lastException = ex;
|
||||
|
||||
if (attempt >= maxRetries)
|
||||
{
|
||||
_logger.LogError(ex, "Rate limit (429) exceeded after {Attempts} attempts. This indicates Google Maps API throttling.", maxRetries);
|
||||
throw new RateLimitException($"Rate limit exceeded after {maxRetries} attempts. Google Maps API is throttling requests.");
|
||||
}
|
||||
|
||||
delay = Math.Min(delay * 2, MAX_RETRY_DELAY_SECONDS);
|
||||
_logger.LogWarning("Rate limited (429 Too Many Requests). Waiting {Delay}s before retry {Attempt}/{Max}", delay, attempt, maxRetries);
|
||||
await Task.Delay(TimeSpan.FromSeconds(delay), cancellationToken);
|
||||
}
|
||||
catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.Forbidden)
|
||||
{
|
||||
_logger.LogError(ex, "Access forbidden (403). Check API key validity and permissions.");
|
||||
throw;
|
||||
}
|
||||
catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.Unauthorized)
|
||||
{
|
||||
_logger.LogError(ex, "Unauthorized (401). API key is invalid or missing.");
|
||||
throw;
|
||||
}
|
||||
catch (HttpRequestException ex) when (ex.StatusCode >= HttpStatusCode.InternalServerError)
|
||||
{
|
||||
attempt++;
|
||||
lastException = ex;
|
||||
|
||||
if (attempt >= maxRetries)
|
||||
{
|
||||
_logger.LogError(ex, "Server error ({StatusCode}) after {Attempts} attempts", ex.StatusCode, maxRetries);
|
||||
throw;
|
||||
}
|
||||
|
||||
delay = Math.Min(delay * 2, MAX_RETRY_DELAY_SECONDS);
|
||||
_logger.LogWarning("Server error ({StatusCode}). Waiting {Delay}s before retry {Attempt}/{Max}", ex.StatusCode, delay, attempt, maxRetries);
|
||||
await Task.Delay(TimeSpan.FromSeconds(delay), cancellationToken);
|
||||
}
|
||||
catch (HttpRequestException ex)
|
||||
{
|
||||
_logger.LogError(ex, "HTTP request failed with status {StatusCode}. Message: {Message}", ex.StatusCode, ex.Message);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
if (lastException != null)
|
||||
{
|
||||
throw new InvalidOperationException($"Retry logic exhausted after {maxRetries} attempts", lastException);
|
||||
}
|
||||
|
||||
throw new InvalidOperationException("Retry logic failed unexpectedly");
|
||||
}
|
||||
|
||||
public async Task<List<DownloadedTileInfoV2>> GetTilesWithMetadataAsync(
|
||||
GeoPoint centerGeoPoint,
|
||||
double radiusM,
|
||||
int zoomLevel,
|
||||
IEnumerable<ExistingTileInfo> existingTiles,
|
||||
CancellationToken token = default)
|
||||
{
|
||||
if (!ALLOWED_ZOOM_LEVELS.Contains(zoomLevel))
|
||||
{
|
||||
throw new ArgumentException($"Zoom level {zoomLevel} is not allowed. Allowed zoom levels are: {string.Join(", ", ALLOWED_ZOOM_LEVELS)}", nameof(zoomLevel));
|
||||
}
|
||||
|
||||
var (latMin, latMax, lonMin, lonMax) = GeoUtils.GetBoundingBox(centerGeoPoint, radiusM);
|
||||
|
||||
var (xMin, yMin) = GeoUtils.WorldToTilePos(new GeoPoint(latMax, lonMin), zoomLevel);
|
||||
var (xMax, yMax) = GeoUtils.WorldToTilePos(new GeoPoint(latMin, lonMax), zoomLevel);
|
||||
|
||||
var tilesToDownload = new List<(int x, int y, GeoPoint center, double tileSizeMeters)>();
|
||||
int skippedCount = 0;
|
||||
|
||||
for (var y = yMin; y <= yMax; y++)
|
||||
{
|
||||
for (var x = xMin; x <= xMax; x++)
|
||||
{
|
||||
var tileCenter = GeoUtils.TileToWorldPos(x, y, zoomLevel);
|
||||
|
||||
var existingTile = existingTiles.FirstOrDefault(t =>
|
||||
Math.Abs(t.Latitude - tileCenter.Lat) < 0.0001 &&
|
||||
Math.Abs(t.Longitude - tileCenter.Lon) < 0.0001 &&
|
||||
t.TileZoom == zoomLevel);
|
||||
|
||||
if (existingTile != null)
|
||||
{
|
||||
skippedCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
var tileSizeMeters = CalculateTileSizeInMeters(zoomLevel, tileCenter.Lat);
|
||||
tilesToDownload.Add((x, y, tileCenter, tileSizeMeters));
|
||||
}
|
||||
}
|
||||
|
||||
if (tilesToDownload.Count == 0)
|
||||
{
|
||||
return new List<DownloadedTileInfoV2>();
|
||||
}
|
||||
|
||||
var sessionToken = await GetSessionToken();
|
||||
|
||||
var downloadTasks = new List<Task<DownloadedTileInfoV2?>>();
|
||||
int sessionTokenUsageCount = 0;
|
||||
|
||||
for (int i = 0; i < tilesToDownload.Count; i++)
|
||||
{
|
||||
var tileInfo = tilesToDownload[i];
|
||||
|
||||
if (sessionTokenUsageCount >= _processingConfig.SessionTokenReuseCount)
|
||||
{
|
||||
sessionToken = await GetSessionToken();
|
||||
sessionTokenUsageCount = 0;
|
||||
}
|
||||
|
||||
var currentToken = sessionToken;
|
||||
var tileIndex = i;
|
||||
sessionTokenUsageCount++;
|
||||
|
||||
var downloadTask = DownloadTileAsync(
|
||||
tileInfo.x,
|
||||
tileInfo.y,
|
||||
tileInfo.center,
|
||||
tileInfo.tileSizeMeters,
|
||||
zoomLevel,
|
||||
currentToken,
|
||||
tileIndex,
|
||||
tilesToDownload.Count,
|
||||
token);
|
||||
|
||||
downloadTasks.Add(downloadTask);
|
||||
}
|
||||
|
||||
var results = await Task.WhenAll(downloadTasks);
|
||||
|
||||
var downloadedTiles = results.Where(r => r != null).Cast<DownloadedTileInfoV2>().ToList();
|
||||
|
||||
return downloadedTiles;
|
||||
}
|
||||
|
||||
private async Task<DownloadedTileInfoV2?> DownloadTileAsync(
|
||||
int x,
|
||||
int y,
|
||||
GeoPoint tileCenter,
|
||||
double tileSizeMeters,
|
||||
int zoomLevel,
|
||||
string? sessionToken,
|
||||
int tileIndex,
|
||||
int totalTiles,
|
||||
CancellationToken token)
|
||||
{
|
||||
var tileKey = $"{zoomLevel}_{x}_{y}";
|
||||
|
||||
var downloadTask = _activeDownloads.GetOrAdd(tileKey, _ => PerformDownloadAsync(
|
||||
x, y, tileCenter, tileSizeMeters, zoomLevel, sessionToken, tileIndex, totalTiles, token));
|
||||
|
||||
try
|
||||
{
|
||||
return await downloadTask;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_activeDownloads.TryRemove(tileKey, out _);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<DownloadedTileInfoV2> PerformDownloadAsync(
|
||||
int x,
|
||||
int y,
|
||||
GeoPoint tileCenter,
|
||||
double tileSizeMeters,
|
||||
int zoomLevel,
|
||||
string? sessionToken,
|
||||
int tileIndex,
|
||||
int totalTiles,
|
||||
CancellationToken token)
|
||||
{
|
||||
await _downloadSemaphore.WaitAsync(token);
|
||||
|
||||
try
|
||||
{
|
||||
if (_processingConfig.DelayBetweenRequestsMs > 0)
|
||||
{
|
||||
await Task.Delay(_processingConfig.DelayBetweenRequestsMs, token);
|
||||
}
|
||||
|
||||
var server = (x + y) % 4;
|
||||
var url = string.Format(TILE_URL_TEMPLATE, server, x, y, zoomLevel, sessionToken);
|
||||
|
||||
var timestamp = DateTime.UtcNow.ToString("yyyyMMddHHmmss");
|
||||
var subdirectory = _storageConfig.GetTileSubdirectoryPath(zoomLevel, x, y);
|
||||
Directory.CreateDirectory(subdirectory);
|
||||
|
||||
var filePath = _storageConfig.GetTileFilePath(zoomLevel, x, y, timestamp);
|
||||
|
||||
var imageBytes = await ExecuteWithRetryAsync(async () =>
|
||||
{
|
||||
using var httpClient = _httpClientFactory.CreateClient();
|
||||
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(USER_AGENT);
|
||||
|
||||
var response = await httpClient.GetAsync(url, token);
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
var errorBody = await response.Content.ReadAsStringAsync(token);
|
||||
_logger.LogError("Tile download failed. Tile: ({X}, {Y}), Status: {StatusCode}, Response: {Response}",
|
||||
x, y, response.StatusCode, errorBody);
|
||||
}
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
return await response.Content.ReadAsByteArrayAsync(token);
|
||||
}, cancellationToken: token);
|
||||
|
||||
await File.WriteAllBytesAsync(filePath, imageBytes, token);
|
||||
|
||||
return new DownloadedTileInfoV2(
|
||||
x, y, zoomLevel, tileCenter.Lat, tileCenter.Lon, filePath, tileSizeMeters);
|
||||
}
|
||||
catch (TaskCanceledException ex)
|
||||
{
|
||||
_logger.LogError(ex, "Tile download cancelled for ({X}, {Y})", x, y);
|
||||
throw;
|
||||
}
|
||||
catch (OperationCanceledException ex)
|
||||
{
|
||||
_logger.LogError(ex, "Tile download operation cancelled for ({X}, {Y})", x, y);
|
||||
throw;
|
||||
}
|
||||
catch (RateLimitException ex)
|
||||
{
|
||||
_logger.LogError(ex, "Rate limit exceeded for tile ({X}, {Y})", x, y);
|
||||
throw;
|
||||
}
|
||||
catch (HttpRequestException ex)
|
||||
{
|
||||
_logger.LogError(ex, "HTTP request failed for tile ({X}, {Y}). StatusCode: {StatusCode}",
|
||||
x, y, ex.StatusCode);
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Unexpected error downloading tile ({X}, {Y})", x, y);
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_downloadSemaphore.Release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user