mirror of
https://github.com/azaion/satellite-provider.git
synced 2026-04-23 01:26:38 +00:00
improve retries
This commit is contained in:
Anton Martynenko
@@ -91,6 +91,14 @@ public class RegionService : IRegionService
|
||||
region.UpdatedAt = DateTime.UtcNow;
|
||||
await _regionRepository.UpdateAsync(region);
|
||||
|
||||
using var timeoutCts = new CancellationTokenSource(TimeSpan.FromMinutes(5));
|
||||
using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCts.Token);
|
||||
|
||||
string? errorMessage = null;
|
||||
List<TileMetadata>? tiles = null;
|
||||
int tilesDownloaded = 0;
|
||||
int tilesReused = 0;
|
||||
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("Downloading tiles for region {RegionId} at ({Lat}, {Lon}) size {Size}m zoom {Zoom}",
|
||||
@@ -108,15 +116,15 @@ public class RegionService : IRegionService
|
||||
_logger.LogInformation("Found {Count} existing tiles for region {RegionId}", existingTileIds.Count, id);
|
||||
|
||||
_logger.LogInformation("Starting tile download for region {RegionId}", id);
|
||||
var tiles = await _tileService.DownloadAndStoreTilesAsync(
|
||||
tiles = await _tileService.DownloadAndStoreTilesAsync(
|
||||
region.Latitude,
|
||||
region.Longitude,
|
||||
region.SizeMeters,
|
||||
region.ZoomLevel,
|
||||
cancellationToken);
|
||||
linkedCts.Token);
|
||||
|
||||
var tilesDownloaded = tiles.Count(t => !existingTileIds.Contains(t.Id));
|
||||
var tilesReused = tiles.Count(t => existingTileIds.Contains(t.Id));
|
||||
tilesDownloaded = tiles.Count(t => !existingTileIds.Contains(t.Id));
|
||||
tilesReused = tiles.Count(t => existingTileIds.Contains(t.Id));
|
||||
|
||||
_logger.LogInformation("Region {RegionId}: Downloaded {Downloaded} tiles, Reused {Reused} tiles",
|
||||
id, tilesDownloaded, tilesReused);
|
||||
@@ -128,12 +136,12 @@ public class RegionService : IRegionService
|
||||
var summaryPath = Path.Combine(readyDir, $"region_{id}_summary.txt");
|
||||
var stitchedImagePath = Path.Combine(readyDir, $"region_{id}_stitched.jpg");
|
||||
|
||||
await GenerateCsvFileAsync(csvPath, tiles, cancellationToken);
|
||||
await GenerateCsvFileAsync(csvPath, tiles, linkedCts.Token);
|
||||
|
||||
_logger.LogInformation("Stitching tiles for region {RegionId}", id);
|
||||
await StitchTilesAsync(tiles, region.Latitude, region.Longitude, region.ZoomLevel, stitchedImagePath, cancellationToken);
|
||||
await StitchTilesAsync(tiles, region.Latitude, region.Longitude, region.ZoomLevel, stitchedImagePath, linkedCts.Token);
|
||||
|
||||
await GenerateSummaryFileAsync(summaryPath, id, region, tiles, tilesDownloaded, tilesReused, stitchedImagePath, processingStartTime, cancellationToken);
|
||||
await GenerateSummaryFileAsync(summaryPath, id, region, tiles, tilesDownloaded, tilesReused, stitchedImagePath, processingStartTime, linkedCts.Token, errorMessage);
|
||||
|
||||
region.Status = "completed";
|
||||
region.CsvFilePath = csvPath;
|
||||
@@ -146,15 +154,72 @@ public class RegionService : IRegionService
|
||||
var duration = (DateTime.UtcNow - startTime).TotalSeconds;
|
||||
_logger.LogInformation("Region {RegionId} processing completed in {Duration:F2}s", id, duration);
|
||||
}
|
||||
catch (OperationCanceledException) when (timeoutCts.IsCancellationRequested)
|
||||
{
|
||||
errorMessage = "Processing timed out after 5 minutes. Unable to download tiles within the time limit.";
|
||||
_logger.LogError("Region {RegionId} processing timed out after 5 minutes", id);
|
||||
await HandleProcessingFailureAsync(id, region, startTime, tiles, tilesDownloaded, tilesReused, errorMessage);
|
||||
}
|
||||
catch (RateLimitException ex)
|
||||
{
|
||||
errorMessage = $"Rate limit exceeded: {ex.Message}. Google Maps API rate limit was reached and retries were exhausted.";
|
||||
_logger.LogError(ex, "Rate limit exceeded for region {RegionId}", id);
|
||||
await HandleProcessingFailureAsync(id, region, startTime, tiles, tilesDownloaded, tilesReused, errorMessage);
|
||||
}
|
||||
catch (HttpRequestException ex)
|
||||
{
|
||||
errorMessage = $"Network error: {ex.Message}. Failed to download tiles from Google Maps.";
|
||||
_logger.LogError(ex, "Network error processing region {RegionId}", id);
|
||||
await HandleProcessingFailureAsync(id, region, startTime, tiles, tilesDownloaded, tilesReused, errorMessage);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
errorMessage = $"Unexpected error: {ex.Message}";
|
||||
_logger.LogError(ex, "Failed to process region {RegionId}: {Message}", id, ex.Message);
|
||||
region.Status = "failed";
|
||||
region.UpdatedAt = DateTime.UtcNow;
|
||||
await _regionRepository.UpdateAsync(region);
|
||||
await HandleProcessingFailureAsync(id, region, startTime, tiles, tilesDownloaded, tilesReused, errorMessage);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task HandleProcessingFailureAsync(
|
||||
Guid id,
|
||||
RegionEntity region,
|
||||
DateTime startTime,
|
||||
List<TileMetadata>? tiles,
|
||||
int tilesDownloaded,
|
||||
int tilesReused,
|
||||
string errorMessage)
|
||||
{
|
||||
region.Status = "failed";
|
||||
region.UpdatedAt = DateTime.UtcNow;
|
||||
|
||||
try
|
||||
{
|
||||
var readyDir = _storageConfig.ReadyDirectory;
|
||||
Directory.CreateDirectory(readyDir);
|
||||
|
||||
var summaryPath = Path.Combine(readyDir, $"region_{id}_summary.txt");
|
||||
region.SummaryFilePath = summaryPath;
|
||||
|
||||
await GenerateSummaryFileAsync(
|
||||
summaryPath,
|
||||
id,
|
||||
region,
|
||||
tiles ?? new List<TileMetadata>(),
|
||||
tilesDownloaded,
|
||||
tilesReused,
|
||||
null,
|
||||
startTime,
|
||||
CancellationToken.None,
|
||||
errorMessage);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to generate error summary for region {RegionId}", id);
|
||||
}
|
||||
|
||||
await _regionRepository.UpdateAsync(region);
|
||||
}
|
||||
|
||||
private async Task<string> StitchTilesAsync(
|
||||
List<TileMetadata> tiles,
|
||||
double centerLatitude,
|
||||
@@ -265,9 +330,10 @@ public class RegionService : IRegionService
|
||||
List<TileMetadata> tiles,
|
||||
int tilesDownloaded,
|
||||
int tilesReused,
|
||||
string stitchedImagePath,
|
||||
string? stitchedImagePath,
|
||||
DateTime startTime,
|
||||
CancellationToken cancellationToken)
|
||||
CancellationToken cancellationToken,
|
||||
string? errorMessage = null)
|
||||
{
|
||||
var endTime = DateTime.UtcNow;
|
||||
var processingTime = (endTime - startTime).TotalSeconds;
|
||||
@@ -279,19 +345,46 @@ public class RegionService : IRegionService
|
||||
summary.AppendLine($"Center: {region.Latitude:F6}, {region.Longitude:F6}");
|
||||
summary.AppendLine($"Size: {region.SizeMeters:F0} meters");
|
||||
summary.AppendLine($"Zoom Level: {region.ZoomLevel}");
|
||||
summary.AppendLine($"Status: {region.Status}");
|
||||
summary.AppendLine();
|
||||
|
||||
if (!string.IsNullOrEmpty(errorMessage))
|
||||
{
|
||||
summary.AppendLine("ERROR:");
|
||||
summary.AppendLine(errorMessage);
|
||||
summary.AppendLine();
|
||||
}
|
||||
|
||||
summary.AppendLine("Processing Statistics:");
|
||||
summary.AppendLine($"- Tiles Downloaded: {tilesDownloaded}");
|
||||
summary.AppendLine($"- Tiles Reused from Cache: {tilesReused}");
|
||||
summary.AppendLine($"- Total Tiles: {tiles.Count}");
|
||||
summary.AppendLine($"- Processing Time: {processingTime:F2} seconds");
|
||||
summary.AppendLine($"- Started: {startTime:yyyy-MM-dd HH:mm:ss} UTC");
|
||||
summary.AppendLine($"- Completed: {endTime:yyyy-MM-dd HH:mm:ss} UTC");
|
||||
|
||||
if (region.Status == "completed")
|
||||
{
|
||||
summary.AppendLine($"- Completed: {endTime:yyyy-MM-dd HH:mm:ss} UTC");
|
||||
}
|
||||
else
|
||||
{
|
||||
summary.AppendLine($"- Failed: {endTime:yyyy-MM-dd HH:mm:ss} UTC");
|
||||
}
|
||||
|
||||
summary.AppendLine();
|
||||
summary.AppendLine("Files Created:");
|
||||
summary.AppendLine($"- CSV: {Path.GetFileName(filePath).Replace("_summary.txt", "_ready.csv")}");
|
||||
summary.AppendLine($"- Stitched Image: {Path.GetFileName(stitchedImagePath)}");
|
||||
summary.AppendLine($"- Stitched Image Path: {stitchedImagePath}");
|
||||
|
||||
if (tiles.Count > 0)
|
||||
{
|
||||
summary.AppendLine($"- CSV: {Path.GetFileName(filePath).Replace("_summary.txt", "_ready.csv")}");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(stitchedImagePath))
|
||||
{
|
||||
summary.AppendLine($"- Stitched Image: {Path.GetFileName(stitchedImagePath)}");
|
||||
summary.AppendLine($"- Stitched Image Path: {stitchedImagePath}");
|
||||
}
|
||||
|
||||
summary.AppendLine($"- Summary: {Path.GetFileName(filePath)}");
|
||||
|
||||
await File.WriteAllTextAsync(filePath, summary.ToString(), cancellationToken);
|
||||
|
||||
Reference in New Issue
Block a user