mirror of
https://github.com/azaion/satellite-provider.git
synced 2026-04-22 05:26:39 +00:00
tiels are cached and reused properly
This commit is contained in:
@@ -5,6 +5,7 @@ obj
|
|||||||
*.DotSettings*
|
*.DotSettings*
|
||||||
*.user
|
*.user
|
||||||
log*
|
log*
|
||||||
|
logs/
|
||||||
Content/
|
Content/
|
||||||
.env
|
.env
|
||||||
tiles/
|
tiles/
|
||||||
|
|||||||
@@ -0,0 +1,376 @@
|
|||||||
|
# Satellite Provider API
|
||||||
|
|
||||||
|
A RESTful API service for downloading, caching, and processing satellite imagery from Google Maps. The service provides endpoints for downloading individual tiles and processing entire regions with intelligent tile caching and versioning.
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
- **Satellite Tile Downloads**: Download individual satellite tiles from Google Maps at various zoom levels
|
||||||
|
- **Region Processing**: Request satellite imagery for entire geographic regions
|
||||||
|
- **Intelligent Tile Caching**: Automatic tile reuse with year-based versioning
|
||||||
|
- **Image Stitching**: Combine multiple tiles into seamless regional images
|
||||||
|
- **CSV Export**: Generate coordinate mappings for downloaded tiles
|
||||||
|
- **Summary Reports**: Detailed processing statistics and metadata
|
||||||
|
|
||||||
|
## Tech Stack
|
||||||
|
|
||||||
|
- **.NET 8.0** - ASP.NET Web API
|
||||||
|
- **PostgreSQL** - Primary data storage
|
||||||
|
- **Dapper** - Lightweight ORM for database access
|
||||||
|
- **DbUp** - Database migrations
|
||||||
|
- **Serilog** - Structured logging
|
||||||
|
- **ImageSharp** - Image processing and stitching
|
||||||
|
- **Docker** - Containerized deployment
|
||||||
|
|
||||||
|
## Tile Versioning Strategy
|
||||||
|
|
||||||
|
### Year-Based Tile Versions
|
||||||
|
|
||||||
|
The service implements a **year-based versioning strategy** for cached satellite tiles. This approach is based on the assumption that Google Maps satellite imagery updates infrequently, typically once per year or less.
|
||||||
|
|
||||||
|
#### How It Works
|
||||||
|
|
||||||
|
1. **Version Assignment**: When a tile is downloaded, it is assigned a version number equal to the current year (e.g., 2025).
|
||||||
|
|
||||||
|
2. **Unique Constraint**: Tiles are uniquely identified by the combination of:
|
||||||
|
- Latitude (center point)
|
||||||
|
- Longitude (center point)
|
||||||
|
- Zoom level
|
||||||
|
- Tile size in meters
|
||||||
|
- **Version** (year)
|
||||||
|
|
||||||
|
3. **Cache Behavior**:
|
||||||
|
- **Same Year**: If requesting a tile that was downloaded in the current year, the cached tile is reused
|
||||||
|
- **New Year**: When the calendar year changes, new tiles are downloaded even for the same coordinates
|
||||||
|
- **Historical Data**: Old tile versions remain in the database for historical reference
|
||||||
|
|
||||||
|
#### Example
|
||||||
|
|
||||||
|
```
|
||||||
|
Request in 2025:
|
||||||
|
- Downloads tile at (47.461747, 37.647063) zoom 18 → version = 2025
|
||||||
|
- Saves to database with version 2025
|
||||||
|
|
||||||
|
Second request in 2025:
|
||||||
|
- Finds tile at (47.461747, 37.647063) zoom 18 version 2025
|
||||||
|
- Reuses cached tile (no download)
|
||||||
|
|
||||||
|
Request in 2026:
|
||||||
|
- Looks for tile at (47.461747, 37.647063) zoom 18 version 2026
|
||||||
|
- Not found (only 2025 version exists)
|
||||||
|
- Downloads new tile → version = 2026
|
||||||
|
- Both 2025 and 2026 versions now exist in database
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Benefits
|
||||||
|
|
||||||
|
- **Automatic Updates**: Ensures fresh satellite imagery each calendar year
|
||||||
|
- **Historical Archive**: Maintains previous years' imagery for comparison
|
||||||
|
- **Efficient Caching**: Maximizes cache hit rate within the same year
|
||||||
|
- **Simple Logic**: No complex timestamp comparisons or manual cache invalidation
|
||||||
|
|
||||||
|
#### Database Schema
|
||||||
|
|
||||||
|
```sql
|
||||||
|
CREATE TABLE tiles (
|
||||||
|
id UUID PRIMARY KEY,
|
||||||
|
zoom_level INT NOT NULL,
|
||||||
|
latitude DOUBLE PRECISION NOT NULL,
|
||||||
|
longitude DOUBLE PRECISION NOT NULL,
|
||||||
|
tile_size_meters DOUBLE PRECISION NOT NULL,
|
||||||
|
tile_size_pixels INT NOT NULL,
|
||||||
|
image_type VARCHAR(10) NOT NULL,
|
||||||
|
maps_version VARCHAR(50),
|
||||||
|
version INT NOT NULL, -- Year-based version
|
||||||
|
file_path VARCHAR(500) NOT NULL,
|
||||||
|
created_at TIMESTAMP NOT NULL,
|
||||||
|
updated_at TIMESTAMP NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX idx_tiles_unique_location
|
||||||
|
ON tiles(latitude, longitude, zoom_level, tile_size_meters, version);
|
||||||
|
```
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
### Project Structure
|
||||||
|
|
||||||
|
```
|
||||||
|
SatelliteProvider/
|
||||||
|
├── SatelliteProvider.Api/ # REST API endpoints
|
||||||
|
├── SatelliteProvider.Common/ # Shared DTOs and interfaces
|
||||||
|
├── SatelliteProvider.DataAccess/ # Database access and migrations
|
||||||
|
├── SatelliteProvider.Services/ # Business logic
|
||||||
|
├── SatelliteProvider.Tests/ # Unit tests
|
||||||
|
└── SatelliteProvider.IntegrationTests/ # Integration tests
|
||||||
|
```
|
||||||
|
|
||||||
|
### Key Components
|
||||||
|
|
||||||
|
- **GoogleMapsDownloaderV2**: Downloads satellite tiles from Google Maps API
|
||||||
|
- **TileService**: Manages tile caching, retrieval, and storage with version control
|
||||||
|
- **RegionService**: Processes region requests, stitches tiles, generates outputs
|
||||||
|
- **RegionRequestQueue**: Background processing queue for region requests
|
||||||
|
- **DatabaseMigrator**: Manages database schema migrations
|
||||||
|
|
||||||
|
## API Endpoints
|
||||||
|
|
||||||
|
### Download Single Tile
|
||||||
|
|
||||||
|
```http
|
||||||
|
POST /api/satellite/tiles/download
|
||||||
|
Content-Type: application/json
|
||||||
|
|
||||||
|
{
|
||||||
|
"latitude": 47.461747,
|
||||||
|
"longitude": 37.647063,
|
||||||
|
"zoomLevel": 18
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response:**
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"id": "550e8400-e29b-41d4-a716-446655440000",
|
||||||
|
"zoomLevel": 18,
|
||||||
|
"latitude": 47.462451,
|
||||||
|
"longitude": 37.646027,
|
||||||
|
"tileSizeMeters": 103.35,
|
||||||
|
"tileSizePixels": 256,
|
||||||
|
"imageType": "jpg",
|
||||||
|
"mapsVersion": "downloaded_2025-10-29",
|
||||||
|
"version": 2025,
|
||||||
|
"filePath": "./tiles/tile_18_158485_91707_20251029103256.jpg",
|
||||||
|
"createdAt": "2025-10-29T10:32:56Z",
|
||||||
|
"updatedAt": "2025-10-29T10:32:56Z"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Request Region Processing
|
||||||
|
|
||||||
|
```http
|
||||||
|
POST /api/satellite/request
|
||||||
|
Content-Type: application/json
|
||||||
|
|
||||||
|
{
|
||||||
|
"id": "3fa85f64-5717-4562-b3fc-2c963f66afa6",
|
||||||
|
"latitude": 47.461747,
|
||||||
|
"longitude": 37.647063,
|
||||||
|
"sizeMeters": 200,
|
||||||
|
"zoomLevel": 18
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Check Region Status
|
||||||
|
|
||||||
|
```http
|
||||||
|
GET /api/satellite/region/{regionId}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response:**
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"id": "3fa85f64-5717-4562-b3fc-2c963f66afa6",
|
||||||
|
"status": "completed",
|
||||||
|
"csvFilePath": "./ready/region_{id}_ready.csv",
|
||||||
|
"summaryFilePath": "./ready/region_{id}_summary.txt",
|
||||||
|
"tilesDownloaded": 5,
|
||||||
|
"tilesReused": 4,
|
||||||
|
"createdAt": "2025-10-29T10:32:56Z",
|
||||||
|
"updatedAt": "2025-10-29T10:32:57Z"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
### Environment Variables
|
||||||
|
|
||||||
|
- `ASPNETCORE_ENVIRONMENT`: Development/Production
|
||||||
|
- `ASPNETCORE_URLS`: HTTP binding address (default: http://+:8080)
|
||||||
|
- `ConnectionStrings__DefaultConnection`: PostgreSQL connection string
|
||||||
|
- `MapConfig__ApiKey`: Google Maps API key
|
||||||
|
|
||||||
|
### appsettings.json
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"Serilog": {
|
||||||
|
"MinimumLevel": {
|
||||||
|
"Default": "Information",
|
||||||
|
"Override": {
|
||||||
|
"Microsoft.AspNetCore": "Warning"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"WriteTo": [
|
||||||
|
{ "Name": "Console" },
|
||||||
|
{
|
||||||
|
"Name": "File",
|
||||||
|
"Args": {
|
||||||
|
"path": "./logs/satellite-provider-.log",
|
||||||
|
"rollingInterval": "Day"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"StorageConfig": {
|
||||||
|
"TilesDirectory": "./tiles",
|
||||||
|
"ReadyDirectory": "./ready"
|
||||||
|
},
|
||||||
|
"ProcessingConfig": {
|
||||||
|
"MaxConcurrentDownloads": 4,
|
||||||
|
"DefaultZoomLevel": 18,
|
||||||
|
"QueueCapacity": 100
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Running with Docker
|
||||||
|
|
||||||
|
### Prerequisites
|
||||||
|
|
||||||
|
- Docker and Docker Compose
|
||||||
|
- Google Maps API key with Tile API enabled
|
||||||
|
|
||||||
|
### Setup
|
||||||
|
|
||||||
|
1. Create `.env` file in project root:
|
||||||
|
```bash
|
||||||
|
GOOGLE_MAPS_API_KEY=your_api_key_here
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Start services:
|
||||||
|
```bash
|
||||||
|
docker-compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
3. View logs:
|
||||||
|
```bash
|
||||||
|
docker-compose logs -f api
|
||||||
|
```
|
||||||
|
|
||||||
|
4. Stop services:
|
||||||
|
```bash
|
||||||
|
docker-compose down
|
||||||
|
```
|
||||||
|
|
||||||
|
### Docker Compose Services
|
||||||
|
|
||||||
|
- **postgres**: PostgreSQL database server
|
||||||
|
- **api**: Satellite Provider API service
|
||||||
|
|
||||||
|
### Volume Mounts
|
||||||
|
|
||||||
|
- `./tiles`: Downloaded tile images
|
||||||
|
- `./ready`: Processed region outputs (CSV, stitched images, summaries)
|
||||||
|
- `./logs`: Application logs
|
||||||
|
|
||||||
|
## Development
|
||||||
|
|
||||||
|
### Local Development Setup
|
||||||
|
|
||||||
|
1. Install .NET 8.0 SDK
|
||||||
|
2. Install PostgreSQL 16
|
||||||
|
3. Set up database:
|
||||||
|
```bash
|
||||||
|
createdb satelliteprovider
|
||||||
|
```
|
||||||
|
|
||||||
|
4. Update connection string in `appsettings.Development.json`
|
||||||
|
5. Run migrations (automatic on startup)
|
||||||
|
6. Start API:
|
||||||
|
```bash
|
||||||
|
cd SatelliteProvider.Api
|
||||||
|
dotnet run
|
||||||
|
```
|
||||||
|
|
||||||
|
### Running Tests
|
||||||
|
|
||||||
|
```bash
|
||||||
|
dotnet test
|
||||||
|
```
|
||||||
|
|
||||||
|
### Integration Tests
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker-compose -f docker-compose.tests.yml up --build
|
||||||
|
```
|
||||||
|
|
||||||
|
## Output Files
|
||||||
|
|
||||||
|
### Region Processing Outputs
|
||||||
|
|
||||||
|
For each processed region, the service generates:
|
||||||
|
|
||||||
|
1. **CSV File** (`region_{id}_ready.csv`):
|
||||||
|
- Tile coordinates and file paths
|
||||||
|
- Ordered by latitude (descending) and longitude (ascending)
|
||||||
|
|
||||||
|
2. **Stitched Image** (`region_{id}_stitched.jpg`):
|
||||||
|
- Combined satellite imagery for the entire region
|
||||||
|
- Red crosshair marking the requested center point
|
||||||
|
|
||||||
|
3. **Summary Report** (`region_{id}_summary.txt`):
|
||||||
|
- Region metadata and coordinates
|
||||||
|
- Processing statistics (tiles downloaded, tiles reused, total tiles)
|
||||||
|
- Processing time and timestamps
|
||||||
|
- File paths for all generated outputs
|
||||||
|
|
||||||
|
### Example Summary Report
|
||||||
|
|
||||||
|
```
|
||||||
|
Region Processing Summary
|
||||||
|
========================
|
||||||
|
Region ID: 5d218e0d-92bc-483c-9e88-fd79200b84e6
|
||||||
|
Center: 47.461747, 37.647063
|
||||||
|
Size: 200 meters
|
||||||
|
Zoom Level: 18
|
||||||
|
|
||||||
|
Processing Statistics:
|
||||||
|
- Tiles Downloaded: 5
|
||||||
|
- Tiles Reused from Cache: 4
|
||||||
|
- Total Tiles: 9
|
||||||
|
- Processing Time: 0.50 seconds
|
||||||
|
- Started: 2025-10-29 10:32:56 UTC
|
||||||
|
- Completed: 2025-10-29 10:32:57 UTC
|
||||||
|
|
||||||
|
Files Created:
|
||||||
|
- CSV: region_5d218e0d-92bc-483c-9e88-fd79200b84e6_ready.csv
|
||||||
|
- Stitched Image: region_5d218e0d-92bc-483c-9e88-fd79200b84e6_stitched.jpg
|
||||||
|
- Summary: region_5d218e0d-92bc-483c-9e88-fd79200b84e6_summary.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
## Zoom Levels
|
||||||
|
|
||||||
|
Supported zoom levels: **15, 16, 17, 18, 19**
|
||||||
|
|
||||||
|
- **Zoom 15**: ~2,600m per tile (larger areas, less detail)
|
||||||
|
- **Zoom 16**: ~1,300m per tile
|
||||||
|
- **Zoom 17**: ~650m per tile
|
||||||
|
- **Zoom 18**: ~103m per tile (recommended for most uses)
|
||||||
|
- **Zoom 19**: ~52m per tile (maximum detail)
|
||||||
|
|
||||||
|
## Performance Considerations
|
||||||
|
|
||||||
|
### Tile Caching
|
||||||
|
|
||||||
|
- First request for a region downloads all required tiles
|
||||||
|
- Subsequent requests (same year) reuse cached tiles
|
||||||
|
- Typical 3x3 region (9 tiles) at zoom 18: ~300ms first request, ~50ms cached
|
||||||
|
|
||||||
|
### Region Processing
|
||||||
|
|
||||||
|
- Processed asynchronously in background queue
|
||||||
|
- Status polling recommended at 1-second intervals
|
||||||
|
- Processing time depends on:
|
||||||
|
- Number of tiles required
|
||||||
|
- Cache hit rate
|
||||||
|
- Network latency to Google Maps API
|
||||||
|
- Image stitching complexity
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
This project is proprietary software. All rights reserved.
|
||||||
|
|
||||||
|
## Support
|
||||||
|
|
||||||
|
For issues, questions, or feature requests, please contact the development team.
|
||||||
|
|
||||||
@@ -9,9 +9,13 @@ using SatelliteProvider.Common.Configs;
|
|||||||
using SatelliteProvider.Common.DTO;
|
using SatelliteProvider.Common.DTO;
|
||||||
using SatelliteProvider.Common.Interfaces;
|
using SatelliteProvider.Common.Interfaces;
|
||||||
using SatelliteProvider.Services;
|
using SatelliteProvider.Services;
|
||||||
|
using Serilog;
|
||||||
|
|
||||||
var builder = WebApplication.CreateBuilder(args);
|
var builder = WebApplication.CreateBuilder(args);
|
||||||
|
|
||||||
|
builder.Host.UseSerilog((context, configuration) =>
|
||||||
|
configuration.ReadFrom.Configuration(context.Configuration));
|
||||||
|
|
||||||
var connectionString = builder.Configuration.GetConnectionString("DefaultConnection")
|
var connectionString = builder.Configuration.GetConnectionString("DefaultConnection")
|
||||||
?? throw new InvalidOperationException("Connection string 'DefaultConnection' not found.");
|
?? throw new InvalidOperationException("Connection string 'DefaultConnection' not found.");
|
||||||
|
|
||||||
@@ -130,6 +134,7 @@ async Task<IResult> DownloadSingleTile([FromBody] DownloadTileRequest request, G
|
|||||||
request.ZoomLevel);
|
request.ZoomLevel);
|
||||||
|
|
||||||
var now = DateTime.UtcNow;
|
var now = DateTime.UtcNow;
|
||||||
|
var currentVersion = now.Year;
|
||||||
var tileEntity = new TileEntity
|
var tileEntity = new TileEntity
|
||||||
{
|
{
|
||||||
Id = Guid.NewGuid(),
|
Id = Guid.NewGuid(),
|
||||||
@@ -140,6 +145,7 @@ async Task<IResult> DownloadSingleTile([FromBody] DownloadTileRequest request, G
|
|||||||
TileSizePixels = 256,
|
TileSizePixels = 256,
|
||||||
ImageType = "jpg",
|
ImageType = "jpg",
|
||||||
MapsVersion = $"downloaded_{now:yyyy-MM-dd}",
|
MapsVersion = $"downloaded_{now:yyyy-MM-dd}",
|
||||||
|
Version = currentVersion,
|
||||||
FilePath = downloadedTile.FilePath,
|
FilePath = downloadedTile.FilePath,
|
||||||
CreatedAt = now,
|
CreatedAt = now,
|
||||||
UpdatedAt = now
|
UpdatedAt = now
|
||||||
@@ -158,6 +164,7 @@ async Task<IResult> DownloadSingleTile([FromBody] DownloadTileRequest request, G
|
|||||||
TileSizePixels = tileEntity.TileSizePixels,
|
TileSizePixels = tileEntity.TileSizePixels,
|
||||||
ImageType = tileEntity.ImageType,
|
ImageType = tileEntity.ImageType,
|
||||||
MapsVersion = tileEntity.MapsVersion,
|
MapsVersion = tileEntity.MapsVersion,
|
||||||
|
Version = currentVersion,
|
||||||
FilePath = tileEntity.FilePath,
|
FilePath = tileEntity.FilePath,
|
||||||
CreatedAt = tileEntity.CreatedAt,
|
CreatedAt = tileEntity.CreatedAt,
|
||||||
UpdatedAt = tileEntity.UpdatedAt
|
UpdatedAt = tileEntity.UpdatedAt
|
||||||
@@ -289,6 +296,7 @@ public record DownloadTileResponse
|
|||||||
public int TileSizePixels { get; set; }
|
public int TileSizePixels { get; set; }
|
||||||
public string ImageType { get; set; } = string.Empty;
|
public string ImageType { get; set; } = string.Empty;
|
||||||
public string? MapsVersion { get; set; }
|
public string? MapsVersion { get; set; }
|
||||||
|
public int Version { get; set; }
|
||||||
public string FilePath { get; set; } = string.Empty;
|
public string FilePath { get; set; } = string.Empty;
|
||||||
public DateTime CreatedAt { get; set; }
|
public DateTime CreatedAt { get; set; }
|
||||||
public DateTime UpdatedAt { get; set; }
|
public DateTime UpdatedAt { get; set; }
|
||||||
|
|||||||
@@ -9,6 +9,8 @@
|
|||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="8.0.21"/>
|
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="8.0.21"/>
|
||||||
<PackageReference Include="Newtonsoft.Json" Version="13.0.4" />
|
<PackageReference Include="Newtonsoft.Json" Version="13.0.4" />
|
||||||
|
<PackageReference Include="Serilog.AspNetCore" Version="8.0.3" />
|
||||||
|
<PackageReference Include="Serilog.Sinks.File" Version="6.0.0" />
|
||||||
<PackageReference Include="SixLabors.ImageSharp" Version="3.1.11" />
|
<PackageReference Include="SixLabors.ImageSharp" Version="3.1.11" />
|
||||||
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2"/>
|
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2"/>
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|||||||
@@ -1,8 +1,10 @@
|
|||||||
{
|
{
|
||||||
"Logging": {
|
"Serilog": {
|
||||||
"LogLevel": {
|
"MinimumLevel": {
|
||||||
"Default": "Information",
|
"Default": "Information",
|
||||||
"Microsoft.AspNetCore": "Warning"
|
"Override": {
|
||||||
|
"Microsoft.AspNetCore": "Warning"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"ConnectionStrings": {
|
"ConnectionStrings": {
|
||||||
|
|||||||
@@ -1,9 +1,23 @@
|
|||||||
{
|
{
|
||||||
"Logging": {
|
"Serilog": {
|
||||||
"LogLevel": {
|
"Using": [ "Serilog.Sinks.Console", "Serilog.Sinks.File" ],
|
||||||
|
"MinimumLevel": {
|
||||||
"Default": "Information",
|
"Default": "Information",
|
||||||
"Microsoft.AspNetCore": "Warning"
|
"Override": {
|
||||||
}
|
"Microsoft.AspNetCore": "Warning"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"WriteTo": [
|
||||||
|
{ "Name": "Console" },
|
||||||
|
{
|
||||||
|
"Name": "File",
|
||||||
|
"Args": {
|
||||||
|
"path": "./logs/satellite-provider-.log",
|
||||||
|
"rollingInterval": "Day",
|
||||||
|
"outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] {Message:lj}{NewLine}{Exception}"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
},
|
},
|
||||||
"AllowedHosts": "*",
|
"AllowedHosts": "*",
|
||||||
"ConnectionStrings": {
|
"ConnectionStrings": {
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ public class TileMetadata
|
|||||||
public int TileSizePixels { get; set; }
|
public int TileSizePixels { get; set; }
|
||||||
public string ImageType { get; set; } = string.Empty;
|
public string ImageType { get; set; } = string.Empty;
|
||||||
public string? MapsVersion { get; set; }
|
public string? MapsVersion { get; set; }
|
||||||
|
public int Version { get; set; }
|
||||||
public string FilePath { get; set; } = string.Empty;
|
public string FilePath { get; set; } = string.Empty;
|
||||||
public DateTime CreatedAt { get; set; }
|
public DateTime CreatedAt { get; set; }
|
||||||
public DateTime UpdatedAt { get; set; }
|
public DateTime UpdatedAt { get; set; }
|
||||||
|
|||||||
@@ -0,0 +1,13 @@
|
|||||||
|
ALTER TABLE tiles ADD COLUMN version INT NOT NULL DEFAULT 2025;
|
||||||
|
|
||||||
|
DELETE FROM tiles a
|
||||||
|
USING tiles b
|
||||||
|
WHERE a.id > b.id
|
||||||
|
AND a.latitude = b.latitude
|
||||||
|
AND a.longitude = b.longitude
|
||||||
|
AND a.zoom_level = b.zoom_level
|
||||||
|
AND a.tile_size_meters = b.tile_size_meters
|
||||||
|
AND a.version = b.version;
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX idx_tiles_unique_location ON tiles(latitude, longitude, zoom_level, tile_size_meters, version);
|
||||||
|
|
||||||
@@ -10,6 +10,7 @@ public class TileEntity
|
|||||||
public int TileSizePixels { get; set; }
|
public int TileSizePixels { get; set; }
|
||||||
public string ImageType { get; set; } = string.Empty;
|
public string ImageType { get; set; } = string.Empty;
|
||||||
public string? MapsVersion { get; set; }
|
public string? MapsVersion { get; set; }
|
||||||
|
public int Version { get; set; }
|
||||||
public string FilePath { get; set; } = string.Empty;
|
public string FilePath { get; set; } = string.Empty;
|
||||||
public DateTime CreatedAt { get; set; }
|
public DateTime CreatedAt { get; set; }
|
||||||
public DateTime UpdatedAt { get; set; }
|
public DateTime UpdatedAt { get; set; }
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ namespace SatelliteProvider.DataAccess.Repositories;
|
|||||||
public interface ITileRepository
|
public interface ITileRepository
|
||||||
{
|
{
|
||||||
Task<TileEntity?> GetByIdAsync(Guid id);
|
Task<TileEntity?> GetByIdAsync(Guid id);
|
||||||
Task<TileEntity?> FindExistingTileAsync(double latitude, double longitude, double tileSizeMeters, int zoomLevel);
|
Task<TileEntity?> FindExistingTileAsync(double latitude, double longitude, double tileSizeMeters, int zoomLevel, int version);
|
||||||
Task<IEnumerable<TileEntity>> GetTilesByRegionAsync(double latitude, double longitude, double sizeMeters, int zoomLevel);
|
Task<IEnumerable<TileEntity>> GetTilesByRegionAsync(double latitude, double longitude, double sizeMeters, int zoomLevel);
|
||||||
Task<Guid> InsertAsync(TileEntity tile);
|
Task<Guid> InsertAsync(TileEntity tile);
|
||||||
Task<int> UpdateAsync(TileEntity tile);
|
Task<int> UpdateAsync(TileEntity tile);
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ public class TileRepository : ITileRepository
|
|||||||
const string sql = @"
|
const string sql = @"
|
||||||
SELECT id, zoom_level as ZoomLevel, latitude, longitude,
|
SELECT id, zoom_level as ZoomLevel, latitude, longitude,
|
||||||
tile_size_meters as TileSizeMeters, tile_size_pixels as TileSizePixels,
|
tile_size_meters as TileSizeMeters, tile_size_pixels as TileSizePixels,
|
||||||
image_type as ImageType, maps_version as MapsVersion,
|
image_type as ImageType, maps_version as MapsVersion, version,
|
||||||
file_path as FilePath, created_at as CreatedAt, updated_at as UpdatedAt
|
file_path as FilePath, created_at as CreatedAt, updated_at as UpdatedAt
|
||||||
FROM tiles
|
FROM tiles
|
||||||
WHERE id = @Id";
|
WHERE id = @Id";
|
||||||
@@ -27,19 +27,20 @@ public class TileRepository : ITileRepository
|
|||||||
return await connection.QuerySingleOrDefaultAsync<TileEntity>(sql, new { Id = id });
|
return await connection.QuerySingleOrDefaultAsync<TileEntity>(sql, new { Id = id });
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task<TileEntity?> FindExistingTileAsync(double latitude, double longitude, double tileSizeMeters, int zoomLevel)
|
public async Task<TileEntity?> FindExistingTileAsync(double latitude, double longitude, double tileSizeMeters, int zoomLevel, int version)
|
||||||
{
|
{
|
||||||
using var connection = new NpgsqlConnection(_connectionString);
|
using var connection = new NpgsqlConnection(_connectionString);
|
||||||
const string sql = @"
|
const string sql = @"
|
||||||
SELECT id, zoom_level as ZoomLevel, latitude, longitude,
|
SELECT id, zoom_level as ZoomLevel, latitude, longitude,
|
||||||
tile_size_meters as TileSizeMeters, tile_size_pixels as TileSizePixels,
|
tile_size_meters as TileSizeMeters, tile_size_pixels as TileSizePixels,
|
||||||
image_type as ImageType, maps_version as MapsVersion,
|
image_type as ImageType, maps_version as MapsVersion, version,
|
||||||
file_path as FilePath, created_at as CreatedAt, updated_at as UpdatedAt
|
file_path as FilePath, created_at as CreatedAt, updated_at as UpdatedAt
|
||||||
FROM tiles
|
FROM tiles
|
||||||
WHERE ABS(latitude - @Latitude) < 0.0001
|
WHERE ABS(latitude - @Latitude) < 0.0001
|
||||||
AND ABS(longitude - @Longitude) < 0.0001
|
AND ABS(longitude - @Longitude) < 0.0001
|
||||||
AND ABS(tile_size_meters - @TileSizeMeters) < 1
|
AND ABS(tile_size_meters - @TileSizeMeters) < 1
|
||||||
AND zoom_level = @ZoomLevel
|
AND zoom_level = @ZoomLevel
|
||||||
|
AND version = @Version
|
||||||
LIMIT 1";
|
LIMIT 1";
|
||||||
|
|
||||||
return await connection.QuerySingleOrDefaultAsync<TileEntity>(sql, new
|
return await connection.QuerySingleOrDefaultAsync<TileEntity>(sql, new
|
||||||
@@ -47,7 +48,8 @@ public class TileRepository : ITileRepository
|
|||||||
Latitude = latitude,
|
Latitude = latitude,
|
||||||
Longitude = longitude,
|
Longitude = longitude,
|
||||||
TileSizeMeters = tileSizeMeters,
|
TileSizeMeters = tileSizeMeters,
|
||||||
ZoomLevel = zoomLevel
|
ZoomLevel = zoomLevel,
|
||||||
|
Version = version
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -55,19 +57,27 @@ public class TileRepository : ITileRepository
|
|||||||
{
|
{
|
||||||
using var connection = new NpgsqlConnection(_connectionString);
|
using var connection = new NpgsqlConnection(_connectionString);
|
||||||
|
|
||||||
var latRange = sizeMeters / 111000.0;
|
const double EARTH_CIRCUMFERENCE_METERS = 40075016.686;
|
||||||
var lonRange = sizeMeters / (111000.0 * Math.Cos(latitude * Math.PI / 180.0));
|
const int TILE_SIZE_PIXELS = 256;
|
||||||
|
var latRad = latitude * Math.PI / 180.0;
|
||||||
|
var metersPerPixel = (EARTH_CIRCUMFERENCE_METERS * Math.Cos(latRad)) / (Math.Pow(2, zoomLevel) * TILE_SIZE_PIXELS);
|
||||||
|
var tileSizeMeters = metersPerPixel * TILE_SIZE_PIXELS;
|
||||||
|
|
||||||
|
var expandedSizeMeters = sizeMeters + (tileSizeMeters * 2);
|
||||||
|
|
||||||
|
var latRange = expandedSizeMeters / 111000.0;
|
||||||
|
var lonRange = expandedSizeMeters / (111000.0 * Math.Cos(latitude * Math.PI / 180.0));
|
||||||
|
|
||||||
const string sql = @"
|
const string sql = @"
|
||||||
SELECT id, zoom_level as ZoomLevel, latitude, longitude,
|
SELECT id, zoom_level as ZoomLevel, latitude, longitude,
|
||||||
tile_size_meters as TileSizeMeters, tile_size_pixels as TileSizePixels,
|
tile_size_meters as TileSizeMeters, tile_size_pixels as TileSizePixels,
|
||||||
image_type as ImageType, maps_version as MapsVersion,
|
image_type as ImageType, maps_version as MapsVersion, version,
|
||||||
file_path as FilePath, created_at as CreatedAt, updated_at as UpdatedAt
|
file_path as FilePath, created_at as CreatedAt, updated_at as UpdatedAt
|
||||||
FROM tiles
|
FROM tiles
|
||||||
WHERE latitude BETWEEN @MinLat AND @MaxLat
|
WHERE latitude BETWEEN @MinLat AND @MaxLat
|
||||||
AND longitude BETWEEN @MinLon AND @MaxLon
|
AND longitude BETWEEN @MinLon AND @MaxLon
|
||||||
AND zoom_level = @ZoomLevel
|
AND zoom_level = @ZoomLevel
|
||||||
ORDER BY latitude DESC, longitude ASC";
|
ORDER BY version DESC, latitude DESC, longitude ASC";
|
||||||
|
|
||||||
return await connection.QueryAsync<TileEntity>(sql, new
|
return await connection.QueryAsync<TileEntity>(sql, new
|
||||||
{
|
{
|
||||||
@@ -84,11 +94,15 @@ public class TileRepository : ITileRepository
|
|||||||
using var connection = new NpgsqlConnection(_connectionString);
|
using var connection = new NpgsqlConnection(_connectionString);
|
||||||
const string sql = @"
|
const string sql = @"
|
||||||
INSERT INTO tiles (id, zoom_level, latitude, longitude, tile_size_meters,
|
INSERT INTO tiles (id, zoom_level, latitude, longitude, tile_size_meters,
|
||||||
tile_size_pixels, image_type, maps_version, file_path,
|
tile_size_pixels, image_type, maps_version, version, file_path,
|
||||||
created_at, updated_at)
|
created_at, updated_at)
|
||||||
VALUES (@Id, @ZoomLevel, @Latitude, @Longitude, @TileSizeMeters,
|
VALUES (@Id, @ZoomLevel, @Latitude, @Longitude, @TileSizeMeters,
|
||||||
@TileSizePixels, @ImageType, @MapsVersion, @FilePath,
|
@TileSizePixels, @ImageType, @MapsVersion, @Version, @FilePath,
|
||||||
@CreatedAt, @UpdatedAt)
|
@CreatedAt, @UpdatedAt)
|
||||||
|
ON CONFLICT (latitude, longitude, zoom_level, tile_size_meters, version)
|
||||||
|
DO UPDATE SET
|
||||||
|
file_path = EXCLUDED.file_path,
|
||||||
|
updated_at = EXCLUDED.updated_at
|
||||||
RETURNING id";
|
RETURNING id";
|
||||||
|
|
||||||
return await connection.ExecuteScalarAsync<Guid>(sql, tile);
|
return await connection.ExecuteScalarAsync<Guid>(sql, tile);
|
||||||
@@ -106,6 +120,7 @@ public class TileRepository : ITileRepository
|
|||||||
tile_size_pixels = @TileSizePixels,
|
tile_size_pixels = @TileSizePixels,
|
||||||
image_type = @ImageType,
|
image_type = @ImageType,
|
||||||
maps_version = @MapsVersion,
|
maps_version = @MapsVersion,
|
||||||
|
version = @Version,
|
||||||
file_path = @FilePath,
|
file_path = @FilePath,
|
||||||
updated_at = @UpdatedAt
|
updated_at = @UpdatedAt
|
||||||
WHERE id = @Id";
|
WHERE id = @Id";
|
||||||
|
|||||||
@@ -84,40 +84,6 @@ public class GoogleMapsDownloaderV2
|
|||||||
var response = await httpClient.GetAsync(url, token);
|
var response = await httpClient.GetAsync(url, token);
|
||||||
response.EnsureSuccessStatusCode();
|
response.EnsureSuccessStatusCode();
|
||||||
|
|
||||||
_logger.LogInformation("=== HTTP Response Headers from Google Maps ===");
|
|
||||||
_logger.LogInformation("Status Code: {StatusCode}", response.StatusCode);
|
|
||||||
|
|
||||||
foreach (var header in response.Headers)
|
|
||||||
{
|
|
||||||
_logger.LogInformation("Header: {Key} = {Value}", header.Key, string.Join(", ", header.Value));
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach (var header in response.Content.Headers)
|
|
||||||
{
|
|
||||||
_logger.LogInformation("Content Header: {Key} = {Value}", header.Key, string.Join(", ", header.Value));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (response.Headers.ETag != null)
|
|
||||||
{
|
|
||||||
_logger.LogInformation("*** ETag Found: {ETag}", response.Headers.ETag.Tag);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (response.Content.Headers.LastModified.HasValue)
|
|
||||||
{
|
|
||||||
_logger.LogInformation("*** Last-Modified Found: {LastModified}", response.Content.Headers.LastModified.Value);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (response.Headers.CacheControl != null)
|
|
||||||
{
|
|
||||||
_logger.LogInformation("*** Cache-Control: MaxAge={MaxAge}, Public={Public}, Private={Private}, MustRevalidate={MustRevalidate}",
|
|
||||||
response.Headers.CacheControl.MaxAge,
|
|
||||||
response.Headers.CacheControl.Public,
|
|
||||||
response.Headers.CacheControl.Private,
|
|
||||||
response.Headers.CacheControl.MustRevalidate);
|
|
||||||
}
|
|
||||||
|
|
||||||
_logger.LogInformation("=== End of Headers ===");
|
|
||||||
|
|
||||||
var imageBytes = await response.Content.ReadAsByteArrayAsync(token);
|
var imageBytes = await response.Content.ReadAsByteArrayAsync(token);
|
||||||
await File.WriteAllBytesAsync(filePath, imageBytes, token);
|
await File.WriteAllBytesAsync(filePath, imageBytes, token);
|
||||||
|
|
||||||
|
|||||||
@@ -29,15 +29,34 @@ public class TileService : ITileService
|
|||||||
int zoomLevel,
|
int zoomLevel,
|
||||||
CancellationToken cancellationToken = default)
|
CancellationToken cancellationToken = default)
|
||||||
{
|
{
|
||||||
var existingTiles = await _tileRepository.GetTilesByRegionAsync(latitude, longitude, sizeMeters, zoomLevel);
|
var currentVersion = DateTime.UtcNow.Year;
|
||||||
var existingTilesList = existingTiles.ToList();
|
|
||||||
|
|
||||||
_logger.LogInformation("Found {Count} existing tiles for region", existingTilesList.Count);
|
var existingTiles = await _tileRepository.GetTilesByRegionAsync(latitude, longitude, sizeMeters, zoomLevel);
|
||||||
|
var existingTilesList = existingTiles.Where(t => t.Version == currentVersion).ToList();
|
||||||
|
|
||||||
|
_logger.LogInformation("Found {Count} existing tiles for region (version {Version})", existingTilesList.Count, currentVersion);
|
||||||
|
|
||||||
|
if (existingTilesList.Any())
|
||||||
|
{
|
||||||
|
_logger.LogInformation("Existing tiles in DB:");
|
||||||
|
foreach (var et in existingTilesList)
|
||||||
|
{
|
||||||
|
_logger.LogInformation(" DB Tile: Lat={Lat:F12}, Lon={Lon:F12}, Zoom={Zoom}", et.Latitude, et.Longitude, et.ZoomLevel);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
var centerPoint = new GeoPoint(latitude, longitude);
|
var centerPoint = new GeoPoint(latitude, longitude);
|
||||||
var downloadedTiles = await _downloader.GetTilesWithMetadataAsync(centerPoint, sizeMeters / 2, zoomLevel, cancellationToken);
|
var downloadedTiles = await _downloader.GetTilesWithMetadataAsync(centerPoint, sizeMeters / 2, zoomLevel, cancellationToken);
|
||||||
|
|
||||||
|
_logger.LogInformation("Downloaded {Count} tiles from Google:", downloadedTiles.Count);
|
||||||
|
foreach (var dt in downloadedTiles)
|
||||||
|
{
|
||||||
|
_logger.LogInformation(" Downloaded: Lat={Lat:F12}, Lon={Lon:F12}, Zoom={Zoom}", dt.CenterLatitude, dt.CenterLongitude, dt.ZoomLevel);
|
||||||
|
}
|
||||||
|
|
||||||
var result = new List<TileMetadata>();
|
var result = new List<TileMetadata>();
|
||||||
|
int reusedCount = 0;
|
||||||
|
int downloadedCount = 0;
|
||||||
|
|
||||||
foreach (var downloadedTile in downloadedTiles)
|
foreach (var downloadedTile in downloadedTiles)
|
||||||
{
|
{
|
||||||
@@ -48,11 +67,30 @@ public class TileService : ITileService
|
|||||||
|
|
||||||
if (existingTile != null)
|
if (existingTile != null)
|
||||||
{
|
{
|
||||||
_logger.LogDebug("Reusing existing tile at ({Lat}, {Lon})", downloadedTile.CenterLatitude, downloadedTile.CenterLongitude);
|
reusedCount++;
|
||||||
|
_logger.LogInformation("REUSED tile at ({Lat:F12}, {Lon:F12}) - matched DB tile ID {Id}", downloadedTile.CenterLatitude, downloadedTile.CenterLongitude, existingTile.Id);
|
||||||
result.Add(MapToMetadata(existingTile));
|
result.Add(MapToMetadata(existingTile));
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
|
downloadedCount++;
|
||||||
|
_logger.LogInformation("NEW tile needed at ({Lat:F12}, {Lon:F12}) - no match in DB", downloadedTile.CenterLatitude, downloadedTile.CenterLongitude);
|
||||||
|
|
||||||
|
var closestTile = existingTilesList
|
||||||
|
.Select(t => new {
|
||||||
|
Tile = t,
|
||||||
|
LatDiff = Math.Abs(t.Latitude - downloadedTile.CenterLatitude),
|
||||||
|
LonDiff = Math.Abs(t.Longitude - downloadedTile.CenterLongitude)
|
||||||
|
})
|
||||||
|
.OrderBy(x => x.LatDiff + x.LonDiff)
|
||||||
|
.FirstOrDefault();
|
||||||
|
|
||||||
|
if (closestTile != null)
|
||||||
|
{
|
||||||
|
_logger.LogInformation(" Closest DB tile: Lat={Lat:F12} (diff={LatDiff:F12}), Lon={Lon:F12} (diff={LonDiff:F12})",
|
||||||
|
closestTile.Tile.Latitude, closestTile.LatDiff, closestTile.Tile.Longitude, closestTile.LonDiff);
|
||||||
|
}
|
||||||
|
|
||||||
var now = DateTime.UtcNow;
|
var now = DateTime.UtcNow;
|
||||||
var tileEntity = new TileEntity
|
var tileEntity = new TileEntity
|
||||||
{
|
{
|
||||||
@@ -64,16 +102,19 @@ public class TileService : ITileService
|
|||||||
TileSizePixels = 256,
|
TileSizePixels = 256,
|
||||||
ImageType = "jpg",
|
ImageType = "jpg",
|
||||||
MapsVersion = $"downloaded_{now:yyyy-MM-dd}",
|
MapsVersion = $"downloaded_{now:yyyy-MM-dd}",
|
||||||
|
Version = currentVersion,
|
||||||
FilePath = downloadedTile.FilePath,
|
FilePath = downloadedTile.FilePath,
|
||||||
CreatedAt = now,
|
CreatedAt = now,
|
||||||
UpdatedAt = now
|
UpdatedAt = now
|
||||||
};
|
};
|
||||||
|
|
||||||
await _tileRepository.InsertAsync(tileEntity);
|
await _tileRepository.InsertAsync(tileEntity);
|
||||||
_logger.LogInformation("Saved new tile {Id} at ({Lat}, {Lon})", tileEntity.Id, tileEntity.Latitude, tileEntity.Longitude);
|
_logger.LogInformation("Saved new tile {Id} at ({Lat:F12}, {Lon:F12}) version {Version}", tileEntity.Id, tileEntity.Latitude, tileEntity.Longitude, currentVersion);
|
||||||
result.Add(MapToMetadata(tileEntity));
|
result.Add(MapToMetadata(tileEntity));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
_logger.LogInformation("Tile processing summary: {Reused} reused, {Downloaded} new", reusedCount, downloadedCount);
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
@@ -106,6 +147,7 @@ public class TileService : ITileService
|
|||||||
TileSizePixels = entity.TileSizePixels,
|
TileSizePixels = entity.TileSizePixels,
|
||||||
ImageType = entity.ImageType,
|
ImageType = entity.ImageType,
|
||||||
MapsVersion = entity.MapsVersion,
|
MapsVersion = entity.MapsVersion,
|
||||||
|
Version = entity.Version,
|
||||||
FilePath = entity.FilePath,
|
FilePath = entity.FilePath,
|
||||||
CreatedAt = entity.CreatedAt,
|
CreatedAt = entity.CreatedAt,
|
||||||
UpdatedAt = entity.UpdatedAt
|
UpdatedAt = entity.UpdatedAt
|
||||||
|
|||||||
@@ -1,4 +1,14 @@
|
|||||||
services:
|
services:
|
||||||
|
postgres:
|
||||||
|
extends:
|
||||||
|
file: docker-compose.yml
|
||||||
|
service: postgres
|
||||||
|
|
||||||
|
api:
|
||||||
|
extends:
|
||||||
|
file: docker-compose.yml
|
||||||
|
service: api
|
||||||
|
|
||||||
integration-tests:
|
integration-tests:
|
||||||
build:
|
build:
|
||||||
context: .
|
context: .
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ services:
|
|||||||
volumes:
|
volumes:
|
||||||
- ./tiles:/app/tiles
|
- ./tiles:/app/tiles
|
||||||
- ./ready:/app/ready
|
- ./ready:/app/ready
|
||||||
|
- ./logs:/app/logs
|
||||||
depends_on:
|
depends_on:
|
||||||
postgres:
|
postgres:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
|
|||||||
Reference in New Issue
Block a user