database and migrations

SatelliteProvider.Api/Program.cs

@@ -2,9 +2,22 @@ using System.ComponentModel.DataAnnotations;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.OpenApi.Models;
 using Swashbuckle.AspNetCore.SwaggerGen;
+using SatelliteProvider.DataAccess;
+using SatelliteProvider.DataAccess.Repositories;
+using SatelliteProvider.Common.Configs;
 
 var builder = WebApplication.CreateBuilder(args);
 
+var connectionString = builder.Configuration.GetConnectionString("DefaultConnection") 
+    ?? throw new InvalidOperationException("Connection string 'DefaultConnection' not found.");
+
+builder.Services.Configure<MapConfig>(builder.Configuration.GetSection("MapConfig"));
+builder.Services.Configure<StorageConfig>(builder.Configuration.GetSection("StorageConfig"));
+builder.Services.Configure<ProcessingConfig>(builder.Configuration.GetSection("ProcessingConfig"));
+
+builder.Services.AddSingleton<ITileRepository>(sp => new TileRepository(connectionString));
+builder.Services.AddSingleton<IRegionRepository>(sp => new RegionRepository(connectionString));
+
 builder.Services.AddEndpointsApiExplorer();
 builder.Services.AddSwaggerGen(c =>
 {
@@ -32,6 +45,13 @@ builder.Services.AddSwaggerGen(c =>
 
 var app = builder.Build();
 
+var logger = app.Services.GetRequiredService<ILogger<Program>>();
+var migrator = new DatabaseMigrator(connectionString, logger as ILogger<DatabaseMigrator>);
+if (!migrator.RunMigrations())
+{
+    throw new Exception("Database migration failed. Application cannot start.");
+}
+
 if (app.Environment.IsDevelopment())
 {
     app.UseSwagger();

SatelliteProvider.Api/SatelliteProvider.Api.csproj

@@ -15,6 +15,7 @@
 
     <ItemGroup>
       <ProjectReference Include="..\SatelliteProvider.Common\SatelliteProvider.Common.csproj" />
+      <ProjectReference Include="..\SatelliteProvider.DataAccess\SatelliteProvider.DataAccess.csproj" />
     </ItemGroup>
 
 </Project>

SatelliteProvider.Api/appsettings.Development.json

@@ -4,5 +4,16 @@
       "Default": "Information",
       "Microsoft.AspNetCore": "Warning"
     }
+  },
+  "ConnectionStrings": {
+    "DefaultConnection": "Host=localhost;Port=5432;Database=satelliteprovider;Username=postgres;Password=postgres"
+  },
+  "MapConfig": {
+    "Service": "GoogleMaps",
+    "ApiKey": "YOUR_API_KEY_HERE"
+  },
+  "StorageConfig": {
+    "TilesDirectory": "./tiles",
+    "ReadyDirectory": "./ready"
   }
 }

SatelliteProvider.Api/appsettings.json

@@ -5,5 +5,21 @@
       "Microsoft.AspNetCore": "Warning"
     }
   },
-  "AllowedHosts": "*"
+  "AllowedHosts": "*",
+  "ConnectionStrings": {
+    "DefaultConnection": "Host=localhost;Database=satelliteprovider;Username=postgres;Password=postgres"
+  },
+  "MapConfig": {
+    "Service": "GoogleMaps",
+    "ApiKey": "YOUR_API_KEY_HERE"
+  },
+  "StorageConfig": {
+    "TilesDirectory": "./tiles",
+    "ReadyDirectory": "./ready"
+  },
+  "ProcessingConfig": {
+    "MaxConcurrentDownloads": 4,
+    "DefaultZoomLevel": 20,
+    "QueueCapacity": 100
+  }
 }

SatelliteProvider.Common/Configs/DatabaseConfig.cs

@@ -0,0 +1,7 @@
+namespace SatelliteProvider.Common.Configs;
+
+public class DatabaseConfig
+{
+    public string ConnectionString { get; set; } = null!;
+}
+

SatelliteProvider.Common/Configs/ProcessingConfig.cs

@@ -0,0 +1,9 @@
+namespace SatelliteProvider.Common.Configs;
+
+public class ProcessingConfig
+{
+    public int MaxConcurrentDownloads { get; set; } = 4;
+    public int DefaultZoomLevel { get; set; } = 20;
+    public int QueueCapacity { get; set; } = 100;
+}
+

SatelliteProvider.Common/Configs/StorageConfig.cs

@@ -0,0 +1,8 @@
+namespace SatelliteProvider.Common.Configs;
+
+public class StorageConfig
+{
+    public string TilesDirectory { get; set; } = "/tiles";
+    public string ReadyDirectory { get; set; } = "/ready";
+}
+

SatelliteProvider.DataAccess/DatabaseMigrator.cs

@@ -0,0 +1,43 @@
+using System.Reflection;
+using DbUp;
+using Microsoft.Extensions.Logging;
+
+namespace SatelliteProvider.DataAccess;
+
+public class DatabaseMigrator
+{
+    private readonly string _connectionString;
+    private readonly ILogger<DatabaseMigrator>? _logger;
+
+    public DatabaseMigrator(string connectionString, ILogger<DatabaseMigrator>? logger = null)
+    {
+        _connectionString = connectionString;
+        _logger = logger;
+    }
+
+    public bool RunMigrations()
+    {
+        _logger?.LogInformation("Starting database migrations...");
+
+        EnsureDatabase.For.PostgresqlDatabase(_connectionString);
+
+        var upgrader = DeployChanges.To
+            .PostgresqlDatabase(_connectionString)
+            .WithScriptsEmbeddedInAssembly(Assembly.GetExecutingAssembly(), 
+                script => script.Contains(".Migrations."))
+            .LogToConsole()
+            .Build();
+
+        var result = upgrader.PerformUpgrade();
+
+        if (!result.Successful)
+        {
+            _logger?.LogError(result.Error, "Database migration failed");
+            return false;
+        }
+
+        _logger?.LogInformation("Database migrations completed successfully");
+        return true;
+    }
+}
+

SatelliteProvider.DataAccess/Migrations/001_CreateTilesTable.sql

@@ -0,0 +1,14 @@
+CREATE TABLE tiles (
+    id UUID PRIMARY KEY,
+    zoom_level INT NOT NULL,
+    latitude DOUBLE PRECISION NOT NULL,
+    longitude DOUBLE PRECISION NOT NULL,
+    tile_size_meters DOUBLE PRECISION NOT NULL,
+    tile_size_pixels INT NOT NULL,
+    image_type VARCHAR(10) NOT NULL,
+    maps_version VARCHAR(50),
+    file_path VARCHAR(500) NOT NULL,
+    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+

SatelliteProvider.DataAccess/Migrations/002_CreateRegionsTable.sql

@@ -0,0 +1,15 @@
+CREATE TABLE regions (
+    id UUID PRIMARY KEY,
+    latitude DOUBLE PRECISION NOT NULL,
+    longitude DOUBLE PRECISION NOT NULL,
+    size_meters DOUBLE PRECISION NOT NULL,
+    zoom_level INT NOT NULL,
+    status VARCHAR(20) NOT NULL,
+    csv_file_path VARCHAR(500),
+    summary_file_path VARCHAR(500),
+    tiles_downloaded INT DEFAULT 0,
+    tiles_reused INT DEFAULT 0,
+    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+

SatelliteProvider.DataAccess/Migrations/003_CreateIndexes.sql

@@ -0,0 +1,4 @@
+CREATE INDEX idx_tiles_composite ON tiles(latitude, longitude, tile_size_meters);
+CREATE INDEX idx_tiles_zoom ON tiles(zoom_level);
+CREATE INDEX idx_regions_status ON regions(status);
+

SatelliteProvider.DataAccess/Models/RegionEntity.cs

@@ -0,0 +1,18 @@
+namespace SatelliteProvider.DataAccess.Models;
+
+public class RegionEntity
+{
+    public Guid Id { get; set; }
+    public double Latitude { get; set; }
+    public double Longitude { get; set; }
+    public double SizeMeters { get; set; }
+    public int ZoomLevel { get; set; }
+    public string Status { get; set; } = string.Empty;
+    public string? CsvFilePath { get; set; }
+    public string? SummaryFilePath { get; set; }
+    public int TilesDownloaded { get; set; }
+    public int TilesReused { get; set; }
+    public DateTime CreatedAt { get; set; }
+    public DateTime UpdatedAt { get; set; }
+}
+

SatelliteProvider.DataAccess/Models/TileEntity.cs

@@ -0,0 +1,17 @@
+namespace SatelliteProvider.DataAccess.Models;
+
+public class TileEntity
+{
+    public Guid Id { get; set; }
+    public int ZoomLevel { get; set; }
+    public double Latitude { get; set; }
+    public double Longitude { get; set; }
+    public double TileSizeMeters { get; set; }
+    public int TileSizePixels { get; set; }
+    public string ImageType { get; set; } = string.Empty;
+    public string? MapsVersion { get; set; }
+    public string FilePath { get; set; } = string.Empty;
+    public DateTime CreatedAt { get; set; }
+    public DateTime UpdatedAt { get; set; }
+}
+

SatelliteProvider.DataAccess/Repositories/IRegionRepository.cs

@@ -0,0 +1,13 @@
+using SatelliteProvider.DataAccess.Models;
+
+namespace SatelliteProvider.DataAccess.Repositories;
+
+public interface IRegionRepository
+{
+    Task<RegionEntity?> GetByIdAsync(Guid id);
+    Task<IEnumerable<RegionEntity>> GetByStatusAsync(string status);
+    Task<Guid> InsertAsync(RegionEntity region);
+    Task<int> UpdateAsync(RegionEntity region);
+    Task<int> DeleteAsync(Guid id);
+}
+

SatelliteProvider.DataAccess/Repositories/ITileRepository.cs

@@ -0,0 +1,14 @@
+using SatelliteProvider.DataAccess.Models;
+
+namespace SatelliteProvider.DataAccess.Repositories;
+
+public interface ITileRepository
+{
+    Task<TileEntity?> GetByIdAsync(Guid id);
+    Task<TileEntity?> FindExistingTileAsync(double latitude, double longitude, double tileSizeMeters, int zoomLevel);
+    Task<IEnumerable<TileEntity>> GetTilesByRegionAsync(double latitude, double longitude, double sizeMeters, int zoomLevel);
+    Task<Guid> InsertAsync(TileEntity tile);
+    Task<int> UpdateAsync(TileEntity tile);
+    Task<int> DeleteAsync(Guid id);
+}
+

SatelliteProvider.DataAccess/Repositories/RegionRepository.cs

@@ -0,0 +1,89 @@
+using Dapper;
+using Npgsql;
+using SatelliteProvider.DataAccess.Models;
+
+namespace SatelliteProvider.DataAccess.Repositories;
+
+public class RegionRepository : IRegionRepository
+{
+    private readonly string _connectionString;
+
+    public RegionRepository(string connectionString)
+    {
+        _connectionString = connectionString;
+    }
+
+    public async Task<RegionEntity?> GetByIdAsync(Guid id)
+    {
+        using var connection = new NpgsqlConnection(_connectionString);
+        const string sql = @"
+            SELECT id, latitude, longitude, size_meters as SizeMeters, 
+                   zoom_level as ZoomLevel, status, 
+                   csv_file_path as CsvFilePath, summary_file_path as SummaryFilePath,
+                   tiles_downloaded as TilesDownloaded, tiles_reused as TilesReused,
+                   created_at as CreatedAt, updated_at as UpdatedAt
+            FROM regions 
+            WHERE id = @Id";
+        
+        return await connection.QuerySingleOrDefaultAsync<RegionEntity>(sql, new { Id = id });
+    }
+
+    public async Task<IEnumerable<RegionEntity>> GetByStatusAsync(string status)
+    {
+        using var connection = new NpgsqlConnection(_connectionString);
+        const string sql = @"
+            SELECT id, latitude, longitude, size_meters as SizeMeters, 
+                   zoom_level as ZoomLevel, status, 
+                   csv_file_path as CsvFilePath, summary_file_path as SummaryFilePath,
+                   tiles_downloaded as TilesDownloaded, tiles_reused as TilesReused,
+                   created_at as CreatedAt, updated_at as UpdatedAt
+            FROM regions 
+            WHERE status = @Status
+            ORDER BY created_at ASC";
+        
+        return await connection.QueryAsync<RegionEntity>(sql, new { Status = status });
+    }
+
+    public async Task<Guid> InsertAsync(RegionEntity region)
+    {
+        using var connection = new NpgsqlConnection(_connectionString);
+        const string sql = @"
+            INSERT INTO regions (id, latitude, longitude, size_meters, zoom_level, 
+                               status, csv_file_path, summary_file_path,
+                               tiles_downloaded, tiles_reused, created_at, updated_at)
+            VALUES (@Id, @Latitude, @Longitude, @SizeMeters, @ZoomLevel, 
+                    @Status, @CsvFilePath, @SummaryFilePath,
+                    @TilesDownloaded, @TilesReused, @CreatedAt, @UpdatedAt)
+            RETURNING id";
+        
+        return await connection.ExecuteScalarAsync<Guid>(sql, region);
+    }
+
+    public async Task<int> UpdateAsync(RegionEntity region)
+    {
+        using var connection = new NpgsqlConnection(_connectionString);
+        const string sql = @"
+            UPDATE regions 
+            SET latitude = @Latitude,
+                longitude = @Longitude,
+                size_meters = @SizeMeters,
+                zoom_level = @ZoomLevel,
+                status = @Status,
+                csv_file_path = @CsvFilePath,
+                summary_file_path = @SummaryFilePath,
+                tiles_downloaded = @TilesDownloaded,
+                tiles_reused = @TilesReused,
+                updated_at = @UpdatedAt
+            WHERE id = @Id";
+        
+        return await connection.ExecuteAsync(sql, region);
+    }
+
+    public async Task<int> DeleteAsync(Guid id)
+    {
+        using var connection = new NpgsqlConnection(_connectionString);
+        const string sql = "DELETE FROM regions WHERE id = @Id";
+        return await connection.ExecuteAsync(sql, new { Id = id });
+    }
+}
+

SatelliteProvider.DataAccess/Repositories/TileRepository.cs

@@ -0,0 +1,123 @@
+using Dapper;
+using Npgsql;
+using SatelliteProvider.DataAccess.Models;
+
+namespace SatelliteProvider.DataAccess.Repositories;
+
+public class TileRepository : ITileRepository
+{
+    private readonly string _connectionString;
+
+    public TileRepository(string connectionString)
+    {
+        _connectionString = connectionString;
+    }
+
+    public async Task<TileEntity?> GetByIdAsync(Guid id)
+    {
+        using var connection = new NpgsqlConnection(_connectionString);
+        const string sql = @"
+            SELECT id, zoom_level as ZoomLevel, latitude, longitude, 
+                   tile_size_meters as TileSizeMeters, tile_size_pixels as TileSizePixels,
+                   image_type as ImageType, maps_version as MapsVersion, 
+                   file_path as FilePath, created_at as CreatedAt, updated_at as UpdatedAt
+            FROM tiles 
+            WHERE id = @Id";
+        
+        return await connection.QuerySingleOrDefaultAsync<TileEntity>(sql, new { Id = id });
+    }
+
+    public async Task<TileEntity?> FindExistingTileAsync(double latitude, double longitude, double tileSizeMeters, int zoomLevel)
+    {
+        using var connection = new NpgsqlConnection(_connectionString);
+        const string sql = @"
+            SELECT id, zoom_level as ZoomLevel, latitude, longitude, 
+                   tile_size_meters as TileSizeMeters, tile_size_pixels as TileSizePixels,
+                   image_type as ImageType, maps_version as MapsVersion, 
+                   file_path as FilePath, created_at as CreatedAt, updated_at as UpdatedAt
+            FROM tiles 
+            WHERE ABS(latitude - @Latitude) < 0.0001 
+              AND ABS(longitude - @Longitude) < 0.0001 
+              AND ABS(tile_size_meters - @TileSizeMeters) < 1
+              AND zoom_level = @ZoomLevel
+            LIMIT 1";
+        
+        return await connection.QuerySingleOrDefaultAsync<TileEntity>(sql, new 
+        { 
+            Latitude = latitude, 
+            Longitude = longitude, 
+            TileSizeMeters = tileSizeMeters,
+            ZoomLevel = zoomLevel
+        });
+    }
+
+    public async Task<IEnumerable<TileEntity>> GetTilesByRegionAsync(double latitude, double longitude, double sizeMeters, int zoomLevel)
+    {
+        using var connection = new NpgsqlConnection(_connectionString);
+        
+        var latRange = sizeMeters / 111000.0;
+        var lonRange = sizeMeters / (111000.0 * Math.Cos(latitude * Math.PI / 180.0));
+        
+        const string sql = @"
+            SELECT id, zoom_level as ZoomLevel, latitude, longitude, 
+                   tile_size_meters as TileSizeMeters, tile_size_pixels as TileSizePixels,
+                   image_type as ImageType, maps_version as MapsVersion, 
+                   file_path as FilePath, created_at as CreatedAt, updated_at as UpdatedAt
+            FROM tiles 
+            WHERE latitude BETWEEN @MinLat AND @MaxLat
+              AND longitude BETWEEN @MinLon AND @MaxLon
+              AND zoom_level = @ZoomLevel
+            ORDER BY latitude DESC, longitude ASC";
+        
+        return await connection.QueryAsync<TileEntity>(sql, new
+        {
+            MinLat = latitude - latRange / 2,
+            MaxLat = latitude + latRange / 2,
+            MinLon = longitude - lonRange / 2,
+            MaxLon = longitude + lonRange / 2,
+            ZoomLevel = zoomLevel
+        });
+    }
+
+    public async Task<Guid> InsertAsync(TileEntity tile)
+    {
+        using var connection = new NpgsqlConnection(_connectionString);
+        const string sql = @"
+            INSERT INTO tiles (id, zoom_level, latitude, longitude, tile_size_meters, 
+                             tile_size_pixels, image_type, maps_version, file_path, 
+                             created_at, updated_at)
+            VALUES (@Id, @ZoomLevel, @Latitude, @Longitude, @TileSizeMeters, 
+                    @TileSizePixels, @ImageType, @MapsVersion, @FilePath, 
+                    @CreatedAt, @UpdatedAt)
+            RETURNING id";
+        
+        return await connection.ExecuteScalarAsync<Guid>(sql, tile);
+    }
+
+    public async Task<int> UpdateAsync(TileEntity tile)
+    {
+        using var connection = new NpgsqlConnection(_connectionString);
+        const string sql = @"
+            UPDATE tiles 
+            SET zoom_level = @ZoomLevel, 
+                latitude = @Latitude, 
+                longitude = @Longitude,
+                tile_size_meters = @TileSizeMeters,
+                tile_size_pixels = @TileSizePixels,
+                image_type = @ImageType,
+                maps_version = @MapsVersion,
+                file_path = @FilePath,
+                updated_at = @UpdatedAt
+            WHERE id = @Id";
+        
+        return await connection.ExecuteAsync(sql, tile);
+    }
+
+    public async Task<int> DeleteAsync(Guid id)
+    {
+        using var connection = new NpgsqlConnection(_connectionString);
+        const string sql = "DELETE FROM tiles WHERE id = @Id";
+        return await connection.ExecuteAsync(sql, new { Id = id });
+    }
+}
+

SatelliteProvider.DataAccess/SatelliteProvider.DataAccess.csproj

@@ -0,0 +1,21 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net8.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <PackageReference Include="Dapper" Version="2.1.35" />
+        <PackageReference Include="Npgsql" Version="9.0.2" />
+        <PackageReference Include="dbup-postgresql" Version="6.0.0" />
+        <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="9.0.10" />
+        <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.10" />
+    </ItemGroup>
+
+    <ItemGroup>
+        <EmbeddedResource Include="Migrations\*.sql" />
+    </ItemGroup>
+
+</Project>

SatelliteProvider.sln

@@ -9,6 +9,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SatelliteProvider.Services"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SatelliteProvider.Tests", "SatelliteProvider.Tests\SatelliteProvider.Tests.csproj", "{A44A2E49-9270-4938-9D34-A31CE63E636C}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SatelliteProvider.DataAccess", "SatelliteProvider.DataAccess\SatelliteProvider.DataAccess.csproj", "{8709915B-313D-4CFA-9E0D-0B312F3EA5C8}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -31,5 +33,9 @@ Global
 		{A44A2E49-9270-4938-9D34-A31CE63E636C}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{A44A2E49-9270-4938-9D34-A31CE63E636C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{A44A2E49-9270-4938-9D34-A31CE63E636C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{8709915B-313D-4CFA-9E0D-0B312F3EA5C8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{8709915B-313D-4CFA-9E0D-0B312F3EA5C8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{8709915B-313D-4CFA-9E0D-0B312F3EA5C8}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{8709915B-313D-4CFA-9E0D-0B312F3EA5C8}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

docker-compose.yml

@@ -0,0 +1,21 @@
+services:
+  postgres:
+    image: postgres:16
+    container_name: satellite-provider-postgres
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: satelliteprovider
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+
+volumes:
+  postgres_data:
+