make structure

add tests
This commit is contained in:
Oleksandr Bezdieniezhnykh
2025-10-26 09:15:06 +02:00
parent e71b806e04
commit a7a645c7ab
24 changed files with 326 additions and 133 deletions
+21
View File
@@ -0,0 +1,21 @@
---
description: Coding rules
alwaysApply: true
---
# Coding preferences
- Always prefer simple solution
- Generate concise code
- Do not put comments in the code
- Do not put logs unless it is an exception, or was asked specifically
- Do not put code annotations unless it was asked specifically
- Your changes should be well correlated with what was requested. In case of any uncertainties ask questions.
- Mocking data is needed only for tests
- When you add new libraries or dependencies make sure you are using the same version of it as other parts of the code
- Focus on the areas of code relevant to the task
- Do not touch code that is unrelated to the task
- Always think about what other methods and areas of code might be affected by the code changes
- When you think you are done with changes, run tests and make sure they are not broken
- Do not rename any databases or tables or table columns without confirmation. Avoid such renaming if possible.
- Do not create diagrams unless I ask explicitly
- Do not commit binaries, create and keep .gitignore up to date and delete binaries after you are done with the task
+9
View File
@@ -0,0 +1,9 @@
---
description: Techstack
alwaysApply: true
---
# Tech Stack
- Use poetry
- Cython for backend
- Using Postgres database
- document api with OpenAPI
+12
View File
@@ -0,0 +1,12 @@
{
"workbench.colorCustomizations": {
"editor.background": "#191A1C",
"sideBar.background": "#191A1C",
"activityBar.background": "#191A1C",
"panel.background": "#191A1C",
"terminal.background": "#191A1C",
"editorGroupHeader.tabsBackground": "#191A1C",
"tab.inactiveBackground": "#191A1C"
}
}
@@ -13,4 +13,8 @@
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2"/> <PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2"/>
</ItemGroup> </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\SatelliteProvider.Common\SatelliteProvider.Common.csproj" />
</ItemGroup>
</Project> </Project>
@@ -1,4 +1,4 @@
namespace SatelliteProvider.Configs; namespace SatelliteProvider.Common.Configs;
public class MapConfig public class MapConfig
{ {
@@ -1,4 +1,4 @@
namespace SatelliteProvider.DTO; namespace SatelliteProvider.Common.DTO;
public class Direction public class Direction
{ {
@@ -1,4 +1,4 @@
namespace SatelliteProvider; namespace SatelliteProvider.Common.DTO;
public class GeoPoint public class GeoPoint
{ {
@@ -1,9 +1,12 @@
namespace SatelliteProvider.DTO; using SatelliteProvider.Common.Utils;
namespace SatelliteProvider.Common.DTO;
public class SatTile public class SatTile
{ {
public int X { get; } public int X { get; }
public int Y { get; } public int Y { get; }
public int Zoom { get; }
public GeoPoint LeftTop { get; } public GeoPoint LeftTop { get; }
public GeoPoint BottomRight { get; } public GeoPoint BottomRight { get; }
public string Url { get; set; } public string Url { get; set; }
@@ -13,13 +16,14 @@ public class SatTile
{ {
X = x; X = x;
Y = y; Y = y;
Zoom = zoom;
Url = url; Url = url;
LeftTop = GeoUtils.TileToWorldPos(x, y, zoom); LeftTop = GeoUtils.TileToWorldPos(x, y, zoom);
BottomRight = GeoUtils.TileToWorldPos(x + 1, y + 1, zoom); BottomRight = GeoUtils.TileToWorldPos(x + 1, y + 1, zoom);
} }
public string FileName => $"tile_lt_{LeftTop.Lat:F6}_{LeftTop.Lon:F6}_br_{BottomRight.Lat:F6}_{BottomRight.Lon:F6}.jpg"; public string FileName => $"{X}.{Y}.{Zoom}.jpg";
public override string ToString() public override string ToString()
{ {
@@ -0,0 +1,8 @@
using SatelliteProvider.Common.DTO;
namespace SatelliteProvider.Common.Interfaces;
public interface ISatelliteDownloader
{
Task GetTiles(GeoPoint geoPoint, double radiusM, int zoomLevel, CancellationToken token = default);
}
@@ -0,0 +1,9 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>
@@ -1,16 +1,16 @@
using SatelliteProvider.DTO; using SatelliteProvider.Common.DTO;
namespace SatelliteProvider; namespace SatelliteProvider.Common.Utils;
public static class GeoUtils public static class GeoUtils
{ {
private const double EARTH_RADIUS = 6378137; private const double EARTH_RADIUS = 6378137;
public static (int x, int y) WorldToTilePos(double lat, double lon, int zoom) public static (int x, int y) WorldToTilePos(GeoPoint point, int zoom)
{ {
var latRad = lat * Math.PI / 180.0; var latRad = point.Lat * Math.PI / 180.0;
var n = Math.Pow(2.0, zoom); var n = Math.Pow(2.0, zoom);
var xTile = (int)Math.Floor((lon + 180.0) / 360.0 * n); var xTile = (int)Math.Floor((point.Lon + 180.0) / 360.0 * n);
var yTile = (int)Math.Floor((1.0 - Math.Log(Math.Tan(latRad) + 1.0 / Math.Cos(latRad)) / Math.PI) / 2.0 * n); var yTile = (int)Math.Floor((1.0 - Math.Log(Math.Tan(latRad) + 1.0 / Math.Cos(latRad)) / Math.PI) / 2.0 * n);
return (xTile, yTile); return (xTile, yTile);
} }
@@ -0,0 +1,104 @@
using System.Collections.Concurrent;
using System.Net.Http.Json;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Newtonsoft.Json;
using SatelliteProvider.Common.Configs;
using SatelliteProvider.Common.DTO;
using SatelliteProvider.Common.Interfaces;
using SatelliteProvider.Common.Utils;
using SixLabors.ImageSharp;
namespace SatelliteProvider.Services;
public class GoogleMapsDownloader(ILogger<GoogleMapsDownloader> logger, IOptions<MapConfig> mapConfig, IHttpClientFactory httpClientFactory)
: ISatelliteDownloader
{
private const string TILE_URL_TEMPLATE = "https://mt{0}.google.com/vt/lyrs=s&x={1}&y={2}&z={3}&token={4}";
private const string USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.0.0 Safari/537.36";
private const int NUM_SERVERS = 4;
private readonly string _apiKey = mapConfig.Value.ApiKey;
private readonly string _satDirectory = Path.Combine(Directory.GetCurrentDirectory(), "maps");
private record SessionResponse(string Session);
private async Task<string?> GetSessionToken()
{
var url = $"https://tile.googleapis.com/v1/createSession?key={_apiKey}";
using var httpClient = httpClientFactory.CreateClient();
try
{
var str = JsonConvert.SerializeObject(new { mapType = "satellite" });
var response = await httpClient.PostAsync(url, new StringContent(str));
response.EnsureSuccessStatusCode();
var sessionResponse = await response.Content.ReadFromJsonAsync<SessionResponse>();
return sessionResponse?.Session;
}
catch (Exception e)
{
logger.LogError(e, "Failed to get session token");
throw;
}
}
public async Task GetTiles(GeoPoint centerGeoPoint, double radiusM, int zoomLevel, CancellationToken token = default)
{
var (latMin, latMax, lonMin, lonMax) = GeoUtils.GetBoundingBox(centerGeoPoint, radiusM);
var (xMin, yMin) = GeoUtils.WorldToTilePos(new GeoPoint(latMax, lonMin), zoomLevel);
var (xMax, yMax) = GeoUtils.WorldToTilePos(new GeoPoint(latMin, lonMax), zoomLevel);
var tilesToDownload = new ConcurrentQueue<SatTile>();
var server = 0;
var sessionToken = await GetSessionToken();
for (var y = yMin; y <= yMax + 1; y++)
for (var x = xMin; x <= xMax + 1; x++)
{
token.ThrowIfCancellationRequested();
var url = string.Format(TILE_URL_TEMPLATE, server, x, y, zoomLevel, sessionToken);
tilesToDownload.Enqueue(new SatTile(x, y, zoomLevel, url));
server = (server + 1) % NUM_SERVERS;
}
var downloadTasks = new List<Task>();
for (int i = 0; i < NUM_SERVERS; i++)
{
downloadTasks.Add(Task.Run(() => DownloadTilesWorker(tilesToDownload, token), token));
}
await Task.WhenAll(downloadTasks);
}
private async Task DownloadTilesWorker(ConcurrentQueue<SatTile> tilesToDownload, CancellationToken token)
{
using var httpClient = httpClientFactory.CreateClient();
while (tilesToDownload.TryDequeue(out var tileInfo))
{
if (token.IsCancellationRequested) break;
try
{
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd(USER_AGENT);
var response = await httpClient.GetAsync(tileInfo.Url, token);
response.EnsureSuccessStatusCode();
var tileData = await response.Content.ReadAsByteArrayAsync(token);
using var tileImage = Image.Load(tileData);
await tileImage.SaveAsync(Path.Combine(_satDirectory, tileInfo.FileName), token);
}
catch (HttpRequestException requestException)
{
logger.LogError(requestException, "Failed to download tile. Url: {Url}", tileInfo.Url);
}
catch (Exception e)
{
logger.LogError(e, "Failed to download tile");
}
}
}
}
@@ -0,0 +1,21 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Http" Version="9.0.10" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.10" />
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="9.0.10" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.4" />
<PackageReference Include="SixLabors.ImageSharp" Version="3.1.11" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\SatelliteProvider.Common\SatelliteProvider.Common.csproj" />
</ItemGroup>
</Project>
@@ -0,0 +1,57 @@
using FluentAssertions;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using SatelliteProvider.Common.Configs;
using SatelliteProvider.Common.DTO;
using SatelliteProvider.Services;
using Xunit;
namespace SatelliteProvider.Tests;
public class GoogleMapsDownloaderTests
{
[Fact]
public async Task IntegrationTest_DownloadRealTiles_ShouldDownloadBytes()
{
var configuration = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json")
.Build();
var mapConfig = new MapConfig();
configuration.GetSection("MapConfig").Bind(mapConfig);
var services = new ServiceCollection();
services.AddHttpClient();
services.AddLogging(builder => builder.AddConsole());
var serviceProvider = services.BuildServiceProvider();
var logger = serviceProvider.GetRequiredService<ILogger<GoogleMapsDownloader>>();
var options = Options.Create(mapConfig);
var httpClientFactory = serviceProvider.GetRequiredService<IHttpClientFactory>();
var downloader = new GoogleMapsDownloader(logger, options, httpClientFactory);
var centerPoint = new GeoPoint(37.7749, -122.4194);
var radius = 200.0;
var zoomLevel = 15;
await downloader.GetTiles(centerPoint, radius, zoomLevel);
var mapsDirectory = Path.Combine(Directory.GetCurrentDirectory(), "maps");
Directory.Exists(mapsDirectory).Should().BeTrue();
var files = Directory.GetFiles(mapsDirectory, "*.jpg");
files.Should().NotBeEmpty();
var totalBytes = files.Sum(file => new FileInfo(file).Length);
totalBytes.Should().BeGreaterThan(0);
foreach (var file in files)
{
File.Delete(file);
}
}
}
@@ -0,0 +1,42 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>
<IsTestProject>true</IsTestProject>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="coverlet.collector" Version="6.0.0" />
<PackageReference Include="FluentAssertions" Version="8.8.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="9.0.10" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="9.0.10" />
<PackageReference Include="Microsoft.Extensions.Http" Version="9.0.10" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.10" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.10" />
<PackageReference Include="Microsoft.Extensions.Options" Version="9.0.10" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
<PackageReference Include="Moq" Version="4.20.72" />
<PackageReference Include="xunit" Version="2.5.3" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.5.3" />
</ItemGroup>
<ItemGroup>
<Using Include="Xunit" />
</ItemGroup>
<ItemGroup>
<None Update="appsettings.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\SatelliteProvider.Services\SatelliteProvider.Services.csproj" />
<ProjectReference Include="..\SatelliteProvider.Common\SatelliteProvider.Common.csproj" />
</ItemGroup>
</Project>
+5
View File
@@ -0,0 +1,5 @@
{
"MapConfig": {
"ApiKey": "AIzaSyAXRBDBOskC5QOHG6VJWzmVJwYKcu6WH8k"
}
}
+20 -1
View File
@@ -1,6 +1,13 @@
Microsoft Visual Studio Solution File, Format Version 12.00 Microsoft Visual Studio Solution File, Format Version 12.00
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SatelliteProvider", "SatelliteProvider\SatelliteProvider.csproj", "{35C6FC8B-92D8-4D8D-BE36-D6B181715019}" #
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SatelliteProvider.Api", "SatelliteProvider.Api\SatelliteProvider.Api.csproj", "{35C6FC8B-92D8-4D8D-BE36-D6B181715019}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SatelliteProvider.Common", "SatelliteProvider.Common\SatelliteProvider.Common.csproj", "{5499248E-F025-4091-9103-6AA02C6CB613}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SatelliteProvider.Services", "SatelliteProvider.Services\SatelliteProvider.Services.csproj", "{452166A0-28C3-429F-B2BD-39041FB7B5A5}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SatelliteProvider.Tests", "SatelliteProvider.Tests\SatelliteProvider.Tests.csproj", "{A44A2E49-9270-4938-9D34-A31CE63E636C}"
EndProject EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -12,5 +19,17 @@ Global
{35C6FC8B-92D8-4D8D-BE36-D6B181715019}.Debug|Any CPU.Build.0 = Debug|Any CPU {35C6FC8B-92D8-4D8D-BE36-D6B181715019}.Debug|Any CPU.Build.0 = Debug|Any CPU
{35C6FC8B-92D8-4D8D-BE36-D6B181715019}.Release|Any CPU.ActiveCfg = Release|Any CPU {35C6FC8B-92D8-4D8D-BE36-D6B181715019}.Release|Any CPU.ActiveCfg = Release|Any CPU
{35C6FC8B-92D8-4D8D-BE36-D6B181715019}.Release|Any CPU.Build.0 = Release|Any CPU {35C6FC8B-92D8-4D8D-BE36-D6B181715019}.Release|Any CPU.Build.0 = Release|Any CPU
{5499248E-F025-4091-9103-6AA02C6CB613}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5499248E-F025-4091-9103-6AA02C6CB613}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5499248E-F025-4091-9103-6AA02C6CB613}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5499248E-F025-4091-9103-6AA02C6CB613}.Release|Any CPU.Build.0 = Release|Any CPU
{452166A0-28C3-429F-B2BD-39041FB7B5A5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{452166A0-28C3-429F-B2BD-39041FB7B5A5}.Debug|Any CPU.Build.0 = Debug|Any CPU
{452166A0-28C3-429F-B2BD-39041FB7B5A5}.Release|Any CPU.ActiveCfg = Release|Any CPU
{452166A0-28C3-429F-B2BD-39041FB7B5A5}.Release|Any CPU.Build.0 = Release|Any CPU
{A44A2E49-9270-4938-9D34-A31CE63E636C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A44A2E49-9270-4938-9D34-A31CE63E636C}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A44A2E49-9270-4938-9D34-A31CE63E636C}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A44A2E49-9270-4938-9D34-A31CE63E636C}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection EndGlobalSection
EndGlobal EndGlobal
@@ -1,15 +0,0 @@
namespace SatelliteProvider.Configs;
public class DirectoriesConfig
{
public string? ApiResourcesDirectory { get; set; } = null!;
public string VideosDirectory { get; set; } = null!;
public string LabelsDirectory { get; set; } = null!;
public string ImagesDirectory { get; set; } = null!;
public string ResultsDirectory { get; set; } = null!;
public string ThumbnailsDirectory { get; set; } = null!;
public string GpsSatDirectory { get; set; } = null!;
public string GpsRouteDirectory { get; set; } = null!;
}
-107
View File
@@ -1,107 +0,0 @@
using System.Collections.Concurrent;
using Microsoft.Extensions.Options;
using Newtonsoft.Json;
using SatelliteProvider.Configs;
using SatelliteProvider.DTO;
using SixLabors.ImageSharp;
namespace SatelliteProvider;
public interface ISatelliteDownloader
{
Task GetTiles(GeoPoint geoPoint, double radiusM, int zoomLevel, CancellationToken token = default);
}
public class SatelliteDownloader(ILogger<SatelliteDownloader> logger, IOptions<MapConfig> mapConfig, IHttpClientFactory httpClientFactory)
: ISatelliteDownloader
{
private const string TILE_URL_TEMPLATE = "https://mt{0}.google.com/vt/lyrs=s&x={1}&y={2}&z={3}&token={4}";
private const int NUM_SERVERS = 4;
private readonly string _apiKey = mapConfig.Value.ApiKey;
private readonly string _satDirectory = Path.Combine(Directory.GetCurrentDirectory(), "maps");
private record SessionResponse(string Session);
private async Task<string?> GetSessionToken()
{
var url = $"https://tile.googleapis.com/v1/createSession?key={_apiKey}";
using var httpClient = httpClientFactory.CreateClient();
try
{
var str = JsonConvert.SerializeObject(new { mapType = "satellite" });
var response = await httpClient.PostAsync(url, new StringContent(str));
response.EnsureSuccessStatusCode();
var sessionResponse = await response.Content.ReadFromJsonAsync<SessionResponse>();
return sessionResponse?.Session;
}
catch (Exception e)
{
logger.LogError(e, e.Message);
throw;
}
}
public async Task GetTiles(GeoPoint centerGeoPoint, double radiusM, int zoomLevel, CancellationToken token = default)
{
var (latMin, latMax, lonMin, lonMax) = GeoUtils.GetBoundingBox(centerGeoPoint, radiusM);
var (xMin, yMin) = GeoUtils.WorldToTilePos(latMax, lonMin, zoomLevel); // Top-left corner
var (xMax, yMax) = GeoUtils.WorldToTilePos(latMin, lonMax, zoomLevel); // Bottom-right corner
var tilesToDownload = new ConcurrentQueue<SatTile>();
var server = 0;
var sessionToken = await GetSessionToken();
for (var y = yMin; y <= yMax + 1; y++)
for (var x = xMin; x <= xMax + 1; x++)
{
token.ThrowIfCancellationRequested();
var url = string.Format(TILE_URL_TEMPLATE, server, x, y, zoomLevel, sessionToken);
tilesToDownload.Enqueue(new SatTile(x, y, zoomLevel, url));
server = (server + 1) % NUM_SERVERS;
}
var downloadTasks = new List<Task>();
int downloadedCount = 0;
for (int i = 0; i < NUM_SERVERS; i++)
{
downloadTasks.Add(Task.Run(async () =>
{
using var httpClient = httpClientFactory.CreateClient();
while (tilesToDownload.TryDequeue(out var tileInfo))
{
if (token.IsCancellationRequested) break;
try
{
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.0.0 Safari/537.36");
var response = await httpClient.GetAsync(tileInfo.Url, token);
response.EnsureSuccessStatusCode();
var tileData = await response.Content.ReadAsByteArrayAsync(token);
using var tileImage = Image.Load(tileData);
await tileImage.SaveAsync(Path.Combine(_satDirectory, tileInfo.FileName), token);
if (tileData.Length > 0)
{
Interlocked.Increment(ref downloadedCount);
}
}
catch (HttpRequestException requestException)
{
logger.LogError(requestException, $"Fail to download tile! Url: {tileInfo.Url}. {requestException.Message}");
}
catch (Exception e)
{
logger.LogError(e, $"Fail to download tile! {e.Message}");
}
}
}, token));
}
await Task.WhenAll(downloadTasks);
}
}