From 75d3a2412f705a8303b8df92047a07394eba4365 Mon Sep 17 00:00:00 2001 From: Alex Bezdieniezhnykh Date: Thu, 3 Jul 2025 19:36:48 +0300 Subject: [PATCH 01/14] fix loader version check --- Azaion.LoaderUI/ApiCredentials.cs | 3 -- Azaion.LoaderUI/Login.xaml.cs | 59 ++++++++++++++++--------------- 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/Azaion.LoaderUI/ApiCredentials.cs b/Azaion.LoaderUI/ApiCredentials.cs index eeee5f0..13b61f4 100644 --- a/Azaion.LoaderUI/ApiCredentials.cs +++ b/Azaion.LoaderUI/ApiCredentials.cs @@ -10,7 +10,4 @@ public class ApiCredentials [Key(nameof(Password))] public string Password { get; set; } = null!; - - public bool IsValid() => - !string.IsNullOrWhiteSpace(Email) && !string.IsNullOrWhiteSpace(Password); } \ No newline at end of file diff --git a/Azaion.LoaderUI/Login.xaml.cs b/Azaion.LoaderUI/Login.xaml.cs index 14e18a4..dfb0539 100644 --- a/Azaion.LoaderUI/Login.xaml.cs +++ b/Azaion.LoaderUI/Login.xaml.cs @@ -1,6 +1,7 @@ using System.Diagnostics; using System.IO; using System.Text; +using System.Text.RegularExpressions; using System.Windows; using System.Windows.Controls; using System.Windows.Input; @@ -44,14 +45,35 @@ public partial class Login Email = TbEmail.Text, Password = TbPassword.Password }; - if (!creds.IsValid()) + if (string.IsNullOrWhiteSpace(creds.Email) || string.IsNullOrWhiteSpace(creds.Password)) return; - - SetControlsStatus(isLoading: true); - _azaionApi.Login(creds); + try { + SetControlsStatus(isLoading: true); + _azaionApi.Login(creds); Validate(creds); + + TbStatus.Foreground = Brushes.Black; + var installerVersion = await GetInstallerVer(); + var localVersion = GetLocalVer(); + + if (installerVersion > localVersion) + { + TbStatus.Text = $"Updating from {localVersion} to {installerVersion}..."; + await DownloadAndRunInstaller(); + TbStatus.Text = $"Installed {installerVersion}!"; + } + else + TbStatus.Text = "Your version is up to date!"; + + Process.Start(Constants.AZAION_SUITE_EXE, $"-e {creds.Email} -p {creds.Password}"); + await Task.Delay(800); + TbStatus.Text = "Loading..."; + while (!Process.GetProcessesByName(Constants.INFERENCE_EXE).Any()) + await Task.Delay(500); + await Task.Delay(1500); + Close(); } catch (Exception exception) { @@ -59,28 +81,7 @@ public partial class Login TbStatus.Foreground = Brushes.Red; TbStatus.Text = exception.Message; SetControlsStatus(isLoading: false); - return; } - TbStatus.Foreground = Brushes.Black; - var installerVersion = await GetInstallerVer(); - var localVersion = GetLocalVer(); - - if (installerVersion > localVersion) - { - TbStatus.Text = $"Updating from {localVersion} to {installerVersion}..."; - await DownloadAndRunInstaller(); - TbStatus.Text = $"Installed {installerVersion}!"; - } - else - TbStatus.Text = "Your version is up to date!"; - - Process.Start(Constants.AZAION_SUITE_EXE, $"-e {creds.Email} -p {creds.Password}"); - await Task.Delay(800); - TbStatus.Text = "Loading..."; - while (!Process.GetProcessesByName(Constants.INFERENCE_EXE).Any()) - await Task.Delay(500); - await Task.Delay(1500); - Close(); } private void Validate(ApiCredentials creds) @@ -172,10 +173,10 @@ public partial class Login ? Constants.SUITE_FOLDER : _dirConfig.SuiteInstallerDirectory; var installerName = await _azaionApi.GetLastInstallerName(installerDir); - var version = installerName - .Replace("AzaionSuite.Iterative.", "") - .Replace(".exe", ""); - return new Version(version); + var match = Regex.Match(installerName, @"\d+(\.\d+)+"); + if (!match.Success) + throw new Exception($"Can't find version in {installerName}"); + return new Version(match.Value); } private Version GetLocalVer() From 6229ca8a03fe80484f545bd9f59a305843a61762 Mon Sep 17 00:00:00 2001 From: Alex Bezdieniezhnykh Date: Sun, 6 Jul 2025 23:22:21 +0300 Subject: [PATCH 02/14] rework autoupdate to script only zoom fix --- Azaion.Common/Controls/CanvasEditor.cs | 28 ++++++++---- Azaion.Common/DTO/ApiCredentials.cs | 12 +++++- Azaion.Common/DTO/DirectoriesConfig.cs | 3 +- Azaion.Common/Security.cs | 59 ++++++++++++++++++++++++++ Azaion.Inference/azaion-inference.spec | 2 +- Azaion.LoaderUI/App.xaml.cs | 6 --- Azaion.LoaderUI/Azaion.LoaderUI.csproj | 9 +++- Azaion.LoaderUI/Login.xaml.cs | 50 ++++++++++------------ Azaion.LoaderUI/updater.cmd | 39 +++++++++++++++++ Azaion.Suite/App.xaml.cs | 43 +++++++------------ build/build_dotnet.cmd | 4 +- build/publish.cmd | 4 +- 12 files changed, 179 insertions(+), 80 deletions(-) create mode 100644 Azaion.Common/Security.cs create mode 100644 Azaion.LoaderUI/updater.cmd diff --git a/Azaion.Common/Controls/CanvasEditor.cs b/Azaion.Common/Controls/CanvasEditor.cs index 3d93002..c2dd310 100644 --- a/Azaion.Common/Controls/CanvasEditor.cs +++ b/Azaion.Common/Controls/CanvasEditor.cs @@ -126,9 +126,26 @@ public class CanvasEditor : Canvas public void SetImageSource(ImageSource? source) { + SetZoom(); _backgroundImage.Source = source; } + private void SetZoom(Matrix? matrix = null) + { + if (matrix == null) + { + _matrixTransform.Matrix = Matrix.Identity; + _isZoomedIn = false; + } + else + { + _matrixTransform.Matrix = matrix.Value; + _isZoomedIn = true; + } + foreach (var detection in CurrentDetections) + detection.UpdateAdornerScale(scale: _matrixTransform.Matrix.M11); + } + private void CanvasWheel(object sender, MouseWheelEventArgs e) { if (Keyboard.Modifiers != ModifierKeys.Control) @@ -139,19 +156,12 @@ public class CanvasEditor : Canvas var matrix = _matrixTransform.Matrix; if (scale < 1 && matrix.M11 * scale < 1.0) - { - _matrixTransform.Matrix = Matrix.Identity; - _isZoomedIn = false; - } + SetZoom(); else { matrix.ScaleAt(scale, scale, mousePos.X, mousePos.Y); - _matrixTransform.Matrix = matrix; - _isZoomedIn = true; + SetZoom(matrix); } - - foreach (var detection in CurrentDetections) - detection.UpdateAdornerScale(scale: _matrixTransform.Matrix.M11); } private void Init(object sender, RoutedEventArgs e) diff --git a/Azaion.Common/DTO/ApiCredentials.cs b/Azaion.Common/DTO/ApiCredentials.cs index d11ab90..7978e40 100644 --- a/Azaion.Common/DTO/ApiCredentials.cs +++ b/Azaion.Common/DTO/ApiCredentials.cs @@ -4,7 +4,8 @@ using MessagePack; namespace Azaion.Common.DTO; [MessagePackObject] -public class ApiCredentials : EventArgs +[Verb("credsManual", HelpText = "Manual Credentials")] +public class ApiCredentials { [Key(nameof(Email))] [Option('e', "email", Required = true, HelpText = "User Email")] @@ -13,4 +14,11 @@ public class ApiCredentials : EventArgs [Key(nameof(Password))] [Option('p', "pass", Required = true, HelpText = "User Password")] public string Password { get; set; } = null!; -} \ No newline at end of file +} + +[Verb("credsEncrypted", isDefault: true, HelpText = "Encrypted Credentials")] +public class ApiCredentialsEncrypted +{ + [Option('c', "creds", Group = "auto", HelpText = "Encrypted Creds")] + public string Creds { get; set; } = null!; +} diff --git a/Azaion.Common/DTO/DirectoriesConfig.cs b/Azaion.Common/DTO/DirectoriesConfig.cs index c4d6f73..5d8a5b8 100644 --- a/Azaion.Common/DTO/DirectoriesConfig.cs +++ b/Azaion.Common/DTO/DirectoriesConfig.cs @@ -1,9 +1,8 @@ namespace Azaion.Common.DTO; public class DirectoriesConfig - { - public string ApiResourcesDirectory { get; set; } = null!; + public string? ApiResourcesDirectory { get; set; } = null!; public string VideosDirectory { get; set; } = null!; public string LabelsDirectory { get; set; } = null!; diff --git a/Azaion.Common/Security.cs b/Azaion.Common/Security.cs new file mode 100644 index 0000000..02980fc --- /dev/null +++ b/Azaion.Common/Security.cs @@ -0,0 +1,59 @@ +using System.Security.Cryptography; +using System.Text; +using Newtonsoft.Json; + +namespace Azaion.Common; + +public class Security +{ + private static string GenDefaultKey() + { + var date = DateTime.UtcNow; + return $"sAzaion_default_dfvkjhg_{date:yyyy}-{date:MM}_{date:dd}_{date:HH}_key"; + } + + public static string Encrypt(T model, string? key = null) where T : class + { + var json = JsonConvert.SerializeObject(model); + var inputBytes = Encoding.UTF8.GetBytes(json); + + var keyBytes = SHA256.HashData(Encoding.UTF8.GetBytes(key ?? GenDefaultKey())); + var iv = RandomNumberGenerator.GetBytes(16); + + using var aes = Aes.Create(); + aes.Key = keyBytes; + aes.IV = iv; + aes.Mode = CipherMode.CFB; + aes.Padding = PaddingMode.ISO10126; + + using var encryptor = aes.CreateEncryptor(); + var ciphertext = encryptor.TransformFinalBlock(inputBytes, 0, inputBytes.Length); + + var result = new byte[iv.Length + ciphertext.Length]; + iv.CopyTo(result, 0); + ciphertext.CopyTo(result, iv.Length); + + return Convert.ToBase64String(result); + } + + public static T Decrypt(string encryptedData, string? key = null) where T : class + { + var ciphertextWithIv = Convert.FromBase64String(encryptedData); + var keyBytes = SHA256.HashData(Encoding.UTF8.GetBytes(key ?? GenDefaultKey())); + + var iv = ciphertextWithIv[..16]; + var ciphertext = ciphertextWithIv[16..]; + + using var aes = Aes.Create(); + aes.Key = keyBytes; + aes.IV = iv; + aes.Mode = CipherMode.CFB; + aes.Padding = PaddingMode.ISO10126; + + using var decryptor = aes.CreateDecryptor(); + var plaintext = decryptor.TransformFinalBlock(ciphertext, 0, ciphertext.Length); + + var json = Encoding.UTF8.GetString(plaintext); + return JsonConvert.DeserializeObject(json)!; + } +} \ No newline at end of file diff --git a/Azaion.Inference/azaion-inference.spec b/Azaion.Inference/azaion-inference.spec index 7ae365f..b2ed95d 100644 --- a/Azaion.Inference/azaion-inference.spec +++ b/Azaion.Inference/azaion-inference.spec @@ -4,7 +4,7 @@ from PyInstaller.utils.hooks import collect_all datas = [('venv\\Lib\\site-packages\\cv2', 'cv2')] binaries = [] -hiddenimports = ['constants_inf', 'file_data', 'remote_command_inf', 'remote_command_handler_inf', 'annotation', 'loader_client', 'ai_config', 'tensorrt_engine', 'onnx_engine', 'inference_engine', 'inference', 'main-inf'] +hiddenimports = ['constants_inf', 'file_data', 'remote_command_inf', 'remote_command_handler_inf', 'annotation', 'loader_client', 'ai_config', 'tensorrt_engine', 'onnx_engine', 'inference_engine', 'inference'] hiddenimports += collect_submodules('cv2') tmp_ret = collect_all('psutil') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] diff --git a/Azaion.LoaderUI/App.xaml.cs b/Azaion.LoaderUI/App.xaml.cs index d879953..ff41625 100644 --- a/Azaion.LoaderUI/App.xaml.cs +++ b/Azaion.LoaderUI/App.xaml.cs @@ -46,11 +46,5 @@ public partial class App host.Services.GetRequiredService().Show(); } - - - - //AFter: - //_loaderClient.Login(credentials); - //_loaderClient.Dispose(); } diff --git a/Azaion.LoaderUI/Azaion.LoaderUI.csproj b/Azaion.LoaderUI/Azaion.LoaderUI.csproj index 7d83672..a61de0e 100644 --- a/Azaion.LoaderUI/Azaion.LoaderUI.csproj +++ b/Azaion.LoaderUI/Azaion.LoaderUI.csproj @@ -24,7 +24,6 @@ - @@ -37,6 +36,14 @@ PreserveNewest + + + PreserveNewest + + + + + diff --git a/Azaion.LoaderUI/Login.xaml.cs b/Azaion.LoaderUI/Login.xaml.cs index dfb0539..f42bdf0 100644 --- a/Azaion.LoaderUI/Login.xaml.cs +++ b/Azaion.LoaderUI/Login.xaml.cs @@ -6,6 +6,7 @@ using System.Windows; using System.Windows.Controls; using System.Windows.Input; using System.Windows.Media; +using Azaion.Common; using MessagePack; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -57,22 +58,34 @@ public partial class Login TbStatus.Foreground = Brushes.Black; var installerVersion = await GetInstallerVer(); var localVersion = GetLocalVer(); + var credsEncrypted = Security.Encrypt(creds); if (installerVersion > localVersion) { TbStatus.Text = $"Updating from {localVersion} to {installerVersion}..."; - await DownloadAndRunInstaller(); - TbStatus.Text = $"Installed {installerVersion}!"; + var (installerName, stream) = await _azaionApi.DownloadInstaller(_dirConfig?.SuiteInstallerDirectory ?? ""); + var localFileStream = new FileStream(installerName, FileMode.Create, FileAccess.Write); + await stream.CopyToAsync(localFileStream); + localFileStream.Close(); + stream.Close(); + Process.Start(new ProcessStartInfo + { + FileName = "cmd.exe", + Arguments = $"/c updater.cmd {Process.GetCurrentProcess().Id} {installerName} {Constants.AZAION_SUITE_EXE} \"{credsEncrypted}\"" + }); } else + { TbStatus.Text = "Your version is up to date!"; - Process.Start(Constants.AZAION_SUITE_EXE, $"-e {creds.Email} -p {creds.Password}"); - await Task.Delay(800); - TbStatus.Text = "Loading..."; - while (!Process.GetProcessesByName(Constants.INFERENCE_EXE).Any()) - await Task.Delay(500); - await Task.Delay(1500); + Process.Start(Constants.AZAION_SUITE_EXE, $"-c {credsEncrypted}"); + await Task.Delay(800); + TbStatus.Text = "Loading..."; + while (!Process.GetProcessesByName(Constants.INFERENCE_EXE).Any()) + await Task.Delay(500); + await Task.Delay(1500); + } + Close(); } catch (Exception exception) @@ -146,26 +159,7 @@ public partial class Login throw; } } - - - private async Task DownloadAndRunInstaller() - { - var (installerName, stream) = await _azaionApi.DownloadInstaller(_dirConfig?.SuiteInstallerDirectory ?? ""); - var localFileStream = new FileStream(installerName, FileMode.Create, FileAccess.Write); - await stream.CopyToAsync(localFileStream); - localFileStream.Close(); - stream.Close(); - var processInfo = new ProcessStartInfo(installerName) - { - UseShellExecute = true, - Arguments = "/VERYSILENT" - }; - - var process = Process.Start(processInfo); - await process!.WaitForExitAsync(); - File.Delete(installerName); - } - + private async Task GetInstallerVer() { TbStatus.Text = "Checking for the newer version..."; diff --git a/Azaion.LoaderUI/updater.cmd b/Azaion.LoaderUI/updater.cmd new file mode 100644 index 0000000..464a1a9 --- /dev/null +++ b/Azaion.LoaderUI/updater.cmd @@ -0,0 +1,39 @@ +@echo off +setlocal + +REM Verify that all four arguments were provided +if "%~4"=="" ( + echo Error: Missing arguments. + echo Usage: %0 ^ ^ ^ ^ + exit /b 1 +) + +set "PARENT_PID=%1" +set "INSTALLER_PATH=%2" +set "MAIN_APP_PATH=%3" +set "CREDS=%~4" + +:WAIT_FOR_PARENT_EXIT +echo Waiting for parent process (PID: %PARENT_PID%) to close... +tasklist /fi "pid eq %PARENT_PID%" | find "%PARENT_PID%" >nul +if %errorlevel% == 0 ( + timeout /t 1 /nobreak >nul + goto WAIT_FOR_PARENT_EXIT +) + +start "" /wait "%INSTALLER_PATH%" /VERYSILENT + +del "%INSTALLER_PATH%" +echo Installed new version %INSTALLER_PATH% + +start "" "%MAIN_APP_PATH%" -c "%CREDS%" + +echo Loading... +:WAIT_FOR_APP_START +timeout /t 1 /nobreak >nul +tasklist /fi "imagename eq azaion-inference.exe" | find "azaion-inference.exe" >nul +if %errorlevel% neq 0 goto WAIT_FOR_APP_START + +timeout /t 5 /nobreak >nul +echo Process started. +endlocal \ No newline at end of file diff --git a/Azaion.Suite/App.xaml.cs b/Azaion.Suite/App.xaml.cs index d392ea8..d5dfc5f 100644 --- a/Azaion.Suite/App.xaml.cs +++ b/Azaion.Suite/App.xaml.cs @@ -54,11 +54,12 @@ public partial class App rollingInterval: RollingInterval.Day) .CreateLogger(); - Parser.Default.ParseArguments(e.Args) - .WithParsed(Start) + Parser.Default.ParseArguments(e.Args) + .WithParsed(Start) + .WithParsed(StartEncrypted) .WithNotParsed(ErrorHandling); } - + private void ErrorHandling(IEnumerable obj) { Log.Fatal($"Error happened: {string.Join(",", obj.Select(x => @@ -70,30 +71,11 @@ public partial class App Current.Shutdown(); } - private Stream GetSystemConfig(LoaderClient loaderClient, string apiDir) + private Stream GetConfig(LoaderClient loaderClient, string filename, string? apiDir) { try { - return loaderClient.LoadFile("config.system.json", apiDir); - } - catch (Exception e) - { - Log.Logger.Error(e, e.Message); - return new MemoryStream(Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new - { - AnnotationConfig = Constants.DefaultAnnotationConfig, - AIRecognitionConfig = Constants.DefaultAIRecognitionConfig, - GpsDeniedConfig = Constants.DefaultGpsDeniedConfig, - ThumbnailConfig = Constants.DefaultThumbnailConfig, - }))); - } - } - - private Stream GetSecuredConfig(LoaderClient loaderClient, string apiDir) - { - try - { - return loaderClient.LoadFile("config.secured.json", apiDir); + return loaderClient.LoadFile(filename, apiDir ?? ""); } catch (Exception e) { @@ -102,6 +84,13 @@ public partial class App } } + private void StartEncrypted(ApiCredentialsEncrypted credsEncrypted) + { + Log.Logger.Information(credsEncrypted.Creds); + Start(Security.Decrypt(credsEncrypted.Creds)); + } + + private void Start(ApiCredentials credentials) { try @@ -109,8 +98,8 @@ public partial class App new ConfigUpdater().CheckConfig(); var initConfig = Constants.ReadInitConfig(Log.Logger); var apiDir = initConfig.DirectoriesConfig.ApiResourcesDirectory; + _loaderClient = new LoaderClient(initConfig.LoaderClientConfig, Log.Logger, _mainCTokenSource.Token); - _loaderClient.StartClient(); _loaderClient.Connect(); _loaderClient.Login(credentials); @@ -121,8 +110,8 @@ public partial class App .ConfigureAppConfiguration((_, config) => config .AddCommandLine(Environment.GetCommandLineArgs()) .AddJsonFile(Constants.CONFIG_PATH, optional: true, reloadOnChange: true) - .AddJsonStream(GetSystemConfig(_loaderClient, apiDir)) - .AddJsonStream(GetSecuredConfig(_loaderClient, apiDir))) + .AddJsonStream(GetConfig(_loaderClient, "config.system.json", apiDir)) + .AddJsonStream(GetConfig(_loaderClient, "config.secured.json", apiDir))) .UseSerilog() .ConfigureServices((context, services) => { diff --git a/build/build_dotnet.cmd b/build/build_dotnet.cmd index 2da085a..9384501 100644 --- a/build/build_dotnet.cmd +++ b/build/build_dotnet.cmd @@ -13,8 +13,8 @@ del dist\config.json robocopy "dist" "dist-azaion" "Azaion.Annotator.dll" "Azaion.Dataset.dll" "Azaion.Common.dll" "Azaion.CommonSecurity.dll" /MOV robocopy "dist" "dist-azaion" "Azaion.Suite.dll" "Azaion.Suite.exe" "Azaion.Suite.runtimeconfig.json" "Azaion.Suite.deps.json" "logo.png" /MOV -robocopy "Azaion.LoaderUI\bin\Release\net8.0-windows\win-x64\publish" "dist-dlls" "Azaion.LoaderUI.dll" "Azaion.LoaderUI.exe" "Azaion.LoaderUI.runtimeconfig.json" ^ - "Azaion.LoaderUI.deps.json" "loaderconfig.json" +robocopy "Azaion.LoaderUI\bin\Release\net8.0-windows\win-x64\publish" "dist-azaion" "Azaion.LoaderUI.dll" "Azaion.LoaderUI.exe" "Azaion.LoaderUI.runtimeconfig.json" "Azaion.LoaderUI.deps.json" "loaderconfig.json" /MOV +robocopy "Azaion.LoaderUI\bin\Release\net8.0-windows\win-x64\publish" "dist-dlls" "updater.cmd" /MOV move dist\config.production.json dist-azaion\config_updated.json diff --git a/build/publish.cmd b/build/publish.cmd index ce16cb0..4831551 100644 --- a/build/publish.cmd +++ b/build/publish.cmd @@ -19,8 +19,8 @@ echo building and upload iterative installer... iscc build\installer.iterative.iss call build\upload.cmd "suite-dev" -echo building full installer -iscc build\installer.full.iss +@rem echo building full installer +@rem iscc build\installer.full.iss cd /d %CURRENT_DIR% echo Done! \ No newline at end of file From 938fd36aeceb595ac518f2571e5712f4cdff65d0 Mon Sep 17 00:00:00 2001 From: Alex Bezdieniezhnykh Date: Tue, 8 Jul 2025 19:33:22 +0300 Subject: [PATCH 03/14] fix zooming in map matcher --- Azaion.Annotator/Controls/MapMatcher.xaml | 14 +++++++------- Azaion.Common/SecurityConstants.cs | 2 +- Azaion.Common/Services/GPSMatcherService.cs | 4 ++-- Azaion.Suite/config.system.json | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Azaion.Annotator/Controls/MapMatcher.xaml b/Azaion.Annotator/Controls/MapMatcher.xaml index 22c9e35..22dbfd0 100644 --- a/Azaion.Annotator/Controls/MapMatcher.xaml +++ b/Azaion.Annotator/Controls/MapMatcher.xaml @@ -137,13 +137,13 @@ HorizontalAlignment="Stretch" VerticalAlignment="Stretch"/> - - - + + + + gpsDeniedConfig.Value.MinKeyPoints) + if (result.KeyPoints >= gpsDeniedConfig.Value.MinKeyPoints) { var direction = _lastGeoPoint.DirectionTo(result.GeoPoint); _directions.Enqueue(direction); diff --git a/Azaion.Suite/config.system.json b/Azaion.Suite/config.system.json index aa45128..c6f777f 100644 --- a/Azaion.Suite/config.system.json +++ b/Azaion.Suite/config.system.json @@ -35,7 +35,7 @@ "ModelBatchSize": 4 }, "GpsDeniedConfig": { - "MinKeyPoints": 12 + "MinKeyPoints": 11 }, "ThumbnailConfig": { "Size": "240,135", "Border": 10 } } \ No newline at end of file From fefd054ea0dce6379287dbb06e280a61a580556c Mon Sep 17 00:00:00 2001 From: Alex Bezdieniezhnykh Date: Fri, 11 Jul 2025 22:46:25 +0300 Subject: [PATCH 04/14] fixed selection on editor fixed image view and play --- Azaion.Annotator/Annotator.xaml.cs | 4 ++-- Azaion.Annotator/AnnotatorEventHandler.cs | 6 ++--- Azaion.Common/Controls/CanvasEditor.cs | 28 +++++++++++------------ 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Azaion.Annotator/Annotator.xaml.cs b/Azaion.Annotator/Annotator.xaml.cs index 958cce2..bc2091f 100644 --- a/Azaion.Annotator/Annotator.xaml.cs +++ b/Azaion.Annotator/Annotator.xaml.cs @@ -304,7 +304,7 @@ public partial class Annotator { if (File.Exists(annotation.ImagePath)) { - Editor.SetImageSource(await annotation.ImagePath.OpenImage()); + Editor.SetBackground(await annotation.ImagePath.OpenImage()); _formState.BackgroundTime = annotation.Time; videoSize = Editor.RenderSize; } @@ -534,7 +534,7 @@ public partial class Annotator if (LvFiles.SelectedIndex == -1) LvFiles.SelectedIndex = 0; - Dispatcher.Invoke(() => Editor.ResetBackground()); + Dispatcher.Invoke(() => Editor.SetBackground(null)); IsInferenceNow = true; AIDetectBtn.IsEnabled = false; diff --git a/Azaion.Annotator/AnnotatorEventHandler.cs b/Azaion.Annotator/AnnotatorEventHandler.cs index 37d1aba..92dc415 100644 --- a/Azaion.Annotator/AnnotatorEventHandler.cs +++ b/Azaion.Annotator/AnnotatorEventHandler.cs @@ -143,7 +143,7 @@ public class AnnotatorEventHandler( if (formState.BackgroundTime.HasValue) { - mainWindow.Editor.ResetBackground(); + mainWindow.Editor.SetBackground(null); formState.BackgroundTime = null; } break; @@ -226,7 +226,7 @@ public class AnnotatorEventHandler( if (mainWindow.LvFiles.SelectedItem == null) return; var mediaInfo = (MediaFileInfo)mainWindow.LvFiles.SelectedItem; - mainWindow.Editor.ResetBackground(); + mainWindow.Editor.SetBackground(null); formState.CurrentMedia = mediaInfo; mainWindow.Title = $"Azaion Annotator - {mediaInfo.Name}"; @@ -262,7 +262,7 @@ public class AnnotatorEventHandler( if (formState.BackgroundTime.HasValue) { //no need to save image, it's already there, just remove background - mainWindow.Editor.ResetBackground(); + mainWindow.Editor.SetBackground(null); formState.BackgroundTime = null; //next item diff --git a/Azaion.Common/Controls/CanvasEditor.cs b/Azaion.Common/Controls/CanvasEditor.cs index c2dd310..15ab798 100644 --- a/Azaion.Common/Controls/CanvasEditor.cs +++ b/Azaion.Common/Controls/CanvasEditor.cs @@ -124,7 +124,7 @@ public class CanvasEditor : Canvas MouseWheel += CanvasWheel; } - public void SetImageSource(ImageSource? source) + public void SetBackground(ImageSource? source) { SetZoom(); _backgroundImage.Source = source; @@ -228,20 +228,20 @@ public class CanvasEditor : Canvas _newAnnotationRect.Height = 0; var width = Math.Abs(endPos.X - _newAnnotationStartPos.X); var height = Math.Abs(endPos.Y - _newAnnotationStartPos.Y); - if (width < MIN_SIZE || height < MIN_SIZE) - return; - - var time = GetTimeFunc(); - var control = CreateDetectionControl(CurrentAnnClass, time, new CanvasLabel + if (width >= MIN_SIZE && height >= MIN_SIZE) { - Width = width, - Height = height, - X = Math.Min(endPos.X, _newAnnotationStartPos.X), - Y = Math.Min(endPos.Y, _newAnnotationStartPos.Y), - Confidence = 1 - }); - control.UpdateLayout(); - CheckLabelBoundaries(control); + var time = GetTimeFunc(); + var control = CreateDetectionControl(CurrentAnnClass, time, new CanvasLabel + { + Width = width, + Height = height, + X = Math.Min(endPos.X, _newAnnotationStartPos.X), + Y = Math.Min(endPos.Y, _newAnnotationStartPos.Y), + Confidence = 1 + }); + control.UpdateLayout(); + CheckLabelBoundaries(control); + } } else if (SelectionState != SelectionState.PanZoomMoving) CheckLabelBoundaries(_curAnn); From fc6e5db795f029c579969043a2bb679c9cab0b61 Mon Sep 17 00:00:00 2001 From: Oleksandr Bezdieniezhnykh Date: Mon, 28 Jul 2025 12:39:52 +0300 Subject: [PATCH 05/14] add manual Tile Processor zoom on video on pause (temp image) --- .gitignore | 3 +- Azaion.Annotator/Annotator.xaml.cs | 23 +-- Azaion.Annotator/AnnotatorEventHandler.cs | 184 ++++++++++++------ .../{SecurityConstants.cs => Constants.cs} | 30 ++- Azaion.Common/Controls/CanvasEditor.cs | 25 ++- Azaion.Common/Controls/DetectionControl.cs | 42 ++-- .../Controls/DetectionLabelPanel.xaml | 59 ++++++ .../Controls/DetectionLabelPanel.xaml.cs | 55 ++++++ Azaion.Common/DTO/AffiliationEnum.cs | 9 + .../DTO/Config/AIRecognitionConfig.cs | 1 + Azaion.Common/DTO/FormState.cs | 4 +- Azaion.Common/DTO/Label.cs | 39 +++- Azaion.Common/Database/DbFactory.cs | 4 +- Azaion.Common/Database/SchemaMigrator.cs | 94 +++++++++ Azaion.Common/Services/TileProcessor.cs | 82 ++++++++ Azaion.Dataset/DatasetExplorerEventHandler.cs | 2 +- Azaion.Inference/README.md | 24 +-- Azaion.Inference/ai_config.pxd | 4 +- Azaion.Inference/ai_config.pyx | 4 + Azaion.Inference/annotation.pxd | 2 +- Azaion.Inference/annotation.pyx | 4 +- Azaion.Inference/inference.pxd | 6 +- Azaion.Inference/inference.pyx | 73 +++++-- Azaion.Inference/requirements.txt | 7 +- Azaion.Inference/setup.py | 42 ++-- Azaion.Inference/setup_old.py | 37 ++++ Azaion.Inference/test/test_inference.py | 8 + Azaion.Loader/requirements.txt | 2 +- Azaion.LoaderUI/App.xaml.cs | 5 +- Azaion.LoaderUI/Constants.cs | 13 -- Azaion.LoaderUI/ConstantsLoader.cs | 7 + Azaion.LoaderUI/Login.xaml.cs | 20 +- Azaion.Suite/App.xaml.cs | 8 +- Azaion.Suite/config.system.json | 3 +- 34 files changed, 716 insertions(+), 209 deletions(-) rename Azaion.Common/{SecurityConstants.cs => Constants.cs} (89%) create mode 100644 Azaion.Common/Controls/DetectionLabelPanel.xaml create mode 100644 Azaion.Common/Controls/DetectionLabelPanel.xaml.cs create mode 100644 Azaion.Common/DTO/AffiliationEnum.cs create mode 100644 Azaion.Common/Database/SchemaMigrator.cs create mode 100644 Azaion.Common/Services/TileProcessor.cs create mode 100644 Azaion.Inference/setup_old.py create mode 100644 Azaion.Inference/test/test_inference.py delete mode 100644 Azaion.LoaderUI/Constants.cs create mode 100644 Azaion.LoaderUI/ConstantsLoader.cs diff --git a/.gitignore b/.gitignore index 5f9d1be..fe8a470 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,5 @@ Azaion*.bin azaion\.*\.big _internal *.spec -dist \ No newline at end of file +dist +*.jpg diff --git a/Azaion.Annotator/Annotator.xaml.cs b/Azaion.Annotator/Annotator.xaml.cs index bc2091f..13bfb44 100644 --- a/Azaion.Annotator/Annotator.xaml.cs +++ b/Azaion.Annotator/Annotator.xaml.cs @@ -56,6 +56,7 @@ public partial class Annotator public Dictionary MediaFilesDict = new(); public IntervalTree TimedAnnotations { get; set; } = new(); + public string MainTitle { get; set; } public Annotator( IConfigUpdater configUpdater, @@ -72,7 +73,9 @@ public partial class Annotator IGpsMatcherService gpsMatcherService) { InitializeComponent(); - + + MainTitle = $"Azaion Annotator {Constants.GetLocalVersion()}"; + Title = MainTitle; _appConfig = appConfig.Value; _configUpdater = configUpdater; _libVLC = libVLC; @@ -194,7 +197,7 @@ public partial class Annotator _formState.CurrentMrl = _mediaPlayer.Media?.Mrl ?? ""; uint vw = 0, vh = 0; _mediaPlayer.Size(0, ref vw, ref vh); - _formState.CurrentVideoSize = new Size(vw, vh); + _formState.CurrentMediaSize = new Size(vw, vh); _formState.CurrentVideoLength = TimeSpan.FromMilliseconds(_mediaPlayer.Length); }; @@ -289,27 +292,23 @@ public partial class Annotator StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}"; Editor.ClearExpiredAnnotations(time); }); - - ShowAnnotation(TimedAnnotations.Query(time).FirstOrDefault(), showImage); + var annotation = TimedAnnotations.Query(time).FirstOrDefault(); + if (annotation != null) ShowAnnotation(annotation, showImage); } - private void ShowAnnotation(Annotation? annotation, bool showImage = false) + private void ShowAnnotation(Annotation annotation, bool showImage = false) { - if (annotation == null) - return; Dispatcher.Invoke(async () => { - var videoSize = _formState.CurrentVideoSize; if (showImage) { if (File.Exists(annotation.ImagePath)) { Editor.SetBackground(await annotation.ImagePath.OpenImage()); _formState.BackgroundTime = annotation.Time; - videoSize = Editor.RenderSize; } } - Editor.CreateDetections(annotation.Time, annotation.Detections, _appConfig.AnnotationConfig.DetectionClasses, videoSize); + Editor.CreateDetections(annotation.Time, annotation.Detections, _appConfig.AnnotationConfig.DetectionClasses, _formState.CurrentMediaSize); }); } @@ -321,7 +320,7 @@ public partial class Annotator var annotations = await _dbFactory.Run(async db => await db.Annotations.LoadWith(x => x.Detections) - .Where(x => x.OriginalMediaName == _formState.VideoName) + .Where(x => x.OriginalMediaName == _formState.MediaName) .OrderBy(x => x.Time) .ToListAsync(token: MainCancellationSource.Token)); @@ -583,13 +582,11 @@ public partial class Annotator private void SoundDetections(object sender, RoutedEventArgs e) { MessageBox.Show("Функція Аудіоаналіз знаходиться в стадії розробки","Система", MessageBoxButton.OK, MessageBoxImage.Information); - _logger.LogInformation("Denys wishes #1. To be implemented"); } private void RunDroneMaintenance(object sender, RoutedEventArgs e) { MessageBox.Show("Функція Аналіз стану БПЛА знаходиться в стадії розробки","Система", MessageBoxButton.OK, MessageBoxImage.Information); - _logger.LogInformation("Denys wishes #2. To be implemented"); } #endregion diff --git a/Azaion.Annotator/AnnotatorEventHandler.cs b/Azaion.Annotator/AnnotatorEventHandler.cs index 92dc415..0607530 100644 --- a/Azaion.Annotator/AnnotatorEventHandler.cs +++ b/Azaion.Annotator/AnnotatorEventHandler.cs @@ -2,6 +2,7 @@ using System.Windows; using System.Windows.Input; using System.Windows.Media; +using System.Windows.Media.Imaging; using Azaion.Annotator.Controls; using Azaion.Annotator.DTO; using Azaion.Common; @@ -47,7 +48,8 @@ public class AnnotatorEventHandler( private const int STEP = 20; private const int LARGE_STEP = 5000; private const int RESULT_WIDTH = 1280; - + private readonly string tempImgPath = Path.Combine(dirConfig.Value.ImagesDirectory, "___temp___.jpg"); + private readonly Dictionary _keysControlEnumDict = new() { { Key.Space, PlaybackControlEnum.Pause }, @@ -139,12 +141,21 @@ public class AnnotatorEventHandler( await Play(cancellationToken); break; case PlaybackControlEnum.Pause: - mediaPlayer.Pause(); - - if (formState.BackgroundTime.HasValue) + if (mediaPlayer.IsPlaying) { - mainWindow.Editor.SetBackground(null); - formState.BackgroundTime = null; + mediaPlayer.Pause(); + mediaPlayer.TakeSnapshot(0, tempImgPath, 0, 0); + mainWindow.Editor.SetBackground(await tempImgPath.OpenImage()); + formState.BackgroundTime = TimeSpan.FromMilliseconds(mediaPlayer.Time); + } + else + { + mediaPlayer.Play(); + if (formState.BackgroundTime.HasValue) + { + mainWindow.Editor.SetBackground(null); + formState.BackgroundTime = null; + } } break; case PlaybackControlEnum.Stop: @@ -159,7 +170,7 @@ public class AnnotatorEventHandler( mainWindow.SeekTo(mediaPlayer.Time + step); break; case PlaybackControlEnum.SaveAnnotations: - await SaveAnnotations(cancellationToken); + await SaveAnnotation(cancellationToken); break; case PlaybackControlEnum.RemoveSelectedAnns: @@ -226,63 +237,125 @@ public class AnnotatorEventHandler( if (mainWindow.LvFiles.SelectedItem == null) return; var mediaInfo = (MediaFileInfo)mainWindow.LvFiles.SelectedItem; - mainWindow.Editor.SetBackground(null); - + formState.CurrentMedia = mediaInfo; - mainWindow.Title = $"Azaion Annotator - {mediaInfo.Name}"; - - - //need to wait a bit for correct VLC playback event handling - await Task.Delay(100, ct); - mediaPlayer.Stop(); - mediaPlayer.Play(new Media(libVLC, mediaInfo.Path)); + mainWindow.Title = $"{mainWindow.MainTitle} - {mediaInfo.Name}"; + + if (mediaInfo.MediaType == MediaTypes.Video) + { + mainWindow.Editor.SetBackground(null); + //need to wait a bit for correct VLC playback event handling + await Task.Delay(100, ct); + mediaPlayer.Stop(); + mediaPlayer.Play(new Media(libVLC, mediaInfo.Path)); + } + else + { + formState.BackgroundTime = TimeSpan.Zero; + var image = await mediaInfo.Path.OpenImage(); + formState.CurrentMediaSize = new Size(image.PixelWidth, image.PixelHeight); + mainWindow.Editor.SetBackground(image); + mediaPlayer.Stop(); + } } //SAVE: MANUAL - private async Task SaveAnnotations(CancellationToken cancellationToken = default) + private async Task SaveAnnotation(CancellationToken cancellationToken = default) { if (formState.CurrentMedia == null) return; var time = formState.BackgroundTime ?? TimeSpan.FromMilliseconds(mediaPlayer.Time); - var originalMediaName = formState.VideoName; - var fName = originalMediaName.ToTimeName(time); - - var currentDetections = mainWindow.Editor.CurrentDetections - .Select(x => new Detection(fName, x.GetLabel(mainWindow.Editor.RenderSize, formState.BackgroundTime.HasValue ? mainWindow.Editor.RenderSize : formState.CurrentVideoSize))) - .ToList(); - - formState.CurrentMedia.HasAnnotations = currentDetections.Count != 0; - mainWindow.LvFiles.Items.Refresh(); - mainWindow.Editor.RemoveAllAnns(); - + var timeName = formState.MediaName.ToTimeName(time); var isVideo = formState.CurrentMedia.MediaType == MediaTypes.Video; - var imgPath = Path.Combine(dirConfig.Value.ImagesDirectory, $"{fName}{Constants.JPG_EXT}"); + var imgPath = Path.Combine(dirConfig.Value.ImagesDirectory, $"{timeName}{Constants.JPG_EXT}"); - if (formState.BackgroundTime.HasValue) + formState.CurrentMedia.HasAnnotations = mainWindow.Editor.CurrentDetections.Count != 0; + var annotations = await SaveAnnotationInner(imgPath, cancellationToken); + if (isVideo) { - //no need to save image, it's already there, just remove background - mainWindow.Editor.SetBackground(null); - formState.BackgroundTime = null; - - //next item - var annGrid = mainWindow.DgAnnotations; - annGrid.SelectedIndex = Math.Min(annGrid.Items.Count, annGrid.SelectedIndex + 1); - mainWindow.OpenAnnotationResult((AnnotationResult)annGrid.SelectedItem); + foreach (var annotation in annotations) + mainWindow.AddAnnotation(annotation); + mediaPlayer.Play(); + + // next item. Probably not needed + // var annGrid = mainWindow.DgAnnotations; + // annGrid.SelectedIndex = Math.Min(annGrid.Items.Count, annGrid.SelectedIndex + 1); + // mainWindow.OpenAnnotationResult((AnnotationResult)annGrid.SelectedItem); } else { - var resultHeight = (uint)Math.Round(RESULT_WIDTH / formState.CurrentVideoSize.Width * formState.CurrentVideoSize.Height); - mediaPlayer.TakeSnapshot(0, imgPath, RESULT_WIDTH, resultHeight); - if (isVideo) - mediaPlayer.Play(); - else - await NextMedia(ct: cancellationToken); + await NextMedia(ct: cancellationToken); } + mainWindow.Editor.SetBackground(null); + formState.BackgroundTime = null; + + mainWindow.LvFiles.Items.Refresh(); + mainWindow.Editor.RemoveAllAnns(); + } - var annotation = await annotationService.SaveAnnotation(originalMediaName, time, currentDetections, token: cancellationToken); - if (isVideo) - mainWindow.AddAnnotation(annotation); + private async Task> SaveAnnotationInner(string imgPath, CancellationToken cancellationToken = default) + { + var canvasDetections = mainWindow.Editor.CurrentDetections.Select(x => x.ToCanvasLabel()).ToList(); + var annotationsResult = new List(); + if (!File.Exists(imgPath)) + { + var source = (mainWindow.Editor.BackgroundImage.Source as BitmapSource)!; + if (source.PixelWidth <= RESULT_WIDTH * 2 && source.PixelHeight <= RESULT_WIDTH * 2) // Allow to be up to 2560*2560 to save to 1280*1280 + { + //Save image + await using var stream = new FileStream(imgPath, FileMode.Create); + var encoder = new JpegBitmapEncoder(); + encoder.Frames.Add(BitmapFrame.Create(source)); + encoder.Save(stream); + await stream.FlushAsync(cancellationToken); + } + else + { + //Tiling + + //1. Restore original picture coordinates + var pictureCoordinatesDetections = canvasDetections.Select(x => new CanvasLabel( + new YoloLabel(x, mainWindow.Editor.RenderSize, formState.CurrentMediaSize), formState.CurrentMediaSize, null, x.Confidence)) + .ToList(); + + //2. Split to 1280*1280 frames + var results = TileProcessor.Split(formState.CurrentMediaSize, pictureCoordinatesDetections, cancellationToken); + + //3. Save each frame as a separate annotation + BitmapEncoder tileEncoder = new JpegBitmapEncoder(); + foreach (var res in results) + { + var mediaName = $"{formState.MediaName}!split!{res.Tile.X}_{res.Tile.Y}!"; + var time = TimeSpan.Zero; + var annotationName = mediaName.ToTimeName(time); + + var tileImgPath = Path.Combine(dirConfig.Value.ImagesDirectory, $"{annotationName}{Constants.JPG_EXT}"); + await using var tileStream = new FileStream(tileImgPath, FileMode.Create); + var bitmap = new CroppedBitmap(source, new Int32Rect((int)res.Tile.X, (int)res.Tile.Y, (int)res.Tile.Width, (int)res.Tile.Height)); + tileEncoder.Frames.Add(BitmapFrame.Create(bitmap)); + tileEncoder.Save(tileStream); + await tileStream.FlushAsync(cancellationToken); + + var frameSize = new Size(res.Tile.Width, res.Tile.Height); + var detections = res.Detections + .Select(det => det.ReframeToSmall(res.Tile)) + .Select(x => new Detection(annotationName, new YoloLabel(x, frameSize))) + .ToList(); + + annotationsResult.Add(await annotationService.SaveAnnotation(mediaName, time, detections, token: cancellationToken)); + } + return annotationsResult; + } + } + + var timeImg = formState.BackgroundTime ?? TimeSpan.FromMilliseconds(mediaPlayer.Time); + var timeName = formState.MediaName.ToTimeName(timeImg); + var currentDetections = canvasDetections.Select(x => + new Detection(timeName, new YoloLabel(x, mainWindow.Editor.RenderSize))) + .ToList(); + var annotation = await annotationService.SaveAnnotation(formState.MediaName, timeImg, currentDetections, token: cancellationToken); + return [annotation]; } public async Task Handle(AnnotationsDeletedEvent notification, CancellationToken ct) @@ -316,21 +389,20 @@ public class AnnotatorEventHandler( }); await dbFactory.DeleteAnnotations(notification.AnnotationNames, ct); - - try + + foreach (var name in notification.AnnotationNames) { - foreach (var name in notification.AnnotationNames) + try { File.Delete(Path.Combine(dirConfig.Value.ImagesDirectory, $"{name}{Constants.JPG_EXT}")); File.Delete(Path.Combine(dirConfig.Value.LabelsDirectory, $"{name}{Constants.TXT_EXT}")); File.Delete(Path.Combine(dirConfig.Value.ThumbnailsDirectory, $"{name}{Constants.THUMBNAIL_PREFIX}{Constants.JPG_EXT}")); File.Delete(Path.Combine(dirConfig.Value.ResultsDirectory, $"{name}{Constants.RESULT_PREFIX}{Constants.JPG_EXT}")); } - } - catch (Exception e) - { - logger.LogError(e, e.Message); - throw; + catch (Exception e) + { + logger.LogError(e, e.Message); + } } //Only validators can send Delete to the queue @@ -403,4 +475,4 @@ public class AnnotatorEventHandler( map.SatelliteMap.Position = pointLatLon; map.SatelliteMap.ZoomAndCenterMarkers(null); } -} +} \ No newline at end of file diff --git a/Azaion.Common/SecurityConstants.cs b/Azaion.Common/Constants.cs similarity index 89% rename from Azaion.Common/SecurityConstants.cs rename to Azaion.Common/Constants.cs index eaa82d0..b92b8aa 100644 --- a/Azaion.Common/SecurityConstants.cs +++ b/Azaion.Common/Constants.cs @@ -1,4 +1,5 @@ -using System.IO; +using System.Diagnostics; +using System.IO; using Azaion.Common.DTO; using Azaion.Common.DTO.Config; using Azaion.Common.Extensions; @@ -11,9 +12,10 @@ namespace Azaion.Common; public class Constants { public const string CONFIG_PATH = "config.json"; - - private const string DEFAULT_API_URL = "https://api.azaion.com"; - + public const string LOADER_CONFIG_PATH = "loaderconfig.json"; + public const string DEFAULT_API_URL = "https://api.azaion.com"; + public const string AZAION_SUITE_EXE = "Azaion.Suite.exe"; + #region ExternalClientsConfig private const string DEFAULT_ZMQ_LOADER_HOST = "127.0.0.1"; @@ -103,14 +105,16 @@ public class Constants TrackingDistanceConfidence = TRACKING_DISTANCE_CONFIDENCE, TrackingProbabilityIncrease = TRACKING_PROBABILITY_INCREASE, TrackingIntersectionThreshold = TRACKING_INTERSECTION_THRESHOLD, + BigImageTileOverlapPercent = DEFAULT_BIG_IMAGE_TILE_OVERLAP_PERCENT, FramePeriodRecognition = DEFAULT_FRAME_PERIOD_RECOGNITION }; - public const double DEFAULT_FRAME_RECOGNITION_SECONDS = 2; - public const double TRACKING_DISTANCE_CONFIDENCE = 0.15; - public const double TRACKING_PROBABILITY_INCREASE = 15; - public const double TRACKING_INTERSECTION_THRESHOLD = 0.8; - public const int DEFAULT_FRAME_PERIOD_RECOGNITION = 4; + public const double DEFAULT_FRAME_RECOGNITION_SECONDS = 2; + public const double TRACKING_DISTANCE_CONFIDENCE = 0.15; + public const double TRACKING_PROBABILITY_INCREASE = 15; + public const double TRACKING_INTERSECTION_THRESHOLD = 0.8; + public const int DEFAULT_BIG_IMAGE_TILE_OVERLAP_PERCENT = 20; + public const int DEFAULT_FRAME_PERIOD_RECOGNITION = 4; # endregion AIRecognitionConfig @@ -251,4 +255,12 @@ public class Constants return DefaultInitConfig; } } + + public static Version GetLocalVersion() + { + var localFileInfo = FileVersionInfo.GetVersionInfo(AZAION_SUITE_EXE); + if (string.IsNullOrWhiteSpace(localFileInfo.ProductVersion)) + throw new Exception($"Can't find {AZAION_SUITE_EXE} and its version"); + return new Version(localFileInfo.FileVersion!); + } } \ No newline at end of file diff --git a/Azaion.Common/Controls/CanvasEditor.cs b/Azaion.Common/Controls/CanvasEditor.cs index 15ab798..48ea36c 100644 --- a/Azaion.Common/Controls/CanvasEditor.cs +++ b/Azaion.Common/Controls/CanvasEditor.cs @@ -6,6 +6,7 @@ using System.Windows.Media; using System.Windows.Media.Imaging; using System.Windows.Shapes; using Azaion.Common.DTO; +using Azaion.Common.Events; using MediatR; using Color = System.Windows.Media.Color; using Image = System.Windows.Controls.Image; @@ -34,10 +35,10 @@ public class CanvasEditor : Canvas private Point _panStartPoint; private bool _isZoomedIn; - private const int MIN_SIZE = 20; + private const int MIN_SIZE = 12; private readonly TimeSpan _viewThreshold = TimeSpan.FromMilliseconds(400); - private Image _backgroundImage { get; set; } = new() { Stretch = Stretch.Fill }; + public Image BackgroundImage { get; set; } = new() { Stretch = Stretch.Uniform }; public IMediator Mediator { get; set; } = null!; public static readonly DependencyProperty GetTimeFuncProp = @@ -113,7 +114,7 @@ public class CanvasEditor : Canvas MouseUp += CanvasMouseUp; SizeChanged += CanvasResized; Cursor = Cursors.Cross; - Children.Insert(0, _backgroundImage); + Children.Insert(0, BackgroundImage); Children.Add(_newAnnotationRect); Children.Add(_horizontalLine); Children.Add(_verticalLine); @@ -127,7 +128,7 @@ public class CanvasEditor : Canvas public void SetBackground(ImageSource? source) { SetZoom(); - _backgroundImage.Source = source; + BackgroundImage.Source = source; } private void SetZoom(Matrix? matrix = null) @@ -142,8 +143,8 @@ public class CanvasEditor : Canvas _matrixTransform.Matrix = matrix.Value; _isZoomedIn = true; } - foreach (var detection in CurrentDetections) - detection.UpdateAdornerScale(scale: _matrixTransform.Matrix.M11); + // foreach (var detection in CurrentDetections) + // detection.UpdateAdornerScale(scale: _matrixTransform.Matrix.M11); } private void CanvasWheel(object sender, MouseWheelEventArgs e) @@ -175,6 +176,8 @@ public class CanvasEditor : Canvas private void CanvasMouseDown(object sender, MouseButtonEventArgs e) { ClearSelections(); + if (e.LeftButton != MouseButtonState.Pressed) + return; if (Keyboard.Modifiers == ModifierKeys.Control && _isZoomedIn) { _panStartPoint = e.GetPosition(this); @@ -182,11 +185,13 @@ public class CanvasEditor : Canvas } else NewAnnotationStart(sender, e); + (sender as UIElement)?.CaptureMouse(); } private void CanvasMouseMove(object sender, MouseEventArgs e) { var pos = e.GetPosition(this); + Mediator.Publish(new SetStatusTextEvent($"Mouse Coordinates: {pos.X}, {pos.Y}")); _horizontalLine.Y1 = _horizontalLine.Y2 = pos.Y; _verticalLine.X1 = _verticalLine.X2 = pos.X; SetLeft(_classNameHint, pos.X + 10); @@ -216,11 +221,14 @@ public class CanvasEditor : Canvas var matrix = _matrixTransform.Matrix; matrix.Translate(delta.X, delta.Y); + _matrixTransform.Matrix = matrix; + Mediator.Publish(new SetStatusTextEvent(_matrixTransform.Matrix.ToString())); } private void CanvasMouseUp(object sender, MouseButtonEventArgs e) { + (sender as UIElement)?.ReleaseMouseCapture(); if (SelectionState == SelectionState.NewAnnCreating) { var endPos = e.GetPosition(this); @@ -279,8 +287,8 @@ public class CanvasEditor : Canvas { _horizontalLine.X2 = e.NewSize.Width; _verticalLine.Y2 = e.NewSize.Height; - _backgroundImage.Width = e.NewSize.Width; - _backgroundImage.Height = e.NewSize.Height; + BackgroundImage.Width = e.NewSize.Width; + BackgroundImage.Height = e.NewSize.Height; } #region Annotation Resizing & Moving @@ -383,7 +391,6 @@ public class CanvasEditor : Canvas private void NewAnnotationStart(object sender, MouseButtonEventArgs e) { _newAnnotationStartPos = e.GetPosition(this); - SetLeft(_newAnnotationRect, _newAnnotationStartPos.X); SetTop(_newAnnotationRect, _newAnnotationStartPos.Y); _newAnnotationRect.MouseMove += NewAnnotationCreatingMove; diff --git a/Azaion.Common/Controls/DetectionControl.cs b/Azaion.Common/Controls/DetectionControl.cs index 4c6bfe8..9974464 100644 --- a/Azaion.Common/Controls/DetectionControl.cs +++ b/Azaion.Common/Controls/DetectionControl.cs @@ -12,15 +12,15 @@ namespace Azaion.Common.Controls; public class DetectionControl : Border { private readonly Action _resizeStart; - private const double RESIZE_RECT_SIZE = 12; + private const double RESIZE_RECT_SIZE = 10; private readonly Grid _grid; - private readonly Label _detectionLabel; + private readonly DetectionLabelPanel _detectionLabelPanel; + //private readonly Label _detectionLabel; public readonly Canvas DetectionLabelContainer; public TimeSpan Time { get; set; } - private readonly double _confidence; - private List _resizedRectangles = new(); + private readonly List _resizedRectangles = new(); private DetectionClass _detectionClass = null!; public DetectionClass DetectionClass @@ -33,9 +33,8 @@ public class DetectionControl : Border BorderThickness = new Thickness(3); foreach (var rect in _resizedRectangles) rect.Stroke = brush; - - _detectionLabel.Background = new SolidColorBrush(value.Color.ToConfidenceColor(_confidence)); - _detectionLabel.Content = _detectionLabelText(value.UIName); + + _detectionLabelPanel.DetectionClass = value; _detectionClass = value; } } @@ -78,10 +77,7 @@ public class DetectionControl : Border DetectionLabelContainer.VerticalAlignment = value.Vertical; } } - - private string _detectionLabelText(string detectionClassName) => - _confidence >= 0.995 ? detectionClassName : $"{detectionClassName}: {_confidence * 100:F0}%"; //double - + public DetectionControl(DetectionClass detectionClass, TimeSpan time, Action resizeStart, CanvasLabel canvasLabel) { @@ -89,7 +85,6 @@ public class DetectionControl : Border Height = canvasLabel.Height; Time = time; _resizeStart = resizeStart; - _confidence = canvasLabel.Confidence; DetectionLabelContainer = new Canvas { @@ -97,16 +92,16 @@ public class DetectionControl : Border VerticalAlignment = VerticalAlignment.Top, ClipToBounds = false, }; - _detectionLabel = new Label + _detectionLabelPanel = new DetectionLabelPanel { - Content = _detectionLabelText(detectionClass.Name), - FontSize = 16, - Visibility = Visibility.Visible + Confidence = canvasLabel.Confidence }; - DetectionLabelContainer.Children.Add(_detectionLabel); + + DetectionLabelContainer.Children.Add(_detectionLabelPanel); _selectionFrame = new Rectangle { + Margin = new Thickness(-3), HorizontalAlignment = HorizontalAlignment.Stretch, VerticalAlignment = VerticalAlignment.Stretch, Stroke = new SolidColorBrush(Colors.Black), @@ -146,12 +141,13 @@ public class DetectionControl : Border var rect = new Rectangle() // small rectangles at the corners and sides { ClipToBounds = false, - Margin = new Thickness(-RESIZE_RECT_SIZE * 0.7), + Margin = new Thickness(-RESIZE_RECT_SIZE), HorizontalAlignment = ha, VerticalAlignment = va, Width = RESIZE_RECT_SIZE, Height = RESIZE_RECT_SIZE, Stroke = new SolidColorBrush(Color.FromArgb(230, 20, 20, 20)), // small rectangles color + StrokeThickness = 0.8, Fill = new SolidColorBrush(Color.FromArgb(150, 80, 80, 80)), Cursor = crs, Name = name, @@ -160,9 +156,9 @@ public class DetectionControl : Border return rect; } - public YoloLabel GetLabel(Size canvasSize, Size? videoSize = null) - { - var label = new CanvasLabel(DetectionClass.YoloId, Canvas.GetLeft(this), Canvas.GetTop(this), Width, Height); - return new YoloLabel(label, canvasSize, videoSize); - } + public CanvasLabel ToCanvasLabel() => + new(DetectionClass.YoloId, Canvas.GetLeft(this), Canvas.GetTop(this), Width, Height); + + public YoloLabel ToYoloLabel(Size canvasSize, Size? videoSize = null) => + new(ToCanvasLabel(), canvasSize, videoSize); } diff --git a/Azaion.Common/Controls/DetectionLabelPanel.xaml b/Azaion.Common/Controls/DetectionLabelPanel.xaml new file mode 100644 index 0000000..629bac4 --- /dev/null +++ b/Azaion.Common/Controls/DetectionLabelPanel.xaml @@ -0,0 +1,59 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Azaion.Common/Controls/DetectionLabelPanel.xaml.cs b/Azaion.Common/Controls/DetectionLabelPanel.xaml.cs new file mode 100644 index 0000000..cb596fb --- /dev/null +++ b/Azaion.Common/Controls/DetectionLabelPanel.xaml.cs @@ -0,0 +1,55 @@ +using System.Windows.Media; +using Azaion.Common.DTO; + +namespace Azaion.Common.Controls +{ + public partial class DetectionLabelPanel + { + private AffiliationEnum _affiliation = AffiliationEnum.None; + private double _confidence; + + public AffiliationEnum Affiliation + { + get => _affiliation; + set + { + _affiliation = value; + UpdateAffiliationImage(); + } + } + + public DetectionClass DetectionClass { get; set; } + + public double Confidence + { + get => _confidence; + set + { + _confidence = value; + + } + } + + public DetectionLabelPanel() + { + InitializeComponent(); + } + + private string _detectionLabelText(string detectionClassName) => + _confidence >= 0.98 ? detectionClassName : $"{detectionClassName}: {_confidence * 100:F0}%"; + + private void UpdateAffiliationImage() + { + if (_affiliation == AffiliationEnum.None) + { + AffiliationImage.Source = null; + return; + } + + if (TryFindResource(_affiliation.ToString()) is DrawingImage drawingImage) + AffiliationImage.Source = drawingImage; + else + AffiliationImage.Source = null; + } + } +} \ No newline at end of file diff --git a/Azaion.Common/DTO/AffiliationEnum.cs b/Azaion.Common/DTO/AffiliationEnum.cs new file mode 100644 index 0000000..9fc6343 --- /dev/null +++ b/Azaion.Common/DTO/AffiliationEnum.cs @@ -0,0 +1,9 @@ +namespace Azaion.Common.DTO; + +public enum AffiliationEnum +{ + None = 0, + Friendly = 10, + Hostile = 20, + Unknown = 30 +} \ No newline at end of file diff --git a/Azaion.Common/DTO/Config/AIRecognitionConfig.cs b/Azaion.Common/DTO/Config/AIRecognitionConfig.cs index 94394ed..c3eff14 100644 --- a/Azaion.Common/DTO/Config/AIRecognitionConfig.cs +++ b/Azaion.Common/DTO/Config/AIRecognitionConfig.cs @@ -12,6 +12,7 @@ public class AIRecognitionConfig [Key("t_dc")] public double TrackingDistanceConfidence { get; set; } [Key("t_pi")] public double TrackingProbabilityIncrease { get; set; } [Key("t_it")] public double TrackingIntersectionThreshold { get; set; } + [Key("ov_p")] public double BigImageTileOverlapPercent { get; set; } [Key("d")] public byte[] Data { get; set; } = null!; [Key("p")] public List Paths { get; set; } = null!; diff --git a/Azaion.Common/DTO/FormState.cs b/Azaion.Common/DTO/FormState.cs index 07e1ba7..fcac093 100644 --- a/Azaion.Common/DTO/FormState.cs +++ b/Azaion.Common/DTO/FormState.cs @@ -6,10 +6,10 @@ namespace Azaion.Common.DTO; public class FormState { public MediaFileInfo? CurrentMedia { get; set; } - public string VideoName => CurrentMedia?.FName ?? ""; + public string MediaName => CurrentMedia?.FName ?? ""; public string CurrentMrl { get; set; } = null!; - public Size CurrentVideoSize { get; set; } + public Size CurrentMediaSize { get; set; } public TimeSpan CurrentVideoLength { get; set; } public TimeSpan? BackgroundTime { get; set; } diff --git a/Azaion.Common/DTO/Label.cs b/Azaion.Common/DTO/Label.cs index 341ba1a..65baec7 100644 --- a/Azaion.Common/DTO/Label.cs +++ b/Azaion.Common/DTO/Label.cs @@ -22,16 +22,36 @@ public abstract class Label public class CanvasLabel : Label { - public double X { get; set; } - public double Y { get; set; } + public double X { get; set; } //left + public double Y { get; set; } //top public double Width { get; set; } public double Height { get; set; } public double Confidence { get; set; } - public CanvasLabel() + public double Bottom { + get => Y + Height; + set => Height = value - Y; } + public double Right + { + get => X + Width; + set => Width = value - X; + } + + public CanvasLabel() { } + + public CanvasLabel(double left, double right, double top, double bottom) + { + X = left; + Y = top; + Width = right - left; + Height = bottom - top; + Confidence = 1; + ClassNumber = -1; + } + public CanvasLabel(int classNumber, double x, double y, double width, double height, double confidence = 1) : base(classNumber) { X = x; @@ -77,6 +97,13 @@ public class CanvasLabel : Label } Confidence = confidence; } + + public CanvasLabel ReframeToSmall(CanvasLabel smallTile) => + new(ClassNumber, X - smallTile.X, Y - smallTile.Y, Width, Height, Confidence); + + public CanvasLabel ReframeFromSmall(CanvasLabel smallTile) => + new(ClassNumber, X + smallTile.X, Y + smallTile.Y, Width, Height, Confidence); + } [MessagePackObject] @@ -193,13 +220,15 @@ public class Detection : YoloLabel { [JsonProperty(PropertyName = "an")][Key("an")] public string AnnotationName { get; set; } = null!; [JsonProperty(PropertyName = "p")][Key("p")] public double Confidence { get; set; } - + [JsonProperty(PropertyName = "dn")][Key("dn")] public string Description { get; set; } + //For db & serialization public Detection(){} - public Detection(string annotationName, YoloLabel label, double confidence = 1) + public Detection(string annotationName, YoloLabel label, string description = "", double confidence = 1) { AnnotationName = annotationName; + Description = description; ClassNumber = label.ClassNumber; CenterX = label.CenterX; CenterY = label.CenterY; diff --git a/Azaion.Common/Database/DbFactory.cs b/Azaion.Common/Database/DbFactory.cs index 72875ae..1b85c50 100644 --- a/Azaion.Common/Database/DbFactory.cs +++ b/Azaion.Common/Database/DbFactory.cs @@ -60,8 +60,10 @@ public class DbFactory : IDbFactory if (!File.Exists(_annConfig.AnnotationsDbFile)) SQLiteConnection.CreateFile(_annConfig.AnnotationsDbFile); RecreateTables(); - + _fileConnection.Open(); + using var db = new AnnotationsDb(_fileDataOptions); + SchemaMigrator.EnsureSchemaUpdated(db, typeof(Annotation), typeof(Detection)); _fileConnection.BackupDatabase(_memoryConnection, "main", "main", -1, null, -1); } diff --git a/Azaion.Common/Database/SchemaMigrator.cs b/Azaion.Common/Database/SchemaMigrator.cs new file mode 100644 index 0000000..1b34996 --- /dev/null +++ b/Azaion.Common/Database/SchemaMigrator.cs @@ -0,0 +1,94 @@ +using System.Data; +using LinqToDB.Data; +using LinqToDB.Mapping; + +namespace Azaion.Common.Database; + +public static class SchemaMigrator +{ + public static void EnsureSchemaUpdated(DataConnection dbConnection, params Type[] entityTypes) + { + var connection = dbConnection.Connection; + var mappingSchema = dbConnection.MappingSchema; + + if (connection.State == ConnectionState.Closed) + { + connection.Open(); + } + + foreach (var type in entityTypes) + { + var entityDescriptor = mappingSchema.GetEntityDescriptor(type); + var tableName = entityDescriptor.Name.Name; + var existingColumns = GetTableColumns(connection, tableName); + + foreach (var column in entityDescriptor.Columns) + { + if (existingColumns.Contains(column.ColumnName, StringComparer.OrdinalIgnoreCase)) + continue; + + var columnDefinition = GetColumnDefinition(column); + dbConnection.Execute($"ALTER TABLE {tableName} ADD COLUMN {columnDefinition}"); + } + } + } + + private static HashSet GetTableColumns(IDbConnection connection, string tableName) + { + var columns = new HashSet(StringComparer.OrdinalIgnoreCase); + using var cmd = connection.CreateCommand(); + cmd.CommandText = $"PRAGMA table_info({tableName})"; + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + columns.Add(reader.GetString(1)); // "name" is in the second column + + return columns; + } + + private static string GetColumnDefinition(ColumnDescriptor column) + { + var type = column.MemberType; + var underlyingType = Nullable.GetUnderlyingType(type) ?? type; + var sqliteType = GetSqliteType(underlyingType); + var defaultClause = GetSqlDefaultValue(type, underlyingType); + + return $"\"{column.ColumnName}\" {sqliteType} {defaultClause}"; + } + + private static string GetSqliteType(Type type) => + type switch + { + _ when type == typeof(int) + || type == typeof(long) + || type == typeof(bool) + || type.IsEnum + => "INTEGER", + + _ when type == typeof(double) + || type == typeof(float) + || type == typeof(decimal) + => "REAL", + + _ when type == typeof(byte[]) + => "BLOB", + + _ => "TEXT" + }; + + private static string GetSqlDefaultValue(Type originalType, Type underlyingType) + { + var isNullable = originalType.IsClass || Nullable.GetUnderlyingType(originalType) != null; + if (isNullable) + return "NULL"; + + var defaultValue = Activator.CreateInstance(underlyingType); + + if (underlyingType == typeof(bool)) + return $"NOT NULL DEFAULT {(Convert.ToBoolean(defaultValue) ? 1 : 0)}"; + + if (underlyingType.IsValueType && defaultValue is IFormattable f) + return $"NOT NULL DEFAULT {f.ToString(null, System.Globalization.CultureInfo.InvariantCulture)}"; + + return $"NOT NULL DEFAULT '{defaultValue}'"; + } +} \ No newline at end of file diff --git a/Azaion.Common/Services/TileProcessor.cs b/Azaion.Common/Services/TileProcessor.cs new file mode 100644 index 0000000..c534595 --- /dev/null +++ b/Azaion.Common/Services/TileProcessor.cs @@ -0,0 +1,82 @@ +using System.Windows; +using System.Windows.Media.Imaging; +using Azaion.Common.DTO; + +namespace Azaion.Common.Services; + +public class TileResult +{ + public CanvasLabel Tile { get; set; } + public List Detections { get; set; } + + public TileResult(CanvasLabel tile, List detections) + { + Tile = tile; + Detections = detections; + } +} + +public static class TileProcessor +{ + private const int MaxTileWidth = 1280; + private const int MaxTileHeight = 1280; + private const int Border = 10; + + public static List Split(Size originalSize, List detections, CancellationToken cancellationToken) + { + var results = new List(); + var processingDetectionList = new List(detections); + + while (processingDetectionList.Count > 0 && !cancellationToken.IsCancellationRequested) + { + var topMostDetection = processingDetectionList + .OrderBy(d => d.Y) + .First(); + + var result = GetDetectionsInTile(originalSize, topMostDetection, processingDetectionList); + processingDetectionList.RemoveAll(x => result.Detections.Contains(x)); + results.Add(result); + } + return results; + } + + private static TileResult GetDetectionsInTile(Size originalSize, CanvasLabel startDet, List allDetections) + { + var tile = new CanvasLabel( + left: Math.Max(startDet.X - Border, 0), + right: Math.Min(startDet.Right + Border, originalSize.Width), + top: Math.Max(startDet.Y - Border, 0), + bottom: Math.Min(startDet.Bottom + Border, originalSize.Height)); + var selectedDetections = new List{startDet}; + + foreach (var det in allDetections) + { + if (det == startDet) + continue; + + var commonTile = new CanvasLabel( + left: Math.Max(Math.Min(tile.X, det.X) - Border, 0), + right: Math.Min(Math.Max(tile.Right, det.Right) + Border, originalSize.Width), + top: Math.Max(Math.Min(tile.Y, det.Y) - Border, 0), + bottom: Math.Min(Math.Max(tile.Bottom, det.Bottom) + Border, originalSize.Height) + ); + + if (commonTile.Width > MaxTileWidth || commonTile.Height > MaxTileHeight) + continue; + + tile = commonTile; + selectedDetections.Add(det); + } + + //normalization, width and height should be at least half of 1280px + tile.Width = Math.Max(tile.Width, MaxTileWidth / 2.0); + tile.Height = Math.Max(tile.Height, MaxTileHeight / 2.0); + + //boundaries check after normalization + tile.Right = Math.Min(tile.Right, originalSize.Width); + tile.Bottom = Math.Min(tile.Bottom, originalSize.Height); + + return new TileResult(tile, selectedDetections); + } + +} \ No newline at end of file diff --git a/Azaion.Dataset/DatasetExplorerEventHandler.cs b/Azaion.Dataset/DatasetExplorerEventHandler.cs index 2a39822..4d66a4e 100644 --- a/Azaion.Dataset/DatasetExplorerEventHandler.cs +++ b/Azaion.Dataset/DatasetExplorerEventHandler.cs @@ -67,7 +67,7 @@ public class DatasetExplorerEventHandler( var a = datasetExplorer.CurrentAnnotation!.Annotation; var detections = datasetExplorer.ExplorerEditor.CurrentDetections - .Select(x => new Detection(a.Name, x.GetLabel(datasetExplorer.ExplorerEditor.RenderSize))) + .Select(x => new Detection(a.Name, x.ToYoloLabel(datasetExplorer.ExplorerEditor.RenderSize))) .ToList(); var index = datasetExplorer.ThumbnailsView.SelectedIndex; var annotation = await annotationService.SaveAnnotation(a.OriginalMediaName, a.Time, detections, token: token); diff --git a/Azaion.Inference/README.md b/Azaion.Inference/README.md index 116a353..842b2ce 100644 --- a/Azaion.Inference/README.md +++ b/Azaion.Inference/README.md @@ -13,23 +13,14 @@ Results (file or annotations) is putted to the other queue, or the same socket,

Installation

-Prepare correct onnx model from YOLO: -```python -from ultralytics import YOLO -import netron - -model = YOLO("azaion.pt") -model.export(format="onnx", imgsz=1280, nms=True, batch=4) -netron.start('azaion.onnx') -``` -Read carefully about [export arguments](https://docs.ultralytics.com/modes/export/), you have to use nms=True, and batching with a proper batch size -

Install libs

https://www.python.org/downloads/ Windows - [Install CUDA](https://developer.nvidia.com/cuda-12-1-0-download-archive) +- [Install Visual Studio Build Tools 2019](https://visualstudio.microsoft.com/downloads/?q=build+tools) + Linux ``` @@ -44,6 +35,17 @@ Linux nvcc --version ``` +Prepare correct onnx model from YOLO: +```python +from ultralytics import YOLO +import netron + +model = YOLO("azaion.pt") +model.export(format="onnx", imgsz=1280, nms=True, batch=4) +netron.start('azaion.onnx') +``` +Read carefully about [export arguments](https://docs.ultralytics.com/modes/export/), you have to use nms=True, and batching with a proper batch size +

Install dependencies

1. Install python with max version 3.11. Pytorch for now supports 3.11 max diff --git a/Azaion.Inference/ai_config.pxd b/Azaion.Inference/ai_config.pxd index b5b19c5..90ebae8 100644 --- a/Azaion.Inference/ai_config.pxd +++ b/Azaion.Inference/ai_config.pxd @@ -7,9 +7,11 @@ cdef class AIRecognitionConfig: cdef public double tracking_probability_increase cdef public double tracking_intersection_threshold + cdef public int big_image_tile_overlap_percent + cdef public bytes file_data cdef public list[str] paths cdef public int model_batch_size @staticmethod - cdef from_msgpack(bytes data) \ No newline at end of file + cdef from_msgpack(bytes data) diff --git a/Azaion.Inference/ai_config.pyx b/Azaion.Inference/ai_config.pyx index 28af40e..acbdf8b 100644 --- a/Azaion.Inference/ai_config.pyx +++ b/Azaion.Inference/ai_config.pyx @@ -9,6 +9,7 @@ cdef class AIRecognitionConfig: tracking_distance_confidence, tracking_probability_increase, tracking_intersection_threshold, + big_image_tile_overlap_percent, file_data, paths, @@ -21,6 +22,7 @@ cdef class AIRecognitionConfig: self.tracking_distance_confidence = tracking_distance_confidence self.tracking_probability_increase = tracking_probability_increase self.tracking_intersection_threshold = tracking_intersection_threshold + self.big_image_tile_overlap_percent = big_image_tile_overlap_percent self.file_data = file_data self.paths = paths @@ -31,6 +33,7 @@ cdef class AIRecognitionConfig: f'probability_increase : {self.tracking_probability_increase}, ' f'intersection_threshold : {self.tracking_intersection_threshold}, ' f'frame_period_recognition : {self.frame_period_recognition}, ' + f'big_image_tile_overlap_percent: {self.big_image_tile_overlap_percent}, ' f'paths: {self.paths}, ' f'model_batch_size: {self.model_batch_size}') @@ -45,6 +48,7 @@ cdef class AIRecognitionConfig: unpacked.get("t_dc", 0.0), unpacked.get("t_pi", 0.0), unpacked.get("t_it", 0.0), + unpacked.get("ov_p", 20), unpacked.get("d", b''), unpacked.get("p", []), diff --git a/Azaion.Inference/annotation.pxd b/Azaion.Inference/annotation.pxd index b8b1b34..932e969 100644 --- a/Azaion.Inference/annotation.pxd +++ b/Azaion.Inference/annotation.pxd @@ -3,7 +3,7 @@ cdef class Detection: cdef public str annotation_name cdef public int cls - cdef public overlaps(self, Detection det2) + cdef public overlaps(self, Detection det2, float confidence_threshold) cdef class Annotation: cdef public str name diff --git a/Azaion.Inference/annotation.pyx b/Azaion.Inference/annotation.pyx index 1d4f481..454eda5 100644 --- a/Azaion.Inference/annotation.pyx +++ b/Azaion.Inference/annotation.pyx @@ -14,13 +14,13 @@ cdef class Detection: def __str__(self): return f'{self.cls}: {self.x:.2f} {self.y:.2f} {self.w:.2f} {self.h:.2f}, prob: {(self.confidence*100):.1f}%' - cdef overlaps(self, Detection det2): + cdef overlaps(self, Detection det2, float confidence_threshold): cdef double overlap_x = 0.5 * (self.w + det2.w) - abs(self.x - det2.x) cdef double overlap_y = 0.5 * (self.h + det2.h) - abs(self.y - det2.y) cdef double overlap_area = max(0.0, overlap_x) * max(0.0, overlap_y) cdef double min_area = min(self.w * self.h, det2.w * det2.h) - return overlap_area / min_area > 0.6 + return overlap_area / min_area > confidence_threshold cdef class Annotation: def __init__(self, str name, long ms, list[Detection] detections): diff --git a/Azaion.Inference/inference.pxd b/Azaion.Inference/inference.pxd index 9e69a25..45952f4 100644 --- a/Azaion.Inference/inference.pxd +++ b/Azaion.Inference/inference.pxd @@ -23,11 +23,13 @@ cdef class Inference: cdef run_inference(self, RemoteCommand cmd) cdef _process_video(self, RemoteCommand cmd, AIRecognitionConfig ai_config, str video_name) - cdef _process_images(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list[str] image_paths) + cpdef _process_images(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list[str] image_paths) + cpdef _process_images_inner(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list frame_data) + cpdef split_to_tiles(self, frame, path, img_w, img_h, overlap_percent) cdef stop(self) cdef preprocess(self, frames) - cdef remove_overlapping_detections(self, list[Detection] detections) + cdef remove_overlapping_detections(self, list[Detection] detections, float confidence_threshold=?) cdef postprocess(self, output, ai_config) cdef split_list_extend(self, lst, chunk_size) diff --git a/Azaion.Inference/inference.pyx b/Azaion.Inference/inference.pyx index d6ff306..5e6b16e 100644 --- a/Azaion.Inference/inference.pyx +++ b/Azaion.Inference/inference.pyx @@ -150,13 +150,13 @@ cdef class Inference: h = y2 - y1 if conf >= ai_config.probability_threshold: detections.append(Detection(x, y, w, h, class_id, conf)) - filtered_detections = self.remove_overlapping_detections(detections) + filtered_detections = self.remove_overlapping_detections(detections, ai_config.tracking_intersection_threshold) results.append(filtered_detections) return results except Exception as e: raise RuntimeError(f"Failed to postprocess: {str(e)}") - cdef remove_overlapping_detections(self, list[Detection] detections): + cdef remove_overlapping_detections(self, list[Detection] detections, float confidence_threshold=0.6): cdef Detection det1, det2 filtered_output = [] filtered_out_indexes = [] @@ -168,7 +168,7 @@ cdef class Inference: res = det1_index for det2_index in range(det1_index + 1, len(detections)): det2 = detections[det2_index] - if det1.overlaps(det2): + if det1.overlaps(det2, confidence_threshold): if det1.confidence > det2.confidence or ( det1.confidence == det2.confidence and det1.cls < det2.cls): # det1 has higher confidence or lower class_id filtered_out_indexes.append(det2_index) @@ -211,9 +211,8 @@ cdef class Inference: images.append(m) # images first, it's faster if len(images) > 0: - for chunk in self.split_list_extend(images, self.engine.get_batch_size()): - constants_inf.log(f'run inference on {" ".join(chunk)}...') - self._process_images(cmd, ai_config, chunk) + constants_inf.log(f'run inference on {" ".join(images)}...') + self._process_images(cmd, ai_config, images) if len(videos) > 0: for v in videos: constants_inf.log(f'run inference on {v}...') @@ -250,8 +249,6 @@ cdef class Inference: _, image = cv2.imencode('.jpg', batch_frames[i]) annotation.image = image.tobytes() self._previous_annotation = annotation - - print(annotation) self.on_annotation(cmd, annotation) batch_frames.clear() @@ -259,15 +256,53 @@ cdef class Inference: v_input.release() - cdef _process_images(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list[str] image_paths): - cdef list frames = [] - cdef list timestamps = [] - self._previous_annotation = None - for image in image_paths: - frame = cv2.imread(image) - frames.append(frame) - timestamps.append(0) + cpdef _process_images(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list[str] image_paths): + cdef list frame_data = [] + for path in image_paths: + frame = cv2.imread(path) + if frame is None: + constants_inf.logerror(f'Failed to read image {path}') + continue + img_h, img_w, _ = frame.shape + if img_h <= 1.5 * self.model_height and img_w <= 1.5 * self.model_width: + frame_data.append((frame, path)) + else: + (split_frames, split_pats) = self.split_to_tiles(frame, path, img_w, img_h, ai_config.big_image_tile_overlap_percent) + frame_data.extend(zip(split_frames, split_pats)) + for chunk in self.split_list_extend(frame_data, self.engine.get_batch_size()): + self._process_images_inner(cmd, ai_config, chunk) + + + cpdef split_to_tiles(self, frame, path, img_w, img_h, overlap_percent): + stride_w = self.model_width * (1 - overlap_percent / 100) + stride_h = self.model_height * (1 - overlap_percent / 100) + n_tiles_x = int(np.ceil((img_w - self.model_width) / stride_w)) + 1 + n_tiles_y = int(np.ceil((img_h - self.model_height) / stride_h)) + 1 + + results = [] + for y_idx in range(n_tiles_y): + for x_idx in range(n_tiles_x): + y_start = y_idx * stride_w + x_start = x_idx * stride_h + + # Ensure the tile doesn't go out of bounds + y_end = min(y_start + self.model_width, img_h) + x_end = min(x_start + self.model_height, img_w) + + # We need to re-calculate start if we are at the edge to get a full 1280x1280 tile + if y_end == img_h: + y_start = img_h - self.model_height + if x_end == img_w: + x_start = img_w - self.model_width + + tile = frame[y_start:y_end, x_start:x_end] + name = path.stem + f'.tile_{x_start}_{y_start}' + path.suffix + results.append((tile, name)) + return results + + cpdef _process_images_inner(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list frame_data): + frames = [frame for frame, _ in frame_data] input_blob = self.preprocess(frames) outputs = self.engine.run(input_blob) @@ -275,7 +310,7 @@ cdef class Inference: list_detections = self.postprocess(outputs, ai_config) for i in range(len(list_detections)): detections = list_detections[i] - annotation = Annotation(image_paths[i], timestamps[i], detections) + annotation = Annotation(frame_data[i][1], 0, detections) _, image = cv2.imencode('.jpg', frames[i]) annotation.image = image.tobytes() self.on_annotation(cmd, annotation) @@ -322,7 +357,9 @@ cdef class Inference: closest_det = prev_det # Check if beyond tracking distance - if min_distance_sq > ai_config.tracking_distance_confidence: + dist_px = ai_config.tracking_distance_confidence * self.model_width + dist_px_sq = dist_px * dist_px + if min_distance_sq > dist_px_sq: return True # Check probability increase diff --git a/Azaion.Inference/requirements.txt b/Azaion.Inference/requirements.txt index 6c9f1c9..61e3564 100644 --- a/Azaion.Inference/requirements.txt +++ b/Azaion.Inference/requirements.txt @@ -7,11 +7,12 @@ cryptography==44.0.2 psutil msgpack pyjwt -zmq +pyzmq requests pyyaml pycuda -tensorrt +tensorrt==10.11.0.33 pynvml boto3 -loguru \ No newline at end of file +loguru +pytest \ No newline at end of file diff --git a/Azaion.Inference/setup.py b/Azaion.Inference/setup.py index 91157a4..9e3edcc 100644 --- a/Azaion.Inference/setup.py +++ b/Azaion.Inference/setup.py @@ -2,19 +2,30 @@ from setuptools import setup, Extension from Cython.Build import cythonize import numpy as np +# debug_args = {} +# trace_line = False + +debug_args = { + 'extra_compile_args': ['-O0', '-g'], + 'extra_link_args': ['-g'], + 'define_macros': [('CYTHON_TRACE_NOGIL', '1')] +} +trace_line = True + extensions = [ - Extension('constants_inf', ['constants_inf.pyx']), - Extension('file_data', ['file_data.pyx']), - Extension('remote_command_inf', ['remote_command_inf.pyx']), - Extension('remote_command_handler_inf', ['remote_command_handler_inf.pyx']), - Extension('annotation', ['annotation.pyx']), - Extension('loader_client', ['loader_client.pyx']), - Extension('ai_config', ['ai_config.pyx']), - Extension('tensorrt_engine', ['tensorrt_engine.pyx'], include_dirs=[np.get_include()]), - Extension('onnx_engine', ['onnx_engine.pyx'], include_dirs=[np.get_include()]), - Extension('inference_engine', ['inference_engine.pyx'], include_dirs=[np.get_include()]), - Extension('inference', ['inference.pyx'], include_dirs=[np.get_include()]), - Extension('main_inference', ['main_inference.pyx']), + Extension('constants_inf', ['constants_inf.pyx'], **debug_args), + Extension('file_data', ['file_data.pyx'], **debug_args), + Extension('remote_command_inf', ['remote_command_inf.pyx'], **debug_args), + Extension('remote_command_handler_inf', ['remote_command_handler_inf.pyx'], **debug_args), + Extension('annotation', ['annotation.pyx'], **debug_args), + Extension('loader_client', ['loader_client.pyx'], **debug_args), + Extension('ai_config', ['ai_config.pyx'], **debug_args), + Extension('tensorrt_engine', ['tensorrt_engine.pyx'], include_dirs=[np.get_include()], **debug_args), + Extension('onnx_engine', ['onnx_engine.pyx'], include_dirs=[np.get_include()], **debug_args), + Extension('inference_engine', ['inference_engine.pyx'], include_dirs=[np.get_include()], **debug_args), + Extension('inference', ['inference.pyx'], include_dirs=[np.get_include()], **debug_args), + Extension('main_inference', ['main_inference.pyx'], **debug_args), + ] setup( @@ -23,10 +34,11 @@ setup( extensions, compiler_directives={ "language_level": 3, - "emit_code_comments" : False, + "emit_code_comments": False, "binding": True, 'boundscheck': False, - 'wraparound': False + 'wraparound': False, + 'linetrace': trace_line } ), install_requires=[ @@ -34,4 +46,4 @@ setup( 'pywin32; platform_system=="Windows"' ], zip_safe=False -) \ No newline at end of file +) diff --git a/Azaion.Inference/setup_old.py b/Azaion.Inference/setup_old.py new file mode 100644 index 0000000..3dec931 --- /dev/null +++ b/Azaion.Inference/setup_old.py @@ -0,0 +1,37 @@ +from setuptools import setup, Extension +from Cython.Build import cythonize +import numpy as np + +extensions = [ + Extension('constants_inf', ['constants_inf.pyx']), + Extension('file_data', ['file_data.pyx']), + Extension('remote_command_inf', ['remote_command_inf.pyx']), + Extension('remote_command_handler_inf', ['remote_command_handler_inf.pyx']), + Extension('annotation', ['annotation.pyx']), + Extension('loader_client', ['loader_client.pyx']), + Extension('ai_config', ['ai_config.pyx']), + Extension('tensorrt_engine', ['tensorrt_engine.pyx'], include_dirs=[np.get_include()]), + Extension('onnx_engine', ['onnx_engine.pyx'], include_dirs=[np.get_include()]), + Extension('inference_engine', ['inference_engine.pyx'], include_dirs=[np.get_include()]), + Extension('inference', ['inference.pyx'], include_dirs=[np.get_include()]), + Extension('main_inference', ['main_inference.pyx']) +] + +setup( + name="azaion.ai", + ext_modules=cythonize( + extensions, + compiler_directives={ + "language_level": 3, + "emit_code_comments" : False, + "binding": True, + 'boundscheck': False, + 'wraparound': False + } + ), + install_requires=[ + 'ultralytics>=8.0.0', + 'pywin32; platform_system=="Windows"' + ], + zip_safe=False +) \ No newline at end of file diff --git a/Azaion.Inference/test/test_inference.py b/Azaion.Inference/test/test_inference.py new file mode 100644 index 0000000..6407ad2 --- /dev/null +++ b/Azaion.Inference/test/test_inference.py @@ -0,0 +1,8 @@ +import inference +from ai_config import AIRecognitionConfig +from remote_command_inf import RemoteCommand + + +def test_process_images(): + inf = inference.Inference(None, None) + inf._process_images(RemoteCommand(30), AIRecognitionConfig(4, 2, 15, 0.15, 15, 0.8, 20, b'test', [], 4), ['test_img01.JPG', 'test_img02.jpg']) \ No newline at end of file diff --git a/Azaion.Loader/requirements.txt b/Azaion.Loader/requirements.txt index 77008ff..2dd5417 100644 --- a/Azaion.Loader/requirements.txt +++ b/Azaion.Loader/requirements.txt @@ -3,7 +3,7 @@ Cython psutil msgpack pyjwt -zmq +pyzmq requests pyyaml boto3 diff --git a/Azaion.LoaderUI/App.xaml.cs b/Azaion.LoaderUI/App.xaml.cs index ff41625..7da8d9e 100644 --- a/Azaion.LoaderUI/App.xaml.cs +++ b/Azaion.LoaderUI/App.xaml.cs @@ -1,4 +1,5 @@ using System.Windows; +using Azaion.Common; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -28,7 +29,7 @@ public partial class App var host = Host.CreateDefaultBuilder() .ConfigureAppConfiguration((_, config) => config .AddCommandLine(Environment.GetCommandLineArgs()) - .AddJsonFile(Constants.CONFIG_JSON_FILE, optional: true)) + .AddJsonFile(Constants.LOADER_CONFIG_PATH, optional: true)) .UseSerilog() .ConfigureServices((context, services) => { @@ -36,7 +37,7 @@ public partial class App services.Configure(context.Configuration.GetSection(nameof(DirectoriesConfig))); services.AddHttpClient((sp, client) => { - client.BaseAddress = new Uri(Constants.API_URL); + client.BaseAddress = new Uri(Constants.DEFAULT_API_URL); client.DefaultRequestHeaders.Add("Accept", "application/json"); client.DefaultRequestHeaders.Add("User-Agent", "Azaion.LoaderUI"); }); diff --git a/Azaion.LoaderUI/Constants.cs b/Azaion.LoaderUI/Constants.cs deleted file mode 100644 index 1171109..0000000 --- a/Azaion.LoaderUI/Constants.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace Azaion.LoaderUI; - -public static class Constants -{ - public const string CONFIG_JSON_FILE = "loaderconfig.json"; - public const string API_URL = "https://api.azaion.com"; - public const string AZAION_SUITE_EXE = "Azaion.Suite.exe"; - public const string SUITE_FOLDER = "suite"; - public const string INFERENCE_EXE = "azaion-inference"; - public const string EXTERNAL_LOADER_PATH = "azaion-loader.exe"; - public const int EXTERNAL_LOADER_PORT = 5020; - public const string EXTERNAL_LOADER_HOST = "127.0.0.1"; -} \ No newline at end of file diff --git a/Azaion.LoaderUI/ConstantsLoader.cs b/Azaion.LoaderUI/ConstantsLoader.cs new file mode 100644 index 0000000..7f38ef6 --- /dev/null +++ b/Azaion.LoaderUI/ConstantsLoader.cs @@ -0,0 +1,7 @@ +namespace Azaion.LoaderUI; + +public static class ConstantsLoader +{ + public const string SUITE_FOLDER = "suite"; + public const int EXTERNAL_LOADER_PORT = 5020; +} \ No newline at end of file diff --git a/Azaion.LoaderUI/Login.xaml.cs b/Azaion.LoaderUI/Login.xaml.cs index f42bdf0..80436e4 100644 --- a/Azaion.LoaderUI/Login.xaml.cs +++ b/Azaion.LoaderUI/Login.xaml.cs @@ -57,7 +57,7 @@ public partial class Login TbStatus.Foreground = Brushes.Black; var installerVersion = await GetInstallerVer(); - var localVersion = GetLocalVer(); + var localVersion = Constants.GetLocalVersion(); var credsEncrypted = Security.Encrypt(creds); if (installerVersion > localVersion) @@ -81,7 +81,7 @@ public partial class Login Process.Start(Constants.AZAION_SUITE_EXE, $"-c {credsEncrypted}"); await Task.Delay(800); TbStatus.Text = "Loading..."; - while (!Process.GetProcessesByName(Constants.INFERENCE_EXE).Any()) + while (!Process.GetProcessesByName(Path.GetFileNameWithoutExtension(Constants.EXTERNAL_INFERENCE_PATH)).Any()) await Task.Delay(500); await Task.Delay(1500); } @@ -106,12 +106,12 @@ public partial class Login process.StartInfo = new ProcessStartInfo { FileName = Constants.EXTERNAL_LOADER_PATH, - Arguments = $"--port {Constants.EXTERNAL_LOADER_PORT} --api {Constants.API_URL}", + Arguments = $"--port {ConstantsLoader.EXTERNAL_LOADER_PORT} --api {Constants.DEFAULT_API_URL}", CreateNoWindow = true }; process.Start(); dealer.Options.Identity = Encoding.UTF8.GetBytes(Guid.NewGuid().ToString("N")); - dealer.Connect($"tcp://{Constants.EXTERNAL_LOADER_HOST}:{Constants.EXTERNAL_LOADER_PORT}"); + dealer.Connect($"tcp://{Constants.DEFAULT_ZMQ_INFERENCE_HOST}:{ConstantsLoader.EXTERNAL_LOADER_PORT}"); var result = SendCommand(dealer, RemoteCommand.Create(CommandType.Login, creds)); if (result.CommandType != CommandType.Ok) @@ -164,7 +164,7 @@ public partial class Login { TbStatus.Text = "Checking for the newer version..."; var installerDir = string.IsNullOrWhiteSpace(_dirConfig?.SuiteInstallerDirectory) - ? Constants.SUITE_FOLDER + ? ConstantsLoader.SUITE_FOLDER : _dirConfig.SuiteInstallerDirectory; var installerName = await _azaionApi.GetLastInstallerName(installerDir); var match = Regex.Match(installerName, @"\d+(\.\d+)+"); @@ -172,15 +172,7 @@ public partial class Login throw new Exception($"Can't find version in {installerName}"); return new Version(match.Value); } - - private Version GetLocalVer() - { - var localFileInfo = FileVersionInfo.GetVersionInfo(Constants.AZAION_SUITE_EXE); - if (string.IsNullOrWhiteSpace(localFileInfo.ProductVersion)) - throw new Exception($"Can't find {Constants.AZAION_SUITE_EXE} and its version"); - return new Version(localFileInfo.FileVersion!); - } - + private void CloseClick(object sender, RoutedEventArgs e) => Close(); private void MainMouseMove(object sender, MouseEventArgs e) diff --git a/Azaion.Suite/App.xaml.cs b/Azaion.Suite/App.xaml.cs index d5dfc5f..1b62fbc 100644 --- a/Azaion.Suite/App.xaml.cs +++ b/Azaion.Suite/App.xaml.cs @@ -153,12 +153,12 @@ public partial class App typeof(Annotator.Annotator).Assembly, typeof(DatasetExplorer).Assembly, typeof(AnnotationService).Assembly)); - services.AddSingleton(_ => new LibVLC()); + services.AddSingleton(_ => new LibVLC("--no-osd", "--no-video-title-show", "--no-snapshot-preview")); services.AddSingleton(); services.AddSingleton(sp => { - var libVLC = sp.GetRequiredService(); - return new MediaPlayer(libVLC); + var libVlc = sp.GetRequiredService(); + return new MediaPlayer(libVlc); }); services.AddSingleton(); services.AddSingleton(); @@ -177,8 +177,6 @@ public partial class App Annotation.InitializeDirs(_host.Services.GetRequiredService>().Value); _host.Services.GetRequiredService(); - // datasetExplorer.Show(); - // datasetExplorer.Hide(); _mediator = _host.Services.GetRequiredService(); diff --git a/Azaion.Suite/config.system.json b/Azaion.Suite/config.system.json index c6f777f..c740a46 100644 --- a/Azaion.Suite/config.system.json +++ b/Azaion.Suite/config.system.json @@ -30,7 +30,8 @@ "TrackingDistanceConfidence": 0.15, "TrackingProbabilityIncrease": 15.0, - "TrackingIntersectionThreshold": 0.8, + "TrackingIntersectionThreshold": 0.6, + "BigImageTileOverlapPercent": 20, "ModelBatchSize": 4 }, From ad782bcbaa285b7960968075e0a9584f91ffe654 Mon Sep 17 00:00:00 2001 From: Oleksandr Bezdieniezhnykh Date: Tue, 12 Aug 2025 14:48:56 +0300 Subject: [PATCH 06/14] splitting python complete --- Azaion.Annotator/Annotator.xaml.cs | 182 ++++++------ Azaion.Annotator/AnnotatorEventHandler.cs | 63 +++-- Azaion.Common/Azaion.Common.csproj | 1 + Azaion.Common/Constants.cs | 88 +++--- Azaion.Common/Controls/CanvasEditor.cs | 41 ++- Azaion.Common/Controls/DetectionControl.cs | 4 +- Azaion.Common/DTO/AnnotationResult.cs | 48 ++-- Azaion.Common/DTO/FormState.cs | 6 +- Azaion.Common/DTO/Label.cs | 58 ++-- Azaion.Common/Database/Annotation.cs | 67 ++++- Azaion.Common/Database/DbFactory.cs | 3 +- Azaion.Common/Services/AnnotationService.cs | 20 +- Azaion.Common/Services/GalleryService.cs | 14 +- Azaion.Common/Services/InferenceClient.cs | 2 +- Azaion.Common/Services/TileProcessor.cs | 41 ++- Azaion.Dataset/DatasetExplorer.xaml | 23 +- Azaion.Dataset/DatasetExplorer.xaml.cs | 12 +- Azaion.Dataset/DatasetExplorerEventHandler.cs | 2 +- Azaion.Inference/annotation.pxd | 1 - Azaion.Inference/annotation.pyx | 30 +- Azaion.Inference/constants_inf.pxd | 6 +- Azaion.Inference/constants_inf.pyx | 16 +- Azaion.Inference/inference.pxd | 15 +- Azaion.Inference/inference.pyx | 129 ++++++--- Azaion.Inference/setup.py | 16 +- Azaion.Inference/test/test_inference.py | 30 +- Azaion.Loader/hardware_service.pyx | 8 +- Azaion.Suite/App.xaml.cs | 4 +- Azaion.Suite/config.json | 8 +- Azaion.Suite/config.system.json | 2 +- Azaion.Test/TileProcessorTest.cs | 263 ++++++++++++++++++ 31 files changed, 834 insertions(+), 369 deletions(-) create mode 100644 Azaion.Test/TileProcessorTest.cs diff --git a/Azaion.Annotator/Annotator.xaml.cs b/Azaion.Annotator/Annotator.xaml.cs index 13bfb44..9feeba4 100644 --- a/Azaion.Annotator/Annotator.xaml.cs +++ b/Azaion.Annotator/Annotator.xaml.cs @@ -29,7 +29,7 @@ namespace Azaion.Annotator; public partial class Annotator { private readonly AppConfig _appConfig; - private readonly LibVLC _libVLC; + private readonly LibVLC _libVlc; private readonly MediaPlayer _mediaPlayer; private readonly IMediator _mediator; private readonly FormState _formState; @@ -42,17 +42,17 @@ public partial class Annotator private readonly IInferenceClient _inferenceClient; private bool _suspendLayout; - private bool _gpsPanelVisible = false; + private bool _gpsPanelVisible; - public readonly CancellationTokenSource MainCancellationSource = new(); + private readonly CancellationTokenSource _mainCancellationSource = new(); public CancellationTokenSource DetectionCancellationSource = new(); - public bool IsInferenceNow = false; + private bool _isInferenceNow; private readonly TimeSpan _thresholdBefore = TimeSpan.FromMilliseconds(50); private readonly TimeSpan _thresholdAfter = TimeSpan.FromMilliseconds(150); public ObservableCollection AllMediaFiles { get; set; } = new(); - public ObservableCollection FilteredMediaFiles { get; set; } = new(); + private ObservableCollection FilteredMediaFiles { get; set; } = new(); public Dictionary MediaFilesDict = new(); public IntervalTree TimedAnnotations { get; set; } = new(); @@ -61,7 +61,7 @@ public partial class Annotator public Annotator( IConfigUpdater configUpdater, IOptions appConfig, - LibVLC libVLC, + LibVLC libVlc, MediaPlayer mediaPlayer, IMediator mediator, FormState formState, @@ -78,7 +78,7 @@ public partial class Annotator Title = MainTitle; _appConfig = appConfig.Value; _configUpdater = configUpdater; - _libVLC = libVLC; + _libVlc = libVlc; _mediaPlayer = mediaPlayer; _mediator = mediator; _formState = formState; @@ -91,7 +91,7 @@ public partial class Annotator Loaded += OnLoaded; Closed += OnFormClosed; Activated += (_, _) => _formState.ActiveWindow = WindowEnum.Annotator; - TbFolder.TextChanged += async (sender, args) => + TbFolder.TextChanged += async (_, _) => { if (!Path.Exists(TbFolder.Text)) return; @@ -179,22 +179,8 @@ public partial class Annotator VideoView.MediaPlayer = _mediaPlayer; //On start playing media - _mediaPlayer.Playing += async (sender, args) => + _mediaPlayer.Playing += (_, _) => { - if (_formState.CurrentMrl == _mediaPlayer.Media?.Mrl) - return; //already loaded all the info - - await Dispatcher.Invoke(async () => await ReloadAnnotations()); - - //show image - if (_formState.CurrentMedia?.MediaType == MediaTypes.Image) - { - await Task.Delay(100); //wait to load the frame and set on pause - ShowTimeAnnotations(TimeSpan.FromMilliseconds(_mediaPlayer.Time), showImage: true); - return; - } - - _formState.CurrentMrl = _mediaPlayer.Media?.Mrl ?? ""; uint vw = 0, vh = 0; _mediaPlayer.Size(0, ref vw, ref vh); _formState.CurrentMediaSize = new Size(vw, vh); @@ -211,12 +197,12 @@ public partial class Annotator var selectedClass = args.DetectionClass; Editor.CurrentAnnClass = selectedClass; _mediator.Publish(new AnnClassSelectedEvent(selectedClass)); - }; + }; - _mediaPlayer.PositionChanged += (o, args) => + _mediaPlayer.PositionChanged += (_, _) => ShowTimeAnnotations(TimeSpan.FromMilliseconds(_mediaPlayer.Time)); - VideoSlider.ValueChanged += (value, newValue) => + VideoSlider.ValueChanged += (_, newValue) => _mediaPlayer.Position = (float)(newValue / VideoSlider.Maximum); VideoSlider.KeyDown += (sender, args) => @@ -227,51 +213,49 @@ public partial class Annotator DgAnnotations.MouseDoubleClick += (sender, args) => { - var dgRow = ItemsControl.ContainerFromElement((DataGrid)sender, (args.OriginalSource as DependencyObject)!) as DataGridRow; - if (dgRow != null) - OpenAnnotationResult((AnnotationResult)dgRow!.Item); + if (ItemsControl.ContainerFromElement((DataGrid)sender, (args.OriginalSource as DependencyObject)!) is DataGridRow dgRow) + OpenAnnotationResult((Annotation)dgRow.Item); }; - DgAnnotations.KeyUp += async (sender, args) => + DgAnnotations.KeyUp += async (_, args) => { switch (args.Key) { - case Key.Up: case Key.Down: //cursor is already moved by system behaviour - OpenAnnotationResult((AnnotationResult)DgAnnotations.SelectedItem); + OpenAnnotationResult((Annotation)DgAnnotations.SelectedItem); break; case Key.Delete: var result = MessageBox.Show("Чи дійсно видалити аннотації?","Підтвердження видалення", MessageBoxButton.OKCancel, MessageBoxImage.Question); if (result != MessageBoxResult.OK) return; - var res = DgAnnotations.SelectedItems.Cast().ToList(); - var annotationNames = res.Select(x => x.Annotation.Name).ToList(); + var res = DgAnnotations.SelectedItems.Cast().ToList(); + var annotationNames = res.Select(x => x.Name).ToList(); await _mediator.Publish(new AnnotationsDeletedEvent(annotationNames)); break; } }; - - Editor.Mediator = _mediator; DgAnnotations.ItemsSource = _formState.AnnotationResults; } - public void OpenAnnotationResult(AnnotationResult res) + private void OpenAnnotationResult(Annotation ann) { _mediaPlayer.SetPause(true); - Editor.RemoveAllAnns(); - _mediaPlayer.Time = (long)res.Annotation.Time.TotalMilliseconds; + if (!ann.IsSplit) + Editor.RemoveAllAnns(); + + _mediaPlayer.Time = (long)ann.Time.TotalMilliseconds; Dispatcher.Invoke(() => { VideoSlider.Value = _mediaPlayer.Position * VideoSlider.Maximum; StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}"; - Editor.ClearExpiredAnnotations(res.Annotation.Time); + Editor.ClearExpiredAnnotations(ann.Time); }); - ShowAnnotation(res.Annotation, showImage: true); + ShowAnnotation(ann, showImage: true, openResult: true); } private void SaveUserSettings() { @@ -284,7 +268,7 @@ public partial class Annotator _configUpdater.Save(_appConfig); } - private void ShowTimeAnnotations(TimeSpan time, bool showImage = false) + public void ShowTimeAnnotations(TimeSpan time, bool showImage = false) { Dispatcher.Invoke(() => { @@ -292,60 +276,68 @@ public partial class Annotator StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}"; Editor.ClearExpiredAnnotations(time); }); - var annotation = TimedAnnotations.Query(time).FirstOrDefault(); - if (annotation != null) ShowAnnotation(annotation, showImage); + var annotations = TimedAnnotations.Query(time).ToList(); + if (!annotations.Any()) + return; + foreach (var ann in annotations) + ShowAnnotation(ann, showImage); } - private void ShowAnnotation(Annotation annotation, bool showImage = false) + private void ShowAnnotation(Annotation annotation, bool showImage = false, bool openResult = false) { Dispatcher.Invoke(async () => { - if (showImage) + if (showImage && !annotation.IsSplit && File.Exists(annotation.ImagePath)) { - if (File.Exists(annotation.ImagePath)) - { - Editor.SetBackground(await annotation.ImagePath.OpenImage()); - _formState.BackgroundTime = annotation.Time; - } + Editor.SetBackground(await annotation.ImagePath.OpenImage()); + _formState.BackgroundTime = annotation.Time; } - Editor.CreateDetections(annotation.Time, annotation.Detections, _appConfig.AnnotationConfig.DetectionClasses, _formState.CurrentMediaSize); + + if (annotation.SplitTile != null && openResult) + { + var canvasTileLocation = new CanvasLabel(new YoloLabel(annotation.SplitTile, _formState.CurrentMediaSize), + RenderSize); + Editor.ZoomTo(new Point(canvasTileLocation.CenterX, canvasTileLocation.CenterY)); + } + else + Editor.CreateDetections(annotation, _appConfig.AnnotationConfig.DetectionClasses, _formState.CurrentMediaSize); }); } - private async Task ReloadAnnotations() + public async Task ReloadAnnotations() { - _formState.AnnotationResults.Clear(); - TimedAnnotations.Clear(); - Editor.RemoveAllAnns(); - - var annotations = await _dbFactory.Run(async db => - await db.Annotations.LoadWith(x => x.Detections) - .Where(x => x.OriginalMediaName == _formState.MediaName) - .OrderBy(x => x.Time) - .ToListAsync(token: MainCancellationSource.Token)); - - TimedAnnotations.Clear(); - _formState.AnnotationResults.Clear(); - foreach (var ann in annotations) + await Dispatcher.InvokeAsync(async () => { - TimedAnnotations.Add(ann.Time.Subtract(_thresholdBefore), ann.Time.Add(_thresholdAfter), ann); - _formState.AnnotationResults.Add(new AnnotationResult(_appConfig.AnnotationConfig.DetectionClassesDict, ann)); - } + _formState.AnnotationResults.Clear(); + TimedAnnotations.Clear(); + Editor.RemoveAllAnns(); + + var annotations = await _dbFactory.Run(async db => + await db.Annotations.LoadWith(x => x.Detections) + .Where(x => x.OriginalMediaName == _formState.MediaName) + .OrderBy(x => x.Time) + .ToListAsync(token: _mainCancellationSource.Token)); + + TimedAnnotations.Clear(); + _formState.AnnotationResults.Clear(); + foreach (var ann in annotations) + { + // Duplicate for speed + TimedAnnotations.Add(ann.Time.Subtract(_thresholdBefore), ann.Time.Add(_thresholdAfter), ann); + _formState.AnnotationResults.Add(ann); + } + }); } //Add manually public void AddAnnotation(Annotation annotation) { - var mediaInfo = (MediaFileInfo)LvFiles.SelectedItem; - if ((mediaInfo?.FName ?? "") != annotation.OriginalMediaName) - return; - var time = annotation.Time; var previousAnnotations = TimedAnnotations.Query(time); TimedAnnotations.Remove(previousAnnotations); TimedAnnotations.Add(time.Subtract(_thresholdBefore), time.Add(_thresholdAfter), annotation); - var existingResult = _formState.AnnotationResults.FirstOrDefault(x => x.Annotation.Time == time); + var existingResult = _formState.AnnotationResults.FirstOrDefault(x => x.Time == time); if (existingResult != null) { try @@ -360,16 +352,14 @@ public partial class Annotator } var dict = _formState.AnnotationResults - .Select((x, i) => new { x.Annotation.Time, Index = i }) + .Select((x, i) => new { x.Time, Index = i }) .ToDictionary(x => x.Time, x => x.Index); var index = dict.Where(x => x.Key < time) .OrderBy(x => time - x.Key) .Select(x => x.Value + 1) .FirstOrDefault(); - - var annRes = new AnnotationResult(_appConfig.AnnotationConfig.DetectionClassesDict, annotation); - _formState.AnnotationResults.Insert(index, annRes); + _formState.AnnotationResults.Insert(index, annotation); } private async Task ReloadFiles() @@ -380,7 +370,7 @@ public partial class Annotator var videoFiles = dir.GetFiles(_appConfig.AnnotationConfig.VideoFormats.ToArray()).Select(x => { - using var media = new Media(_libVLC, x.FullName); + var media = new Media(_libVlc, x.FullName); media.Parse(); var fInfo = new MediaFileInfo { @@ -403,14 +393,16 @@ public partial class Annotator var allFileNames = allFiles.Select(x => x.FName).ToList(); - var labelsDict = await _dbFactory.Run(async db => await db.Annotations - .GroupBy(x => x.Name.Substring(0, x.Name.Length - 7)) + var labelsDict = await _dbFactory.Run(async db => + await db.Annotations + .GroupBy(x => x.OriginalMediaName) .Where(x => allFileNames.Contains(x.Key)) - .ToDictionaryAsync(x => x.Key, x => x.Key)); - + .Select(x => x.Key) + .ToDictionaryAsync(x => x, x => x)); + foreach (var mediaFile in allFiles) mediaFile.HasAnnotations = labelsDict.ContainsKey(mediaFile.FName); - + AllMediaFiles = new ObservableCollection(allFiles); MediaFilesDict = AllMediaFiles.GroupBy(x => x.Name) .ToDictionary(gr => gr.Key, gr => gr.First()); @@ -420,13 +412,13 @@ public partial class Annotator private void OnFormClosed(object? sender, EventArgs e) { - MainCancellationSource.Cancel(); + _mainCancellationSource.Cancel(); _inferenceService.StopInference(); DetectionCancellationSource.Cancel(); _mediaPlayer.Stop(); _mediaPlayer.Dispose(); - _libVLC.Dispose(); + _libVlc.Dispose(); } private void OpenContainingFolder(object sender, RoutedEventArgs e) @@ -447,13 +439,10 @@ public partial class Annotator StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}"; } - private void SeekTo(TimeSpan time) => - SeekTo((long)time.TotalMilliseconds); + private void OpenFolderItemClick(object sender, RoutedEventArgs e) => OpenFolder(); + private void OpenFolderButtonClick(object sender, RoutedEventArgs e) => OpenFolder(); - private async void OpenFolderItemClick(object sender, RoutedEventArgs e) => await OpenFolder(); - private async void OpenFolderButtonClick(object sender, RoutedEventArgs e) => await OpenFolder(); - - private async Task OpenFolder() + private void OpenFolder() { var dlg = new CommonOpenFileDialog { @@ -468,7 +457,6 @@ public partial class Annotator _appConfig.DirectoriesConfig.VideosDirectory = dlg.FileName; TbFolder.Text = dlg.FileName; - await Task.CompletedTask; } private void TbFilter_OnTextChanged(object sender, TextChangedEventArgs e) @@ -525,7 +513,7 @@ public partial class Annotator public async Task AutoDetect() { - if (IsInferenceNow) + if (_isInferenceNow) return; if (LvFiles.Items.IsEmpty) @@ -535,7 +523,7 @@ public partial class Annotator Dispatcher.Invoke(() => Editor.SetBackground(null)); - IsInferenceNow = true; + _isInferenceNow = true; AIDetectBtn.IsEnabled = false; DetectionCancellationSource = new CancellationTokenSource(); @@ -550,7 +538,7 @@ public partial class Annotator await _inferenceService.RunInference(files, DetectionCancellationSource.Token); LvFiles.Items.Refresh(); - IsInferenceNow = false; + _isInferenceNow = false; StatusHelp.Text = "Розпізнавання зваершено"; AIDetectBtn.IsEnabled = true; } @@ -596,7 +584,7 @@ public class GradientStyleSelector : StyleSelector { public override Style? SelectStyle(object item, DependencyObject container) { - if (container is not DataGridRow row || row.DataContext is not AnnotationResult result) + if (container is not DataGridRow row || row.DataContext is not Annotation result) return null; var style = new Style(typeof(DataGridRow)); diff --git a/Azaion.Annotator/AnnotatorEventHandler.cs b/Azaion.Annotator/AnnotatorEventHandler.cs index 0607530..ee52a7e 100644 --- a/Azaion.Annotator/AnnotatorEventHandler.cs +++ b/Azaion.Annotator/AnnotatorEventHandler.cs @@ -23,7 +23,7 @@ using MediaPlayer = LibVLCSharp.Shared.MediaPlayer; namespace Azaion.Annotator; public class AnnotatorEventHandler( - LibVLC libVLC, + LibVLC libVlc, MediaPlayer mediaPlayer, Annotator mainWindow, FormState formState, @@ -47,8 +47,7 @@ public class AnnotatorEventHandler( { private const int STEP = 20; private const int LARGE_STEP = 5000; - private const int RESULT_WIDTH = 1280; - private readonly string tempImgPath = Path.Combine(dirConfig.Value.ImagesDirectory, "___temp___.jpg"); + private readonly string _tempImgPath = Path.Combine(dirConfig.Value.ImagesDirectory, "___temp___.jpg"); private readonly Dictionary _keysControlEnumDict = new() { @@ -144,8 +143,8 @@ public class AnnotatorEventHandler( if (mediaPlayer.IsPlaying) { mediaPlayer.Pause(); - mediaPlayer.TakeSnapshot(0, tempImgPath, 0, 0); - mainWindow.Editor.SetBackground(await tempImgPath.OpenImage()); + mediaPlayer.TakeSnapshot(0, _tempImgPath, 0, 0); + mainWindow.Editor.SetBackground(await _tempImgPath.OpenImage()); formState.BackgroundTime = TimeSpan.FromMilliseconds(mediaPlayer.Time); } else @@ -238,16 +237,21 @@ public class AnnotatorEventHandler( return; var mediaInfo = (MediaFileInfo)mainWindow.LvFiles.SelectedItem; + if (formState.CurrentMedia == mediaInfo) + return; //already loaded + formState.CurrentMedia = mediaInfo; mainWindow.Title = $"{mainWindow.MainTitle} - {mediaInfo.Name}"; - + + await mainWindow.ReloadAnnotations(); + if (mediaInfo.MediaType == MediaTypes.Video) { mainWindow.Editor.SetBackground(null); //need to wait a bit for correct VLC playback event handling await Task.Delay(100, ct); mediaPlayer.Stop(); - mediaPlayer.Play(new Media(libVLC, mediaInfo.Path)); + mediaPlayer.Play(new Media(libVlc, mediaInfo.Path)); } else { @@ -256,6 +260,7 @@ public class AnnotatorEventHandler( formState.CurrentMediaSize = new Size(image.PixelWidth, image.PixelHeight); mainWindow.Editor.SetBackground(image); mediaPlayer.Stop(); + mainWindow.ShowTimeAnnotations(TimeSpan.Zero, showImage: true); } } @@ -282,13 +287,14 @@ public class AnnotatorEventHandler( // var annGrid = mainWindow.DgAnnotations; // annGrid.SelectedIndex = Math.Min(annGrid.Items.Count, annGrid.SelectedIndex + 1); // mainWindow.OpenAnnotationResult((AnnotationResult)annGrid.SelectedItem); + + mainWindow.Editor.SetBackground(null); + formState.BackgroundTime = null; } else { await NextMedia(ct: cancellationToken); } - mainWindow.Editor.SetBackground(null); - formState.BackgroundTime = null; mainWindow.LvFiles.Items.Refresh(); mainWindow.Editor.RemoveAllAnns(); @@ -301,7 +307,7 @@ public class AnnotatorEventHandler( if (!File.Exists(imgPath)) { var source = (mainWindow.Editor.BackgroundImage.Source as BitmapSource)!; - if (source.PixelWidth <= RESULT_WIDTH * 2 && source.PixelHeight <= RESULT_WIDTH * 2) // Allow to be up to 2560*2560 to save to 1280*1280 + if (source.PixelWidth <= Constants.AI_TILE_SIZE * 2 && source.PixelHeight <= Constants.AI_TILE_SIZE * 2) // Allow to be up to 2560*2560 to save to 1280*1280 { //Save image await using var stream = new FileStream(imgPath, FileMode.Create); @@ -314,28 +320,28 @@ public class AnnotatorEventHandler( { //Tiling - //1. Restore original picture coordinates - var pictureCoordinatesDetections = canvasDetections.Select(x => new CanvasLabel( + //1. Convert from RenderSize to CurrentMediaSize + var detectionCoords = canvasDetections.Select(x => new CanvasLabel( new YoloLabel(x, mainWindow.Editor.RenderSize, formState.CurrentMediaSize), formState.CurrentMediaSize, null, x.Confidence)) .ToList(); - //2. Split to 1280*1280 frames - var results = TileProcessor.Split(formState.CurrentMediaSize, pictureCoordinatesDetections, cancellationToken); + //2. Split to frames + var results = TileProcessor.Split(formState.CurrentMediaSize, detectionCoords, cancellationToken); //3. Save each frame as a separate annotation - BitmapEncoder tileEncoder = new JpegBitmapEncoder(); foreach (var res in results) { - var mediaName = $"{formState.MediaName}!split!{res.Tile.X}_{res.Tile.Y}!"; var time = TimeSpan.Zero; - var annotationName = mediaName.ToTimeName(time); + var annotationName = $"{formState.MediaName}{Constants.SPLIT_SUFFIX}{res.Tile.Left:0000}_{res.Tile.Top:0000}!".ToTimeName(time); var tileImgPath = Path.Combine(dirConfig.Value.ImagesDirectory, $"{annotationName}{Constants.JPG_EXT}"); await using var tileStream = new FileStream(tileImgPath, FileMode.Create); - var bitmap = new CroppedBitmap(source, new Int32Rect((int)res.Tile.X, (int)res.Tile.Y, (int)res.Tile.Width, (int)res.Tile.Height)); - tileEncoder.Frames.Add(BitmapFrame.Create(bitmap)); + var bitmap = new CroppedBitmap(source, new Int32Rect((int)res.Tile.Left, (int)res.Tile.Top, (int)res.Tile.Width, (int)res.Tile.Height)); + + var tileEncoder = new JpegBitmapEncoder { Frames = [BitmapFrame.Create(bitmap)] }; tileEncoder.Save(tileStream); await tileStream.FlushAsync(cancellationToken); + tileStream.Close(); var frameSize = new Size(res.Tile.Width, res.Tile.Height); var detections = res.Detections @@ -343,18 +349,18 @@ public class AnnotatorEventHandler( .Select(x => new Detection(annotationName, new YoloLabel(x, frameSize))) .ToList(); - annotationsResult.Add(await annotationService.SaveAnnotation(mediaName, time, detections, token: cancellationToken)); + annotationsResult.Add(await annotationService.SaveAnnotation(formState.MediaName, annotationName, time, detections, token: cancellationToken)); } return annotationsResult; } } var timeImg = formState.BackgroundTime ?? TimeSpan.FromMilliseconds(mediaPlayer.Time); - var timeName = formState.MediaName.ToTimeName(timeImg); + var annName = formState.MediaName.ToTimeName(timeImg); var currentDetections = canvasDetections.Select(x => - new Detection(timeName, new YoloLabel(x, mainWindow.Editor.RenderSize))) + new Detection(annName, new YoloLabel(x, mainWindow.Editor.RenderSize))) .ToList(); - var annotation = await annotationService.SaveAnnotation(formState.MediaName, timeImg, currentDetections, token: cancellationToken); + var annotation = await annotationService.SaveAnnotation(formState.MediaName, annName, timeImg, currentDetections, token: cancellationToken); return [annotation]; } @@ -367,15 +373,15 @@ public class AnnotatorEventHandler( var namesSet = notification.AnnotationNames.ToHashSet(); var remainAnnotations = formState.AnnotationResults - .Where(x => !namesSet.Contains(x.Annotation?.Name ?? "")).ToList(); + .Where(x => !namesSet.Contains(x.Name)).ToList(); formState.AnnotationResults.Clear(); foreach (var ann in remainAnnotations) formState.AnnotationResults.Add(ann); - var timedAnnsToRemove = mainWindow.TimedAnnotations + var timedAnnotationsToRemove = mainWindow.TimedAnnotations .Where(x => namesSet.Contains(x.Value.Name)) .Select(x => x.Value).ToList(); - mainWindow.TimedAnnotations.Remove(timedAnnsToRemove); + mainWindow.TimedAnnotations.Remove(timedAnnotationsToRemove); if (formState.AnnotationResults.Count == 0) { @@ -420,7 +426,10 @@ public class AnnotatorEventHandler( { mainWindow.Dispatcher.Invoke(() => { - mainWindow.AddAnnotation(e.Annotation); + + var mediaInfo = (MediaFileInfo)mainWindow.LvFiles.SelectedItem; + if ((mediaInfo?.FName ?? "") == e.Annotation.OriginalMediaName) + mainWindow.AddAnnotation(e.Annotation); var log = string.Join(Environment.NewLine, e.Annotation.Detections.Select(det => $"Розпізнавання {e.Annotation.OriginalMediaName}: {annotationConfig.Value.DetectionClassesDict[det.ClassNumber].ShortName}: " + diff --git a/Azaion.Common/Azaion.Common.csproj b/Azaion.Common/Azaion.Common.csproj index ca47d48..3f1339f 100644 --- a/Azaion.Common/Azaion.Common.csproj +++ b/Azaion.Common/Azaion.Common.csproj @@ -4,6 +4,7 @@ enable enable true + 12 diff --git a/Azaion.Common/Constants.cs b/Azaion.Common/Constants.cs index b92b8aa..fe26b29 100644 --- a/Azaion.Common/Constants.cs +++ b/Azaion.Common/Constants.cs @@ -9,13 +9,15 @@ using System.Windows; namespace Azaion.Common; -public class Constants +public static class Constants { public const string CONFIG_PATH = "config.json"; public const string LOADER_CONFIG_PATH = "loaderconfig.json"; public const string DEFAULT_API_URL = "https://api.azaion.com"; public const string AZAION_SUITE_EXE = "Azaion.Suite.exe"; + public const int AI_TILE_SIZE = 1280; + #region ExternalClientsConfig private const string DEFAULT_ZMQ_LOADER_HOST = "127.0.0.1"; @@ -27,11 +29,11 @@ public class Constants public static readonly string ExternalGpsDeniedPath = Path.Combine(EXTERNAL_GPS_DENIED_FOLDER, "image-matcher.exe"); public const string DEFAULT_ZMQ_INFERENCE_HOST = "127.0.0.1"; - public const int DEFAULT_ZMQ_INFERENCE_PORT = 5227; + private const int DEFAULT_ZMQ_INFERENCE_PORT = 5227; - public const string DEFAULT_ZMQ_GPS_DENIED_HOST = "127.0.0.1"; - public const int DEFAULT_ZMQ_GPS_DENIED_PORT = 5255; - public const int DEFAULT_ZMQ_GPS_DENIED_PUBLISH_PORT = 5256; + private const string DEFAULT_ZMQ_GPS_DENIED_HOST = "127.0.0.1"; + private const int DEFAULT_ZMQ_GPS_DENIED_PORT = 5255; + private const int DEFAULT_ZMQ_GPS_DENIED_PUBLISH_PORT = 5256; #endregion ExternalClientsConfig @@ -42,41 +44,33 @@ public class Constants # endregion - public const string JPG_EXT = ".jpg"; + public const string JPG_EXT = ".jpg"; public const string TXT_EXT = ".txt"; #region DirectoriesConfig - public const string DEFAULT_VIDEO_DIR = "video"; - public const string DEFAULT_LABELS_DIR = "labels"; - public const string DEFAULT_IMAGES_DIR = "images"; - public const string DEFAULT_RESULTS_DIR = "results"; - public const string DEFAULT_THUMBNAILS_DIR = "thumbnails"; - public const string DEFAULT_GPS_SAT_DIRECTORY = "satellitesDir"; - public const string DEFAULT_GPS_ROUTE_DIRECTORY = "routeDir"; + private const string DEFAULT_VIDEO_DIR = "video"; + private const string DEFAULT_LABELS_DIR = "labels"; + private const string DEFAULT_IMAGES_DIR = "images"; + private const string DEFAULT_RESULTS_DIR = "results"; + private const string DEFAULT_THUMBNAILS_DIR = "thumbnails"; + private const string DEFAULT_GPS_SAT_DIRECTORY = "satellitesDir"; + private const string DEFAULT_GPS_ROUTE_DIRECTORY = "routeDir"; #endregion #region AnnotatorConfig - public static readonly AnnotationConfig DefaultAnnotationConfig = new() - { - DetectionClasses = DefaultAnnotationClasses!, - VideoFormats = DefaultVideoFormats!, - ImageFormats = DefaultImageFormats!, - AnnotationsDbFile = DEFAULT_ANNOTATIONS_DB_FILE - }; - private static readonly List DefaultAnnotationClasses = [ new() { Id = 0, Name = "ArmorVehicle", ShortName = "Броня", Color = "#FF0000".ToColor() }, new() { Id = 1, Name = "Truck", ShortName = "Вантаж.", Color = "#00FF00".ToColor() }, new() { Id = 2, Name = "Vehicle", ShortName = "Машина", Color = "#0000FF".ToColor() }, - new() { Id = 3, Name = "Atillery", ShortName = "Арта", Color = "#FFFF00".ToColor() }, + new() { Id = 3, Name = "Artillery", ShortName = "Арта", Color = "#FFFF00".ToColor() }, new() { Id = 4, Name = "Shadow", ShortName = "Тінь", Color = "#FF00FF".ToColor() }, new() { Id = 5, Name = "Trenches", ShortName = "Окопи", Color = "#00FFFF".ToColor() }, new() { Id = 6, Name = "MilitaryMan", ShortName = "Військов", Color = "#188021".ToColor() }, new() { Id = 7, Name = "TyreTracks", ShortName = "Накати", Color = "#800000".ToColor() }, - new() { Id = 8, Name = "AdditArmoredTank", ShortName = "Танк.захист", Color = "#008000".ToColor() }, + new() { Id = 8, Name = "AdditionArmoredTank",ShortName = "Танк.захист", Color = "#008000".ToColor() }, new() { Id = 9, Name = "Smoke", ShortName = "Дим", Color = "#000080".ToColor() }, new() { Id = 10, Name = "Plane", ShortName = "Літак", Color = "#000080".ToColor() }, new() { Id = 11, Name = "Moto", ShortName = "Мото", Color = "#808000".ToColor() }, @@ -86,20 +80,28 @@ public class Constants new() { Id = 15, Name = "Building", ShortName = "Будівля", Color = "#ffb6c1".ToColor() }, new() { Id = 16, Name = "Caponier", ShortName = "Капонір", Color = "#ffb6c1".ToColor() }, ]; + + private static readonly List DefaultVideoFormats = ["mp4", "mov", "avi"]; + private static readonly List DefaultImageFormats = ["jpg", "jpeg", "png", "bmp"]; - public static readonly List DefaultVideoFormats = ["mp4", "mov", "avi"]; - public static readonly List DefaultImageFormats = ["jpg", "jpeg", "png", "bmp"]; + private static readonly AnnotationConfig DefaultAnnotationConfig = new() + { + DetectionClasses = DefaultAnnotationClasses, + VideoFormats = DefaultVideoFormats, + ImageFormats = DefaultImageFormats, + AnnotationsDbFile = DEFAULT_ANNOTATIONS_DB_FILE + }; + + private const int DEFAULT_LEFT_PANEL_WIDTH = 250; + private const int DEFAULT_RIGHT_PANEL_WIDTH = 250; - public static int DEFAULT_LEFT_PANEL_WIDTH = 250; - public static int DEFAULT_RIGHT_PANEL_WIDTH = 250; - - public const string DEFAULT_ANNOTATIONS_DB_FILE = "annotations.db"; + private const string DEFAULT_ANNOTATIONS_DB_FILE = "annotations.db"; # endregion AnnotatorConfig # region AIRecognitionConfig - public static readonly AIRecognitionConfig DefaultAIRecognitionConfig = new() + private static readonly AIRecognitionConfig DefaultAIRecognitionConfig = new() { FrameRecognitionSeconds = DEFAULT_FRAME_RECOGNITION_SECONDS, TrackingDistanceConfidence = TRACKING_DISTANCE_CONFIDENCE, @@ -109,18 +111,18 @@ public class Constants FramePeriodRecognition = DEFAULT_FRAME_PERIOD_RECOGNITION }; - public const double DEFAULT_FRAME_RECOGNITION_SECONDS = 2; - public const double TRACKING_DISTANCE_CONFIDENCE = 0.15; - public const double TRACKING_PROBABILITY_INCREASE = 15; - public const double TRACKING_INTERSECTION_THRESHOLD = 0.8; - public const int DEFAULT_BIG_IMAGE_TILE_OVERLAP_PERCENT = 20; - public const int DEFAULT_FRAME_PERIOD_RECOGNITION = 4; + private const double DEFAULT_FRAME_RECOGNITION_SECONDS = 2; + private const double TRACKING_DISTANCE_CONFIDENCE = 0.15; + private const double TRACKING_PROBABILITY_INCREASE = 15; + private const double TRACKING_INTERSECTION_THRESHOLD = 0.8; + private const int DEFAULT_BIG_IMAGE_TILE_OVERLAP_PERCENT = 20; + private const int DEFAULT_FRAME_PERIOD_RECOGNITION = 4; # endregion AIRecognitionConfig # region GpsDeniedConfig - public static readonly GpsDeniedConfig DefaultGpsDeniedConfig = new() + private static readonly GpsDeniedConfig DefaultGpsDeniedConfig = new() { MinKeyPoints = 11 }; @@ -129,15 +131,15 @@ public class Constants #region Thumbnails - public static readonly ThumbnailConfig DefaultThumbnailConfig = new() + private static readonly Size DefaultThumbnailSize = new(240, 135); + + private static readonly ThumbnailConfig DefaultThumbnailConfig = new() { Size = DefaultThumbnailSize, Border = DEFAULT_THUMBNAIL_BORDER }; - public static readonly Size DefaultThumbnailSize = new(240, 135); - - public const int DEFAULT_THUMBNAIL_BORDER = 10; + private const int DEFAULT_THUMBNAIL_BORDER = 10; public const string THUMBNAIL_PREFIX = "_thumb"; public const string RESULT_PREFIX = "_result"; @@ -163,10 +165,10 @@ public class Constants #endregion - public const string CSV_PATH = "matches.csv"; + public const string SPLIT_SUFFIX = "!split!"; - public static readonly InitConfig DefaultInitConfig = new() + private static readonly InitConfig DefaultInitConfig = new() { LoaderClientConfig = new LoaderClientConfig { diff --git a/Azaion.Common/Controls/CanvasEditor.cs b/Azaion.Common/Controls/CanvasEditor.cs index 48ea36c..ff4969e 100644 --- a/Azaion.Common/Controls/CanvasEditor.cs +++ b/Azaion.Common/Controls/CanvasEditor.cs @@ -5,6 +5,7 @@ using System.Windows.Input; using System.Windows.Media; using System.Windows.Media.Imaging; using System.Windows.Shapes; +using Azaion.Common.Database; using Azaion.Common.DTO; using Azaion.Common.Events; using MediatR; @@ -39,7 +40,6 @@ public class CanvasEditor : Canvas private readonly TimeSpan _viewThreshold = TimeSpan.FromMilliseconds(400); public Image BackgroundImage { get; set; } = new() { Stretch = Stretch.Uniform }; - public IMediator Mediator { get; set; } = null!; public static readonly DependencyProperty GetTimeFuncProp = DependencyProperty.Register( @@ -191,7 +191,6 @@ public class CanvasEditor : Canvas private void CanvasMouseMove(object sender, MouseEventArgs e) { var pos = e.GetPosition(this); - Mediator.Publish(new SetStatusTextEvent($"Mouse Coordinates: {pos.X}, {pos.Y}")); _horizontalLine.Y1 = _horizontalLine.Y2 = pos.Y; _verticalLine.X1 = _verticalLine.X2 = pos.X; SetLeft(_classNameHint, pos.X + 10); @@ -223,7 +222,6 @@ public class CanvasEditor : Canvas matrix.Translate(delta.X, delta.Y); _matrixTransform.Matrix = matrix; - Mediator.Publish(new SetStatusTextEvent(_matrixTransform.Matrix.ToString())); } private void CanvasMouseUp(object sender, MouseButtonEventArgs e) @@ -243,8 +241,8 @@ public class CanvasEditor : Canvas { Width = width, Height = height, - X = Math.Min(endPos.X, _newAnnotationStartPos.X), - Y = Math.Min(endPos.Y, _newAnnotationStartPos.Y), + Left = Math.Min(endPos.X, _newAnnotationStartPos.X), + Top = Math.Min(endPos.Y, _newAnnotationStartPos.Y), Confidence = 1 }); control.UpdateLayout(); @@ -415,13 +413,26 @@ public class CanvasEditor : Canvas SetTop(_newAnnotationRect, currentPos.Y); } - public void CreateDetections(TimeSpan time, IEnumerable detections, List detectionClasses, Size videoSize) + public void CreateDetections(Annotation annotation, List detectionClasses, Size mediaSize) { - foreach (var detection in detections) + var splitTile = annotation.SplitTile; + foreach (var detection in annotation.Detections) { var detectionClass = DetectionClass.FromYoloId(detection.ClassNumber, detectionClasses); - var canvasLabel = new CanvasLabel(detection, RenderSize, videoSize, detection.Confidence); - CreateDetectionControl(detectionClass, time, canvasLabel); + CanvasLabel canvasLabel; + if (splitTile == null) + canvasLabel = new CanvasLabel(detection, RenderSize, mediaSize, detection.Confidence); + else + { + canvasLabel = new CanvasLabel(detection, new Size(Constants.AI_TILE_SIZE, Constants.AI_TILE_SIZE), null, detection.Confidence) + .ReframeFromSmall(splitTile); + + //From CurrentMediaSize to Render Size + var yoloLabel = new YoloLabel(canvasLabel, mediaSize); + canvasLabel = new CanvasLabel(yoloLabel, RenderSize, mediaSize, canvasLabel.Confidence); + } + + CreateDetectionControl(detectionClass, annotation.Time, canvasLabel); } } @@ -429,8 +440,8 @@ public class CanvasEditor : Canvas { var detectionControl = new DetectionControl(detectionClass, time, AnnotationResizeStart, canvasLabel); detectionControl.MouseDown += AnnotationPositionStart; - SetLeft(detectionControl, canvasLabel.X ); - SetTop(detectionControl, canvasLabel.Y); + SetLeft(detectionControl, canvasLabel.Left ); + SetTop(detectionControl, canvasLabel.Top); Children.Add(detectionControl); CurrentDetections.Add(detectionControl); _newAnnotationRect.Fill = new SolidColorBrush(detectionClass.Color); @@ -472,4 +483,12 @@ public class CanvasEditor : Canvas } public void ResetBackground() => Background = new SolidColorBrush(Color.FromArgb(1, 0, 0, 0)); + + public void ZoomTo(Point point) + { + SetZoom(); + var matrix = _matrixTransform.Matrix; + matrix.ScaleAt(2, 2, point.X, point.Y); + SetZoom(matrix); + } } \ No newline at end of file diff --git a/Azaion.Common/Controls/DetectionControl.cs b/Azaion.Common/Controls/DetectionControl.cs index 9974464..39c8340 100644 --- a/Azaion.Common/Controls/DetectionControl.cs +++ b/Azaion.Common/Controls/DetectionControl.cs @@ -30,7 +30,7 @@ public class DetectionControl : Border { var brush = new SolidColorBrush(value.Color.ToConfidenceColor()); BorderBrush = brush; - BorderThickness = new Thickness(3); + BorderThickness = new Thickness(1); foreach (var rect in _resizedRectangles) rect.Stroke = brush; @@ -141,7 +141,7 @@ public class DetectionControl : Border var rect = new Rectangle() // small rectangles at the corners and sides { ClipToBounds = false, - Margin = new Thickness(-RESIZE_RECT_SIZE), + Margin = new Thickness(-1.1 * RESIZE_RECT_SIZE), HorizontalAlignment = ha, VerticalAlignment = va, Width = RESIZE_RECT_SIZE, diff --git a/Azaion.Common/DTO/AnnotationResult.cs b/Azaion.Common/DTO/AnnotationResult.cs index d3a42a9..48f635a 100644 --- a/Azaion.Common/DTO/AnnotationResult.cs +++ b/Azaion.Common/DTO/AnnotationResult.cs @@ -3,31 +3,33 @@ using Azaion.Common.Database; namespace Azaion.Common.DTO; -public class AnnotationResult -{ - public Annotation Annotation { get; set; } - public List<(Color Color, double Confidence)> Colors { get; private set; } +// public class AnnotationResult +//{ + //public Annotation Annotation { get; set; } + - public string ImagePath { get; set; } - public string TimeStr { get; set; } - public string ClassName { get; set; } + //public string ImagePath { get; set; } + //public string TimeStr { get; set; } + + //public List<(Color Color, double Confidence)> Colors { get; private set; } +// public string ClassName { get; set; } - public AnnotationResult(Dictionary allDetectionClasses, Annotation annotation) - { + // public AnnotationResult(Dictionary allDetectionClasses, Annotation annotation) + // { - Annotation = annotation; + //Annotation = annotation; - TimeStr = $"{annotation.Time:h\\:mm\\:ss}"; - ImagePath = annotation.ImagePath; + //TimeStr = $"{annotation.Time:h\\:mm\\:ss}"; + //ImagePath = annotation.ImagePath; - var detectionClasses = annotation.Detections.Select(x => x.ClassNumber).Distinct().ToList(); - - Colors = annotation.Detections - .Select(d => (allDetectionClasses[d.ClassNumber].Color, d.Confidence)) - .ToList(); - - ClassName = detectionClasses.Count > 1 - ? string.Join(", ", detectionClasses.Select(x => allDetectionClasses[x].UIName)) - : allDetectionClasses[detectionClasses.FirstOrDefault()].UIName; - } -} \ No newline at end of file + // var detectionClasses = annotation.Detections.Select(x => x.ClassNumber).Distinct().ToList(); + // ClassName = detectionClasses.Count > 1 + // ? string.Join(", ", detectionClasses.Select(x => allDetectionClasses[x].UIName)) + // : allDetectionClasses[detectionClasses.FirstOrDefault()].UIName; + // + // Colors = annotation.Detections + // .Select(d => (allDetectionClasses[d.ClassNumber].Color, d.Confidence)) + // .ToList(); + + // } +// } \ No newline at end of file diff --git a/Azaion.Common/DTO/FormState.cs b/Azaion.Common/DTO/FormState.cs index fcac093..7d694d5 100644 --- a/Azaion.Common/DTO/FormState.cs +++ b/Azaion.Common/DTO/FormState.cs @@ -1,5 +1,6 @@ using System.Collections.ObjectModel; using System.Windows; +using Azaion.Common.Database; namespace Azaion.Common.DTO; @@ -7,13 +8,12 @@ public class FormState { public MediaFileInfo? CurrentMedia { get; set; } public string MediaName => CurrentMedia?.FName ?? ""; - - public string CurrentMrl { get; set; } = null!; + public Size CurrentMediaSize { get; set; } public TimeSpan CurrentVideoLength { get; set; } public TimeSpan? BackgroundTime { get; set; } public int CurrentVolume { get; set; } = 100; - public ObservableCollection AnnotationResults { get; set; } = []; + public ObservableCollection AnnotationResults { get; set; } = []; public WindowEnum ActiveWindow { get; set; } } \ No newline at end of file diff --git a/Azaion.Common/DTO/Label.cs b/Azaion.Common/DTO/Label.cs index 65baec7..0a6130b 100644 --- a/Azaion.Common/DTO/Label.cs +++ b/Azaion.Common/DTO/Label.cs @@ -22,52 +22,56 @@ public abstract class Label public class CanvasLabel : Label { - public double X { get; set; } //left - public double Y { get; set; } //top + public double Left { get; set; } + public double Top { get; set; } public double Width { get; set; } public double Height { get; set; } public double Confidence { get; set; } public double Bottom { - get => Y + Height; - set => Height = value - Y; + get => Top + Height; + set => Height = value - Top; } public double Right { - get => X + Width; - set => Width = value - X; + get => Left + Width; + set => Width = value - Left; } + + public double CenterX => Left + Width / 2.0; + public double CenterY => Top + Height / 2.0; + public Size Size => new(Width, Height); public CanvasLabel() { } public CanvasLabel(double left, double right, double top, double bottom) { - X = left; - Y = top; + Left = left; + Top = top; Width = right - left; Height = bottom - top; Confidence = 1; ClassNumber = -1; } - public CanvasLabel(int classNumber, double x, double y, double width, double height, double confidence = 1) : base(classNumber) + public CanvasLabel(int classNumber, double left, double top, double width, double height, double confidence = 1) : base(classNumber) { - X = x; - Y = y; + Left = left; + Top = top; Width = width; Height = height; Confidence = confidence; } - public CanvasLabel(YoloLabel label, Size canvasSize, Size? videoSize = null, double confidence = 1) + public CanvasLabel(YoloLabel label, Size canvasSize, Size? mediaSize = null, double confidence = 1) { var cw = canvasSize.Width; var ch = canvasSize.Height; var canvasAr = cw / ch; - var videoAr = videoSize.HasValue - ? videoSize.Value.Width / videoSize.Value.Height + var videoAr = mediaSize.HasValue + ? mediaSize.Value.Width / mediaSize.Value.Height : canvasAr; ClassNumber = label.ClassNumber; @@ -80,8 +84,8 @@ public class CanvasLabel : Label var realHeight = cw / videoAr; //real video height in pixels on canvas var blackStripHeight = (ch - realHeight) / 2.0; //height of black strips at the top and bottom - X = left * cw; - Y = top * realHeight + blackStripHeight; + Left = left * cw; + Top = top * realHeight + blackStripHeight; Width = label.Width * cw; Height = label.Height * realHeight; } @@ -90,8 +94,8 @@ public class CanvasLabel : Label var realWidth = ch * videoAr; //real video width in pixels on canvas var blackStripWidth = (cw - realWidth) / 2.0; //height of black strips at the top and bottom - X = left * realWidth + blackStripWidth; - Y = top * ch; + Left = left * realWidth + blackStripWidth; + Top = top * ch; Width = label.Width * realWidth; Height = label.Height * ch; } @@ -99,10 +103,10 @@ public class CanvasLabel : Label } public CanvasLabel ReframeToSmall(CanvasLabel smallTile) => - new(ClassNumber, X - smallTile.X, Y - smallTile.Y, Width, Height, Confidence); + new(ClassNumber, Left - smallTile.Left, Top - smallTile.Top, Width, Height, Confidence); public CanvasLabel ReframeFromSmall(CanvasLabel smallTile) => - new(ClassNumber, X + smallTile.X, Y + smallTile.Y, Width, Height, Confidence); + new(ClassNumber, Left + smallTile.Left, Top + smallTile.Top, Width, Height, Confidence); } @@ -132,13 +136,13 @@ public class YoloLabel : Label public RectangleF ToRectangle() => new((float)(CenterX - Width / 2.0), (float)(CenterY - Height / 2.0), (float)Width, (float)Height); - public YoloLabel(CanvasLabel canvasLabel, Size canvasSize, Size? videoSize = null) + public YoloLabel(CanvasLabel canvasLabel, Size canvasSize, Size? mediaSize = null) { var cw = canvasSize.Width; var ch = canvasSize.Height; var canvasAr = cw / ch; - var videoAr = videoSize.HasValue - ? videoSize.Value.Width / videoSize.Value.Height + var videoAr = mediaSize.HasValue + ? mediaSize.Value.Width / mediaSize.Value.Height : canvasAr; ClassNumber = canvasLabel.ClassNumber; @@ -146,20 +150,20 @@ public class YoloLabel : Label double left, top; if (videoAr > canvasAr) //100% width { - left = canvasLabel.X / cw; + left = canvasLabel.Left / cw; Width = canvasLabel.Width / cw; var realHeight = cw / videoAr; //real video height in pixels on canvas var blackStripHeight = (ch - realHeight) / 2.0; //height of black strips at the top and bottom - top = (canvasLabel.Y - blackStripHeight) / realHeight; + top = (canvasLabel.Top - blackStripHeight) / realHeight; Height = canvasLabel.Height / realHeight; } else //100% height { - top = canvasLabel.Y / ch; + top = canvasLabel.Top / ch; Height = canvasLabel.Height / ch; var realWidth = ch * videoAr; //real video width in pixels on canvas var blackStripWidth = (cw - realWidth) / 2.0; //height of black strips at the top and bottom - left = (canvasLabel.X - blackStripWidth) / realWidth; + left = (canvasLabel.Left - blackStripWidth) / realWidth; Width = canvasLabel.Width / realWidth; } diff --git a/Azaion.Common/Database/Annotation.cs b/Azaion.Common/Database/Annotation.cs index c89f2ab..be7970e 100644 --- a/Azaion.Common/Database/Annotation.cs +++ b/Azaion.Common/Database/Annotation.cs @@ -1,4 +1,5 @@ using System.IO; +using System.Windows.Media; using Azaion.Common.DTO; using Azaion.Common.DTO.Config; using Azaion.Common.DTO.Queue; @@ -12,12 +13,14 @@ public class Annotation private static string _labelsDir = null!; private static string _imagesDir = null!; private static string _thumbDir = null!; - - public static void InitializeDirs(DirectoriesConfig config) + private static Dictionary _detectionClassesDict; + + public static void Init(DirectoriesConfig config, Dictionary detectionClassesDict) { _labelsDir = config.LabelsDirectory; _imagesDir = config.ImagesDirectory; _thumbDir = config.ThumbnailsDirectory; + _detectionClassesDict = detectionClassesDict; } [Key("n")] public string Name { get; set; } = null!; @@ -40,12 +43,64 @@ public class Annotation [Key("lon")]public double Lon { get; set; } #region Calculated - [IgnoreMember]public List Classes => Detections.Select(x => x.ClassNumber).ToList(); - [IgnoreMember]public string ImagePath => Path.Combine(_imagesDir, $"{Name}{ImageExtension}"); - [IgnoreMember]public string LabelPath => Path.Combine(_labelsDir, $"{Name}.txt"); - [IgnoreMember]public string ThumbPath => Path.Combine(_thumbDir, $"{Name}{Constants.THUMBNAIL_PREFIX}.jpg"); + [IgnoreMember] public List Classes => Detections.Select(x => x.ClassNumber).ToList(); + [IgnoreMember] public string ImagePath => Path.Combine(_imagesDir, $"{Name}{ImageExtension}"); + [IgnoreMember] public string LabelPath => Path.Combine(_labelsDir, $"{Name}.txt"); + [IgnoreMember] public string ThumbPath => Path.Combine(_thumbDir, $"{Name}{Constants.THUMBNAIL_PREFIX}.jpg"); + [IgnoreMember] public bool IsSplit => Name.Contains(Constants.SPLIT_SUFFIX); + private CanvasLabel? _splitTile; + [IgnoreMember] public CanvasLabel? SplitTile + { + get + { + if (!IsSplit) + return null; + if (_splitTile != null) + return _splitTile; + + var startCoordIndex = Name.IndexOf(Constants.SPLIT_SUFFIX, StringComparison.Ordinal) + Constants.SPLIT_SUFFIX.Length; + var coordsStr = Name.Substring(startCoordIndex, 9).Split('_'); + _splitTile = new CanvasLabel + { + Left = double.Parse(coordsStr[0]), + Top = double.Parse(coordsStr[1]), + Width = Constants.AI_TILE_SIZE, + Height = Constants.AI_TILE_SIZE + }; + return _splitTile; + } + } + + [IgnoreMember] public string TimeStr => $"{Time:h\\:mm\\:ss}"; + + private List<(Color Color, double Confidence)>? _colors; + [IgnoreMember] public List<(Color Color, double Confidence)> Colors => _colors ??= Detections + .Select(d => (_detectionClassesDict[d.ClassNumber].Color, d.Confidence)) + .ToList(); + + private string _className; + [IgnoreMember] public string ClassName + { + get + { + if (string.IsNullOrEmpty(_className)) + { + var detectionClasses = Detections.Select(x => x.ClassNumber).Distinct().ToList(); + _className = detectionClasses.Count > 1 + ? string.Join(", ", detectionClasses.Select(x => _detectionClassesDict[x].UIName)) + : _detectionClassesDict[detectionClasses.FirstOrDefault()].UIName; + } + return _className; + } + } + + #endregion Calculated + + + + } [MessagePackObject] diff --git a/Azaion.Common/Database/DbFactory.cs b/Azaion.Common/Database/DbFactory.cs index 1b85c50..6103939 100644 --- a/Azaion.Common/Database/DbFactory.cs +++ b/Azaion.Common/Database/DbFactory.cs @@ -1,4 +1,5 @@ using System.Data.SQLite; +using System.Diagnostics; using System.IO; using Azaion.Common.DTO; using Azaion.Common.DTO.Config; @@ -48,7 +49,7 @@ public class DbFactory : IDbFactory .UseDataProvider(SQLiteTools.GetDataProvider()) .UseConnection(_memoryConnection) .UseMappingSchema(AnnotationsDbSchemaHolder.MappingSchema) - ;//.UseTracing(TraceLevel.Info, t => logger.LogInformation(t.SqlText)); + .UseTracing(TraceLevel.Info, t => logger.LogInformation(t.SqlText)); _fileConnection = new SQLiteConnection(FileConnStr); diff --git a/Azaion.Common/Services/AnnotationService.cs b/Azaion.Common/Services/AnnotationService.cs index e34edb6..190c396 100644 --- a/Azaion.Common/Services/AnnotationService.cs +++ b/Azaion.Common/Services/AnnotationService.cs @@ -94,6 +94,7 @@ public class AnnotationService : IAnnotationService await SaveAnnotationInner( msg.CreatedDate, msg.OriginalMediaName, + msg.Name, msg.Time, JsonConvert.DeserializeObject>(msg.Detections) ?? [], msg.Source, @@ -136,16 +137,16 @@ public class AnnotationService : IAnnotationService public async Task SaveAnnotation(AnnotationImage a, CancellationToken ct = default) { a.Time = TimeSpan.FromMilliseconds(a.Milliseconds); - return await SaveAnnotationInner(DateTime.UtcNow, a.OriginalMediaName, a.Time, a.Detections.ToList(), + return await SaveAnnotationInner(DateTime.UtcNow, a.OriginalMediaName, a.Name, a.Time, a.Detections.ToList(), SourceEnum.AI, new MemoryStream(a.Image), _api.CurrentUser.Role, _api.CurrentUser.Email, token: ct); } //Manual - public async Task SaveAnnotation(string originalMediaName, TimeSpan time, List detections, Stream? stream = null, CancellationToken token = default) => - await SaveAnnotationInner(DateTime.UtcNow, originalMediaName, time, detections, SourceEnum.Manual, stream, + public async Task SaveAnnotation(string originalMediaName, string annotationName, TimeSpan time, List detections, Stream? stream = null, CancellationToken token = default) => + await SaveAnnotationInner(DateTime.UtcNow, originalMediaName, annotationName, time, detections, SourceEnum.Manual, stream, _api.CurrentUser.Role, _api.CurrentUser.Email, token: token); - private async Task SaveAnnotationInner(DateTime createdDate, string originalMediaName, TimeSpan time, + private async Task SaveAnnotationInner(DateTime createdDate, string originalMediaName, string annotationName, TimeSpan time, List detections, SourceEnum source, Stream? stream, RoleEnum userRole, string createdEmail, @@ -153,21 +154,20 @@ public class AnnotationService : IAnnotationService CancellationToken token = default) { var status = AnnotationStatus.Created; - var fName = originalMediaName.ToTimeName(time); var annotation = await _dbFactory.RunWrite(async db => { var ann = await db.Annotations .LoadWith(x => x.Detections) - .FirstOrDefaultAsync(x => x.Name == fName, token: token); + .FirstOrDefaultAsync(x => x.Name == annotationName, token: token); - await db.Detections.DeleteAsync(x => x.AnnotationName == fName, token: token); + await db.Detections.DeleteAsync(x => x.AnnotationName == annotationName, token: token); if (ann != null) //Annotation is already exists { status = AnnotationStatus.Edited; var annotationUpdatable = db.Annotations - .Where(x => x.Name == fName) + .Where(x => x.Name == annotationName) .Set(x => x.Source, source); if (userRole.IsValidator() && source == SourceEnum.Manual) @@ -188,7 +188,7 @@ public class AnnotationService : IAnnotationService ann = new Annotation { CreatedDate = createdDate, - Name = fName, + Name = annotationName, OriginalMediaName = originalMediaName, Time = time, ImageExtension = Constants.JPG_EXT, @@ -264,6 +264,6 @@ public class AnnotationService : IAnnotationService public interface IAnnotationService { Task SaveAnnotation(AnnotationImage a, CancellationToken ct = default); - Task SaveAnnotation(string originalMediaName, TimeSpan time, List detections, Stream? stream = null, CancellationToken token = default); + Task SaveAnnotation(string originalMediaName, string annotationName, TimeSpan time, List detections, Stream? stream = null, CancellationToken token = default); Task ValidateAnnotations(List annotationNames, bool fromQueue = false, CancellationToken token = default); } \ No newline at end of file diff --git a/Azaion.Common/Services/GalleryService.cs b/Azaion.Common/Services/GalleryService.cs index 883437d..bb1e611 100644 --- a/Azaion.Common/Services/GalleryService.cs +++ b/Azaion.Common/Services/GalleryService.cs @@ -237,11 +237,11 @@ public class GalleryService( .ToList(); if (annotation.Detections.Any()) { - var labelsMinX = labels.Min(x => x.X); - var labelsMaxX = labels.Max(x => x.X + x.Width); + var labelsMinX = labels.Min(x => x.Left); + var labelsMaxX = labels.Max(x => x.Left + x.Width); - var labelsMinY = labels.Min(x => x.Y); - var labelsMaxY = labels.Max(x => x.Y + x.Height); + var labelsMinY = labels.Min(x => x.Top); + var labelsMaxY = labels.Max(x => x.Top + x.Height); var labelsHeight = labelsMaxY - labelsMinY + 2 * border; var labelsWidth = labelsMaxX - labelsMinX + 2 * border; @@ -270,7 +270,7 @@ public class GalleryService( var color = _annotationConfig.DetectionClassesDict[label.ClassNumber].Color; var brush = new SolidBrush(Color.FromArgb(color.A, color.R, color.G, color.B)); - g.DrawRectangle(new Pen(brush, width: 3), (float)((label.X - frameX) / scale), (float)((label.Y - frameY) / scale), (float)(label.Width / scale), (float)(label.Height / scale)); + g.DrawRectangle(new Pen(brush, width: 3), (float)((label.Left - frameX) / scale), (float)((label.Top - frameY) / scale), (float)(label.Width / scale), (float)(label.Height / scale)); } bitmap.Save(annotation.ThumbPath, ImageFormat.Jpeg); @@ -291,10 +291,10 @@ public class GalleryService( var color = detClass.Color; var brush = new SolidBrush(Color.FromArgb(color.A, color.R, color.G, color.B)); var det = new CanvasLabel(detection, new Size(originalImage.Width, originalImage.Height)); - g.DrawRectangle(new Pen(brush, width: 3), (float)det.X, (float)det.Y, (float)det.Width, (float)det.Height); + g.DrawRectangle(new Pen(brush, width: 3), (float)det.Left, (float)det.Top, (float)det.Width, (float)det.Height); var label = detection.Confidence >= 0.995 ? detClass.UIName : $"{detClass.UIName}: {detection.Confidence * 100:F0}%"; - g.DrawTextBox(label, new PointF((float)(det.X + det.Width / 2.0), (float)(det.Y - 24)), brush, Brushes.Black); + g.DrawTextBox(label, new PointF((float)(det.Left + det.Width / 2.0), (float)(det.Top - 24)), brush, Brushes.Black); } var imagePath = Path.Combine(_dirConfig.ResultsDirectory, $"{annotation.Name}{Constants.RESULT_PREFIX}.jpg"); diff --git a/Azaion.Common/Services/InferenceClient.cs b/Azaion.Common/Services/InferenceClient.cs index 7e59620..714311f 100644 --- a/Azaion.Common/Services/InferenceClient.cs +++ b/Azaion.Common/Services/InferenceClient.cs @@ -49,7 +49,7 @@ public class InferenceClient : IInferenceClient Arguments = $"-p {_inferenceClientConfig.ZeroMqPort} -lp {_loaderClientConfig.ZeroMqPort} -a {_inferenceClientConfig.ApiUrl}", CreateNoWindow = true }; - process.Start(); + //process.Start(); } catch (Exception e) { diff --git a/Azaion.Common/Services/TileProcessor.cs b/Azaion.Common/Services/TileProcessor.cs index c534595..71084ec 100644 --- a/Azaion.Common/Services/TileProcessor.cs +++ b/Azaion.Common/Services/TileProcessor.cs @@ -18,10 +18,8 @@ public class TileResult public static class TileProcessor { - private const int MaxTileWidth = 1280; - private const int MaxTileHeight = 1280; - private const int Border = 10; - + public const int BORDER = 10; + public static List Split(Size originalSize, List detections, CancellationToken cancellationToken) { var results = new List(); @@ -30,7 +28,7 @@ public static class TileProcessor while (processingDetectionList.Count > 0 && !cancellationToken.IsCancellationRequested) { var topMostDetection = processingDetectionList - .OrderBy(d => d.Y) + .OrderBy(d => d.Top) .First(); var result = GetDetectionsInTile(originalSize, topMostDetection, processingDetectionList); @@ -42,11 +40,8 @@ public static class TileProcessor private static TileResult GetDetectionsInTile(Size originalSize, CanvasLabel startDet, List allDetections) { - var tile = new CanvasLabel( - left: Math.Max(startDet.X - Border, 0), - right: Math.Min(startDet.Right + Border, originalSize.Width), - top: Math.Max(startDet.Y - Border, 0), - bottom: Math.Min(startDet.Bottom + Border, originalSize.Height)); + var tile = new CanvasLabel(startDet.Left, startDet.Right, startDet.Top, startDet.Bottom); + var maxSize = new List { startDet.Width + BORDER, startDet.Height + BORDER, Constants.AI_TILE_SIZE }.Max(); var selectedDetections = new List{startDet}; foreach (var det in allDetections) @@ -55,26 +50,26 @@ public static class TileProcessor continue; var commonTile = new CanvasLabel( - left: Math.Max(Math.Min(tile.X, det.X) - Border, 0), - right: Math.Min(Math.Max(tile.Right, det.Right) + Border, originalSize.Width), - top: Math.Max(Math.Min(tile.Y, det.Y) - Border, 0), - bottom: Math.Min(Math.Max(tile.Bottom, det.Bottom) + Border, originalSize.Height) + left: Math.Min(tile.Left, det.Left), + right: Math.Max(tile.Right, det.Right), + top: Math.Min(tile.Top, det.Top), + bottom: Math.Max(tile.Bottom, det.Bottom) ); - - if (commonTile.Width > MaxTileWidth || commonTile.Height > MaxTileHeight) + + if (commonTile.Width + BORDER > maxSize || commonTile.Height + BORDER > maxSize) continue; tile = commonTile; selectedDetections.Add(det); } - - //normalization, width and height should be at least half of 1280px - tile.Width = Math.Max(tile.Width, MaxTileWidth / 2.0); - tile.Height = Math.Max(tile.Height, MaxTileHeight / 2.0); - //boundaries check after normalization - tile.Right = Math.Min(tile.Right, originalSize.Width); - tile.Bottom = Math.Min(tile.Bottom, originalSize.Height); + // boundary-aware centering + var centerX = selectedDetections.Average(x => x.CenterX); + var centerY = selectedDetections.Average(d => d.CenterY); + tile.Width = maxSize; + tile.Height = maxSize; + tile.Left = Math.Max(0, Math.Min(originalSize.Width - maxSize, centerX - tile.Width / 2.0)); + tile.Top = Math.Max(0, Math.Min(originalSize.Height - maxSize, centerY - tile.Height / 2.0)); return new TileResult(tile, selectedDetections); } diff --git a/Azaion.Dataset/DatasetExplorer.xaml b/Azaion.Dataset/DatasetExplorer.xaml index 17f05b7..2791540 100644 --- a/Azaion.Dataset/DatasetExplorer.xaml +++ b/Azaion.Dataset/DatasetExplorer.xaml @@ -80,7 +80,7 @@ - + Показувати лише анотації з об'єктами + + + + + + + appConfig, ILogger logger, @@ -199,9 +201,8 @@ public partial class DatasetExplorer }; SwitchTab(toEditor: true); - var time = ann.Time; ExplorerEditor.RemoveAllAnns(); - ExplorerEditor.CreateDetections(time, ann.Detections, _appConfig.AnnotationConfig.DetectionClasses, ExplorerEditor.RenderSize); + ExplorerEditor.CreateDetections(ann, _appConfig.AnnotationConfig.DetectionClasses, ExplorerEditor.RenderSize); } catch (Exception e) { @@ -261,6 +262,7 @@ public partial class DatasetExplorer SelectedAnnotationDict.Clear(); var annThumbnails = _annotationsDict[ExplorerEditor.CurrentAnnClass.YoloId] .WhereIf(withDetectionsOnly, x => x.Value.Detections.Any()) + .WhereIf(TbSearch.Text.Length > 2, x => x.Key.ToLower().Contains(TbSearch.Text)) .Select(x => new AnnotationThumbnail(x.Value, _azaionApi.CurrentUser.Role.IsValidator())) .OrderBy(x => !x.IsSeed) .ThenByDescending(x =>x.Annotation.CreatedDate); @@ -295,4 +297,10 @@ public partial class DatasetExplorer _configUpdater.Save(_appConfig); await ReloadThumbnails(); } + + private void TbSearch_OnTextChanged(object sender, TextChangedEventArgs e) + { + TbSearch.Foreground = TbSearch.Text.Length > 2 ? Brushes.Black : Brushes.Gray; + ThrottleExt.Throttle(ReloadThumbnails, SearchActionId, TimeSpan.FromMilliseconds(400));; + } } diff --git a/Azaion.Dataset/DatasetExplorerEventHandler.cs b/Azaion.Dataset/DatasetExplorerEventHandler.cs index 4d66a4e..60ca2d3 100644 --- a/Azaion.Dataset/DatasetExplorerEventHandler.cs +++ b/Azaion.Dataset/DatasetExplorerEventHandler.cs @@ -70,7 +70,7 @@ public class DatasetExplorerEventHandler( .Select(x => new Detection(a.Name, x.ToYoloLabel(datasetExplorer.ExplorerEditor.RenderSize))) .ToList(); var index = datasetExplorer.ThumbnailsView.SelectedIndex; - var annotation = await annotationService.SaveAnnotation(a.OriginalMediaName, a.Time, detections, token: token); + var annotation = await annotationService.SaveAnnotation(a.OriginalMediaName, a.Name, a.Time, detections, token: token); await ValidateAnnotations([annotation], token); await datasetExplorer.EditAnnotation(index + 1); break; diff --git a/Azaion.Inference/annotation.pxd b/Azaion.Inference/annotation.pxd index 932e969..8bfc0bc 100644 --- a/Azaion.Inference/annotation.pxd +++ b/Azaion.Inference/annotation.pxd @@ -12,5 +12,4 @@ cdef class Annotation: cdef public list[Detection] detections cdef public bytes image - cdef format_time(self, ms) cdef bytes serialize(self) diff --git a/Azaion.Inference/annotation.pyx b/Azaion.Inference/annotation.pyx index 454eda5..485c5cb 100644 --- a/Azaion.Inference/annotation.pyx +++ b/Azaion.Inference/annotation.pyx @@ -1,5 +1,5 @@ import msgpack -from pathlib import Path +cimport constants_inf cdef class Detection: def __init__(self, double x, double y, double w, double h, int cls, double confidence): @@ -14,6 +14,17 @@ cdef class Detection: def __str__(self): return f'{self.cls}: {self.x:.2f} {self.y:.2f} {self.w:.2f} {self.h:.2f}, prob: {(self.confidence*100):.1f}%' + def __eq__(self, other): + if not isinstance(other, Detection): + return False + + if max(abs(self.x - other.x), + abs(self.y - other.y), + abs(self.w - other.w), + abs(self.h - other.h)) > constants_inf.TILE_DUPLICATE_CONFIDENCE_THRESHOLD: + return False + return True + cdef overlaps(self, Detection det2, float confidence_threshold): cdef double overlap_x = 0.5 * (self.w + det2.w) - abs(self.x - det2.x) cdef double overlap_y = 0.5 * (self.h + det2.h) - abs(self.y - det2.y) @@ -23,9 +34,9 @@ cdef class Detection: return overlap_area / min_area > confidence_threshold cdef class Annotation: - def __init__(self, str name, long ms, list[Detection] detections): - self.original_media_name = Path(name).stem.replace(" ", "") - self.name = f'{self.original_media_name}_{self.format_time(ms)}' + def __init__(self, str name, str original_media_name, long ms, list[Detection] detections): + self.name = name + self.original_media_name = original_media_name self.time = ms self.detections = detections if detections is not None else [] for d in self.detections: @@ -42,17 +53,6 @@ cdef class Annotation: ) return f"{self.name}: {detections_str}" - cdef format_time(self, ms): - # Calculate hours, minutes, seconds, and hundreds of milliseconds. - h = ms // 3600000 # Total full hours. - ms_remaining = ms % 3600000 - m = ms_remaining // 60000 # Full minutes. - ms_remaining %= 60000 - s = ms_remaining // 1000 # Full seconds. - f = (ms_remaining % 1000) // 100 # Hundreds of milliseconds. - h = h % 10 - return f"{h}{m:02}{s:02}{f}" - cdef bytes serialize(self): return msgpack.packb({ "n": self.name, diff --git a/Azaion.Inference/constants_inf.pxd b/Azaion.Inference/constants_inf.pxd index 19ac2a3..0dad79f 100644 --- a/Azaion.Inference/constants_inf.pxd +++ b/Azaion.Inference/constants_inf.pxd @@ -13,5 +13,9 @@ cdef str MODELS_FOLDER cdef int SMALL_SIZE_KB +cdef str SPLIT_SUFFIX +cdef int TILE_DUPLICATE_CONFIDENCE_THRESHOLD + cdef log(str log_message) -cdef logerror(str error) \ No newline at end of file +cdef logerror(str error) +cdef format_time(int ms) \ No newline at end of file diff --git a/Azaion.Inference/constants_inf.pyx b/Azaion.Inference/constants_inf.pyx index d552486..1630bc5 100644 --- a/Azaion.Inference/constants_inf.pyx +++ b/Azaion.Inference/constants_inf.pyx @@ -12,6 +12,9 @@ cdef str MODELS_FOLDER = "models" cdef int SMALL_SIZE_KB = 3 +cdef str SPLIT_SUFFIX = "!split!" +cdef int TILE_DUPLICATE_CONFIDENCE_THRESHOLD = 5 + logger.remove() log_format = "[{time:HH:mm:ss} {level}] {message}" logger.add( @@ -40,4 +43,15 @@ cdef log(str log_message): logger.info(log_message) cdef logerror(str error): - logger.error(error) \ No newline at end of file + logger.error(error) + +cdef format_time(int ms): + # Calculate hours, minutes, seconds, and hundreds of milliseconds. + h = ms // 3600000 # Total full hours. + ms_remaining = ms % 3600000 + m = ms_remaining // 60000 # Full minutes. + ms_remaining %= 60000 + s = ms_remaining // 1000 # Full seconds. + f = (ms_remaining % 1000) // 100 # Hundreds of milliseconds. + h = h % 10 + return f"{h}{m:02}{s:02}{f}" diff --git a/Azaion.Inference/inference.pxd b/Azaion.Inference/inference.pxd index 45952f4..781c08a 100644 --- a/Azaion.Inference/inference.pxd +++ b/Azaion.Inference/inference.pxd @@ -9,23 +9,26 @@ cdef class Inference: cdef InferenceEngine engine cdef object on_annotation cdef Annotation _previous_annotation + cdef dict[str, list(Detection)] _tile_detections cdef AIRecognitionConfig ai_config cdef bint stop_signal cdef str model_input cdef int model_width cdef int model_height + cdef int tile_width + cdef int tile_height cdef build_tensor_engine(self, object updater_callback) - cdef init_ai(self) + cpdef init_ai(self) cdef bint is_building_engine cdef bint is_video(self, str filepath) cdef run_inference(self, RemoteCommand cmd) cdef _process_video(self, RemoteCommand cmd, AIRecognitionConfig ai_config, str video_name) - cpdef _process_images(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list[str] image_paths) - cpdef _process_images_inner(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list frame_data) - cpdef split_to_tiles(self, frame, path, img_w, img_h, overlap_percent) + cdef _process_images(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list[str] image_paths) + cdef _process_images_inner(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list frame_data) + cpdef split_to_tiles(self, frame, path, overlap_percent) cdef stop(self) cdef preprocess(self, frames) @@ -33,4 +36,6 @@ cdef class Inference: cdef postprocess(self, output, ai_config) cdef split_list_extend(self, lst, chunk_size) - cdef bint is_valid_annotation(self, Annotation annotation, AIRecognitionConfig ai_config) + cdef bint is_valid_video_annotation(self, Annotation annotation, AIRecognitionConfig ai_config) + cdef bint is_valid_image_annotation(self, Annotation annotation) + cdef remove_tiled_duplicates(self, Annotation annotation) diff --git a/Azaion.Inference/inference.pyx b/Azaion.Inference/inference.pyx index 5e6b16e..05ddc48 100644 --- a/Azaion.Inference/inference.pyx +++ b/Azaion.Inference/inference.pyx @@ -1,5 +1,7 @@ import mimetypes import time +from pathlib import Path + import cv2 import numpy as np cimport constants_inf @@ -54,6 +56,8 @@ cdef class Inference: self.model_input = None self.model_width = 0 self.model_height = 0 + self.tile_width = 0 + self.tile_height = 0 self.engine = None self.is_building_engine = False @@ -93,7 +97,7 @@ cdef class Inference: except Exception as e: updater_callback(f'Error. {str(e)}') - cdef init_ai(self): + cpdef init_ai(self): if self.engine is not None: return @@ -114,6 +118,8 @@ cdef class Inference: self.engine = OnnxEngine(res.data) self.model_height, self.model_width = self.engine.get_input_shape() + self.tile_width = self.model_width + self.tile_height = self.model_height cdef preprocess(self, frames): blobs = [cv2.dnn.blobFromImage(frame, @@ -211,11 +217,11 @@ cdef class Inference: images.append(m) # images first, it's faster if len(images) > 0: - constants_inf.log(f'run inference on {" ".join(images)}...') + constants_inf.log(f'run inference on {" ".join(images)}...') self._process_images(cmd, ai_config, images) if len(videos) > 0: for v in videos: - constants_inf.log(f'run inference on {v}...') + constants_inf.log(f'run inference on {v}...') self._process_video(cmd, ai_config, v) @@ -223,8 +229,10 @@ cdef class Inference: cdef int frame_count = 0 cdef list batch_frames = [] cdef list[int] batch_timestamps = [] + cdef Annotation annotation self._previous_annotation = None + v_input = cv2.VideoCapture(video_name) while v_input.isOpened() and not self.stop_signal: ret, frame = v_input.read() @@ -244,8 +252,12 @@ cdef class Inference: list_detections = self.postprocess(outputs, ai_config) for i in range(len(list_detections)): detections = list_detections[i] - annotation = Annotation(video_name, batch_timestamps[i], detections) - if self.is_valid_annotation(annotation, ai_config): + + original_media_name = Path(video_name).stem.replace(" ", "") + name = f'{original_media_name}_{constants_inf.format_time(batch_timestamps[i])}' + annotation = Annotation(name, original_media_name, batch_timestamps[i], detections) + + if self.is_valid_video_annotation(annotation, ai_config): _, image = cv2.imencode('.jpg', batch_frames[i]) annotation.image = image.tobytes() self._previous_annotation = annotation @@ -256,71 +268,104 @@ cdef class Inference: v_input.release() - cpdef _process_images(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list[str] image_paths): - cdef list frame_data = [] + cdef _process_images(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list[str] image_paths): + cdef list frame_data + self._tile_detections = {} for path in image_paths: + frame_data = [] frame = cv2.imread(path) + img_h, img_w, _ = frame.shape if frame is None: constants_inf.logerror(f'Failed to read image {path}') continue - img_h, img_w, _ = frame.shape + original_media_name = Path( path).stem.replace(" ", "") if img_h <= 1.5 * self.model_height and img_w <= 1.5 * self.model_width: - frame_data.append((frame, path)) + frame_data.append((frame, original_media_name, f'{original_media_name}_000000')) else: - (split_frames, split_pats) = self.split_to_tiles(frame, path, img_w, img_h, ai_config.big_image_tile_overlap_percent) - frame_data.extend(zip(split_frames, split_pats)) + res = self.split_to_tiles(frame, path, ai_config.big_image_tile_overlap_percent) + frame_data.extend(res) + if len(frame_data) > self.engine.get_batch_size(): + for chunk in self.split_list_extend(frame_data, self.engine.get_batch_size()): + self._process_images_inner(cmd, ai_config, chunk) for chunk in self.split_list_extend(frame_data, self.engine.get_batch_size()): self._process_images_inner(cmd, ai_config, chunk) - cpdef split_to_tiles(self, frame, path, img_w, img_h, overlap_percent): - stride_w = self.model_width * (1 - overlap_percent / 100) - stride_h = self.model_height * (1 - overlap_percent / 100) - n_tiles_x = int(np.ceil((img_w - self.model_width) / stride_w)) + 1 - n_tiles_y = int(np.ceil((img_h - self.model_height) / stride_h)) + 1 + cpdef split_to_tiles(self, frame, path, overlap_percent): + constants_inf.log(f'splitting image {path} to tiles...') + img_h, img_w, _ = frame.shape + stride_w = int(self.tile_width * (1 - overlap_percent / 100)) + stride_h = int(self.tile_height * (1 - overlap_percent / 100)) results = [] - for y_idx in range(n_tiles_y): - for x_idx in range(n_tiles_x): - y_start = y_idx * stride_w - x_start = x_idx * stride_h + original_media_name = Path( path).stem.replace(" ", "") + for y in range(0, img_h, stride_h): + for x in range(0, img_w, stride_w): + x_end = min(x + self.tile_width, img_w) + y_end = min(y + self.tile_height, img_h) - # Ensure the tile doesn't go out of bounds - y_end = min(y_start + self.model_width, img_h) - x_end = min(x_start + self.model_height, img_w) + # correct x,y for the close-to-border tiles + if x_end - x < self.tile_width: + if img_w - (x - stride_w) <= self.tile_width: + continue # the previous tile already covered the last gap + x = img_w - self.tile_width + if y_end - y < self.tile_height: + if img_h - (y - stride_h) <= self.tile_height: + continue # the previous tile already covered the last gap + y = img_h - self.tile_height - # We need to re-calculate start if we are at the edge to get a full 1280x1280 tile - if y_end == img_h: - y_start = img_h - self.model_height - if x_end == img_w: - x_start = img_w - self.model_width - - tile = frame[y_start:y_end, x_start:x_end] - name = path.stem + f'.tile_{x_start}_{y_start}' + path.suffix - results.append((tile, name)) + tile = frame[y:y_end, x:x_end] + name = f'{original_media_name}{constants_inf.SPLIT_SUFFIX}{x:04d}_{y:04d}!_000000' + results.append((tile, original_media_name, name)) return results - cpdef _process_images_inner(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list frame_data): - frames = [frame for frame, _ in frame_data] + cdef _process_images_inner(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list frame_data): + cdef list frames, original_media_names, names + cdef Annotation annotation + frames, original_media_names, names = map(list, zip(*frame_data)) input_blob = self.preprocess(frames) - outputs = self.engine.run(input_blob) list_detections = self.postprocess(outputs, ai_config) for i in range(len(list_detections)): - detections = list_detections[i] - annotation = Annotation(frame_data[i][1], 0, detections) - _, image = cv2.imencode('.jpg', frames[i]) - annotation.image = image.tobytes() - self.on_annotation(cmd, annotation) + annotation = Annotation(names[i], original_media_names[i], 0, list_detections[i]) + if self.is_valid_image_annotation(annotation): + _, image = cv2.imencode('.jpg', frames[i]) + annotation.image = image.tobytes() + self.on_annotation(cmd, annotation) cdef stop(self): self.stop_signal = True - cdef bint is_valid_annotation(self, Annotation annotation, AIRecognitionConfig ai_config): - # No detections, invalid + cdef remove_tiled_duplicates(self, Annotation annotation): + right = annotation.name.rindex('!') + left = annotation.name.index(constants_inf.SPLIT_SUFFIX) + len(constants_inf.SPLIT_SUFFIX) + x_str, y_str = annotation.name[left:right].split('_') + x = int(x_str) + y = int(y_str) + + for det in annotation.detections: + x1 = det.x * self.tile_width + y1 = det.y * self.tile_height + det_abs = Detection(x + x1, y + y1, det.w * self.tile_width, det.h * self.tile_height, det.cls, det.confidence) + detections = self._tile_detections.setdefault(annotation.original_media_name, []) + if det_abs in detections: + annotation.detections.remove(det) + else: + detections.append(det_abs) + + cdef bint is_valid_image_annotation(self, Annotation annotation): + if constants_inf.SPLIT_SUFFIX in annotation.name: + self.remove_tiled_duplicates(annotation) + if not annotation.detections: + return False + return True + + cdef bint is_valid_video_annotation(self, Annotation annotation, AIRecognitionConfig ai_config): + if constants_inf.SPLIT_SUFFIX in annotation.name: + self.remove_tiled_duplicates(annotation) if not annotation.detections: return False diff --git a/Azaion.Inference/setup.py b/Azaion.Inference/setup.py index 9e3edcc..54901f1 100644 --- a/Azaion.Inference/setup.py +++ b/Azaion.Inference/setup.py @@ -2,15 +2,15 @@ from setuptools import setup, Extension from Cython.Build import cythonize import numpy as np -# debug_args = {} -# trace_line = False +debug_args = {} +trace_line = False -debug_args = { - 'extra_compile_args': ['-O0', '-g'], - 'extra_link_args': ['-g'], - 'define_macros': [('CYTHON_TRACE_NOGIL', '1')] -} -trace_line = True +# debug_args = { +# 'extra_compile_args': ['-O0', '-g'], +# 'extra_link_args': ['-g'], +# 'define_macros': [('CYTHON_TRACE_NOGIL', '1')] +# } +# trace_line = True extensions = [ Extension('constants_inf', ['constants_inf.pyx'], **debug_args), diff --git a/Azaion.Inference/test/test_inference.py b/Azaion.Inference/test/test_inference.py index 6407ad2..e3047d3 100644 --- a/Azaion.Inference/test/test_inference.py +++ b/Azaion.Inference/test/test_inference.py @@ -1,8 +1,30 @@ import inference from ai_config import AIRecognitionConfig -from remote_command_inf import RemoteCommand +from unittest.mock import Mock +import numpy as np + +from loader_client import LoaderClient -def test_process_images(): - inf = inference.Inference(None, None) - inf._process_images(RemoteCommand(30), AIRecognitionConfig(4, 2, 15, 0.15, 15, 0.8, 20, b'test', [], 4), ['test_img01.JPG', 'test_img02.jpg']) \ No newline at end of file +def test_split_to_tiles(): + loader_client = LoaderClient("test", 0) + ai_config = AIRecognitionConfig( + frame_period_recognition=4, + frame_recognition_seconds=2, + probability_threshold=0.2, + + tracking_distance_confidence=0.15, + tracking_probability_increase=0.15, + tracking_intersection_threshold=0.6, + big_image_tile_overlap_percent=20, + + file_data=None, + paths=[], + model_batch_size=4 + ) + inf = inference.Inference(loader_client, ai_config) + test_frame = np.zeros((6336, 8448, 3), dtype=np.uint8) + + inf.init_ai() + inf.split_to_tiles(test_frame, 'test_image.jpg', ai_config.big_image_tile_overlap_percent) + diff --git a/Azaion.Loader/hardware_service.pyx b/Azaion.Loader/hardware_service.pyx index 997ef1b..ccc6641 100644 --- a/Azaion.Loader/hardware_service.pyx +++ b/Azaion.Loader/hardware_service.pyx @@ -2,9 +2,14 @@ import os import subprocess cimport constants cdef class HardwareService: + cdef str _CACHED_HW_INFO = None @staticmethod cdef str get_hardware_info(): + global _CACHED_HW_INFO + if _CACHED_HW_INFO is not None: + return _CACHED_HW_INFO + if os.name == 'nt': # windows os_command = ( "powershell -Command \"" @@ -34,5 +39,6 @@ cdef class HardwareService: cdef str drive_serial = lines[len_lines-1] cdef str res = f'CPU: {cpu}. GPU: {gpu}. Memory: {memory}. DriveSerial: {drive_serial}' - constants.log(f'Gathered hardware: {res}') + constants.log(f'Gathered hardware: {res}') + _CACHED_HW_INFO = res return res diff --git a/Azaion.Suite/App.xaml.cs b/Azaion.Suite/App.xaml.cs index 1b62fbc..0f7b671 100644 --- a/Azaion.Suite/App.xaml.cs +++ b/Azaion.Suite/App.xaml.cs @@ -175,7 +175,9 @@ public partial class App }) .Build(); - Annotation.InitializeDirs(_host.Services.GetRequiredService>().Value); + Annotation.Init(_host.Services.GetRequiredService>().Value, + _host.Services.GetRequiredService>().Value.DetectionClassesDict); + _host.Services.GetRequiredService(); _mediator = _host.Services.GetRequiredService(); diff --git a/Azaion.Suite/config.json b/Azaion.Suite/config.json index 748dbb9..c5595a5 100644 --- a/Azaion.Suite/config.json +++ b/Azaion.Suite/config.json @@ -17,10 +17,10 @@ "DirectoriesConfig": { "ApiResourcesDirectory": "stage", "VideosDirectory": "E:\\Azaion6", - "LabelsDirectory": "E:\\labels", - "ImagesDirectory": "E:\\images", - "ResultsDirectory": "E:\\results", - "ThumbnailsDirectory": "E:\\thumbnails", + "LabelsDirectory": "E:\\labels_test", + "ImagesDirectory": "E:\\images_test", + "ResultsDirectory": "E:\\results_test", + "ThumbnailsDirectory": "E:\\thumbnails_test", "GpsSatDirectory": "satellitesDir", "GpsRouteDirectory": "routeDir" }, diff --git a/Azaion.Suite/config.system.json b/Azaion.Suite/config.system.json index c740a46..7b2dce5 100644 --- a/Azaion.Suite/config.system.json +++ b/Azaion.Suite/config.system.json @@ -29,7 +29,7 @@ "ProbabilityThreshold": 0.25, "TrackingDistanceConfidence": 0.15, - "TrackingProbabilityIncrease": 15.0, + "TrackingProbabilityIncrease": 0.15, "TrackingIntersectionThreshold": 0.6, "BigImageTileOverlapPercent": 20, diff --git a/Azaion.Test/TileProcessorTest.cs b/Azaion.Test/TileProcessorTest.cs new file mode 100644 index 0000000..3c7b98e --- /dev/null +++ b/Azaion.Test/TileProcessorTest.cs @@ -0,0 +1,263 @@ +using System.Windows; +using Azaion.Common; +using Azaion.Common.DTO; +using Azaion.Common.Services; +using Xunit; + +namespace Azaion.Annotator.Test; + + public class TileProcessorTest +{ + private const int IMAGE_SIZE = 5000; + + [Fact] + public void Split_DetectionsNearImageCorners_ShouldCreateFourTiles() + { + // Arrange + var originalSize = new Size(IMAGE_SIZE, IMAGE_SIZE); + var detections = new List + { + new(10, 60, 10, 60), // Top-left corner + new(IMAGE_SIZE - 60, IMAGE_SIZE - 10, 10, 60), // Top-right corner + new(10, 60, IMAGE_SIZE - 60, IMAGE_SIZE - 10), // Bottom-left corner + new(IMAGE_SIZE - 60, IMAGE_SIZE - 10, IMAGE_SIZE - 60, IMAGE_SIZE - 10) // Bottom-right corner + }; + + // Act + var results = TileProcessor.Split(originalSize, detections, CancellationToken.None); + + // Assert + Assert.Equal(4, results.Count); + } + + [Fact] + public void Split_DetectionsFarApartButFitInOneTile_ShouldCreateOneTile() + { + // Arrange + var originalSize = new Size(IMAGE_SIZE, IMAGE_SIZE); + var detections = new List + { + new(100, 150, 100, 150), + new(1200, 1250, 1200, 1250) + }; + + // Act + var results = TileProcessor.Split(originalSize, detections, CancellationToken.None); + + // Assert + Assert.Single(results); + Assert.Equal(2, results[0].Detections.Count); + } + + [Fact] + public void Split_DetectionsTooFarApart_ShouldCreateMultipleTiles() + { + // Arrange + var originalSize = new Size(IMAGE_SIZE, IMAGE_SIZE); + var detections = new List + { + new(100, 150, 100, 150), + new(2000, 2050, 2000, 2050) // More than Constants.AI_TILE_SIZE away + }; + + // Act + var results = TileProcessor.Split(originalSize, detections, CancellationToken.None); + + // Assert + Assert.Equal(2, results.Count); + Assert.Contains(results, r => r.Detections.Count == 1 && r.Detections.Contains(detections[0])); + Assert.Contains(results, r => r.Detections.Count == 1 && r.Detections.Contains(detections[1])); + } + + [Fact] + public void Split_ComplexScenario_ShouldCreateCorrectNumberOfTiles() + { + // Arrange + var originalSize = new Size(IMAGE_SIZE, IMAGE_SIZE); + var detections = new List + { + // Group 1 (should be tiled together) + new(100, 150, 100, 150), + new(200, 250, 200, 250), + new(500, 550, 500, 550), + // Group 2 (far from group 1, should be in a separate tile) + new(3000, 3050, 3000, 3050), + new(3100, 3150, 3100, 3150), + }; + + // Act + var results = TileProcessor.Split(originalSize, detections, CancellationToken.None); + + // Assert + Assert.Equal(2, results.Count); + var group1Tile = results.FirstOrDefault(r => r.Detections.Count == 3); + var group2Tile = results.FirstOrDefault(r => r.Detections.Count == 2); + + Assert.NotNull(group1Tile); + Assert.NotNull(group2Tile); + + Assert.Contains(detections[0], group1Tile.Detections); + Assert.Contains(detections[1], group1Tile.Detections); + Assert.Contains(detections[2], group1Tile.Detections); + + Assert.Contains(detections[3], group2Tile.Detections); + Assert.Contains(detections[4], group2Tile.Detections); + } + + [Fact] + public void Split_NoDetections_ShouldReturnEmptyList() + { + // Arrange + var originalSize = new Size(IMAGE_SIZE, IMAGE_SIZE); + var detections = new List(); + + // Act + var results = TileProcessor.Split(originalSize, detections, CancellationToken.None); + + // Assert + Assert.Empty(results); + } + + [Fact] + public void Split_OneDetection_ShouldCreateOneTile() + { + // Arrange + var originalSize = new Size(IMAGE_SIZE, IMAGE_SIZE); + var detections = new List { new(100, 150, 100, 150) }; + + // Act + var results = TileProcessor.Split(originalSize, detections, CancellationToken.None); + + // Assert + Assert.Single(results); + Assert.Single(results[0].Detections); + Assert.Equal(detections[0], results[0].Detections[0]); + } + + [Fact] + public void Split_DetectionsOnTileBoundary_ShouldFitInOneTile() + { + // Arrange + var originalSize = new Size(IMAGE_SIZE, IMAGE_SIZE); + // Combined width is 1270. 1270 + BORDER (10) is not > Constants.AI_TILE_SIZE (1280), so they fit. + var detections = new List + { + new(0, 50, 0, 50), + new(Constants.AI_TILE_SIZE - TileProcessor.BORDER - 50, Constants.AI_TILE_SIZE - TileProcessor.BORDER, 0, 50) + }; + + // Act + var results = TileProcessor.Split(originalSize, detections, CancellationToken.None); + + // Assert + Assert.Single(results); + Assert.Equal(2, results[0].Detections.Count); + } + + [Fact] + public void Split_DetectionsJustOverTileBoundary_ShouldCreateTwoTiles() + { + // Arrange + var originalSize = new Size(IMAGE_SIZE, IMAGE_SIZE); + // Combined width is 1271. 1271 + BORDER (10) is > Constants.AI_TILE_SIZE (1280), so they don't fit. + var detections = new List + { + new(0, 50, 1000, 1050), // Top-most + new(Constants.AI_TILE_SIZE - TileProcessor.BORDER - 49, Constants.AI_TILE_SIZE - TileProcessor.BORDER + 1, 0, 50) + }; + + // Act + var results = TileProcessor.Split(originalSize, detections, CancellationToken.None); + + // Assert + Assert.Equal(2, results.Count); + } + + [Fact] + public void Split_ResultingTiles_ShouldBeWithinImageBoundaries() + { + // Arrange + var originalSize = new Size(IMAGE_SIZE, IMAGE_SIZE); + var detections = new List + { + new(10, 60, 10, 60), // Top-left corner + new(IMAGE_SIZE - 60, IMAGE_SIZE - 10, IMAGE_SIZE - 60, IMAGE_SIZE - 10) // Bottom-right corner + }; + + // Act + var results = TileProcessor.Split(originalSize, detections, CancellationToken.None); + + // Assert + Assert.Equal(2, results.Count); + foreach (var result in results) + { + var tile = result.Tile; + Assert.True(tile.Left >= 0, $"Tile Left boundary {tile.Left} is out of bounds."); + Assert.True(tile.Top >= 0, $"Tile Top boundary {tile.Top} is out of bounds."); + Assert.True(tile.Right <= originalSize.Width, $"Tile Right boundary {tile.Right} is out of bounds."); + Assert.True(tile.Bottom <= originalSize.Height, $"Tile Bottom boundary {tile.Bottom} is out of bounds."); + } + } + + [Fact] + public void Split_ChainedDetections_ShouldCreateOneTile() + { + // Arrange + var originalSize = new Size(IMAGE_SIZE, IMAGE_SIZE); + var detections = new List + { + new(100, 200, 100, 200), // Detection A + new(600, 700, 600, 700), // Detection B (close to A) + new(1100, 1200, 1100, 1200) // Detection C (close to B, but far from A) + }; + + // Act + var results = TileProcessor.Split(originalSize, detections, CancellationToken.None); + + // Assert + Assert.Single(results); + Assert.Equal(3, results[0].Detections.Count); + } + + [Fact] + public void Split_SingleDetectionLargerThanTileSize_ShouldCreateOneTile() + { + // Arrange + var originalSize = new Size(IMAGE_SIZE, IMAGE_SIZE); + var largeDetection = new CanvasLabel(100, 100 + Constants.AI_TILE_SIZE + 100, 100, 200); + var detections = new List { largeDetection }; + + // Act + var results = TileProcessor.Split(originalSize, detections, CancellationToken.None); + + // Assert + Assert.Single(results); + var resultTile = results[0]; + Assert.Single(resultTile.Detections); + Assert.Equal(largeDetection, resultTile.Detections[0]); + // The tile should be at least as large as the detection it contains. + Assert.True(resultTile.Tile.Width >= largeDetection.Width); + Assert.True(resultTile.Tile.Height >= largeDetection.Height); + } + + [Fact] + public void Split_LargeDetectionWithNearbySmallDetection_ShouldCreateOneTile() + { + // Arrange + var originalSize = new Size(IMAGE_SIZE, IMAGE_SIZE); + var largeTallDetection = new CanvasLabel(100, 150, 100, 100 + Constants.AI_TILE_SIZE + 200); + var smallDetectionNearby = new CanvasLabel(largeTallDetection.Right + 15, largeTallDetection.Right + 35, 700, 720); + + var detections = new List { largeTallDetection, smallDetectionNearby }; + + // Act + var results = TileProcessor.Split(originalSize, detections, CancellationToken.None); + + // Assert + Assert.Single(results); + Assert.Equal(2, results[0].Detections.Count); + Assert.Contains(largeTallDetection, results[0].Detections); + Assert.Contains(smallDetectionNearby, results[0].Detections); + } + +} \ No newline at end of file From 9e4dc5404ca896bd305689dc51d4c5bfefaa6130 Mon Sep 17 00:00:00 2001 From: Oleksandr Bezdieniezhnykh Date: Tue, 12 Aug 2025 14:53:14 +0300 Subject: [PATCH 07/14] remove cpdef, add constants h --- Azaion.Inference/constants_inf.h | 55 ++++++++++++++++++++++++++++ Azaion.Inference/inference.pxd | 4 +- Azaion.Inference/inference.pyx | 4 +- Azaion.Inference/onnx_engine.pyx | 6 +-- Azaion.Inference/tensorrt_engine.pxd | 6 +-- Azaion.Inference/tensorrt_engine.pyx | 6 +-- 6 files changed, 68 insertions(+), 13 deletions(-) create mode 100644 Azaion.Inference/constants_inf.h diff --git a/Azaion.Inference/constants_inf.h b/Azaion.Inference/constants_inf.h new file mode 100644 index 0000000..620b985 --- /dev/null +++ b/Azaion.Inference/constants_inf.h @@ -0,0 +1,55 @@ +/* Generated by Cython 3.1.2 */ + +#ifndef __PYX_HAVE__constants_inf +#define __PYX_HAVE__constants_inf + +#include "Python.h" + +#ifndef __PYX_HAVE_API__constants_inf + +#ifdef CYTHON_EXTERN_C + #undef __PYX_EXTERN_C + #define __PYX_EXTERN_C CYTHON_EXTERN_C +#elif defined(__PYX_EXTERN_C) + #ifdef _MSC_VER + #pragma message ("Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead.") + #else + #warning Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead. + #endif +#else + #ifdef __cplusplus + #define __PYX_EXTERN_C extern "C" + #else + #define __PYX_EXTERN_C extern + #endif +#endif + +#ifndef DL_IMPORT + #define DL_IMPORT(_T) _T +#endif + +__PYX_EXTERN_C int TILE_DUPLICATE_CONFIDENCE_THRESHOLD; + +#endif /* !__PYX_HAVE_API__constants_inf */ + +/* WARNING: the interface of the module init function changed in CPython 3.5. */ +/* It now returns a PyModuleDef instance instead of a PyModule instance. */ + +/* WARNING: Use PyImport_AppendInittab("constants_inf", PyInit_constants_inf) instead of calling PyInit_constants_inf directly from Python 3.5 */ +PyMODINIT_FUNC PyInit_constants_inf(void); + +#if PY_VERSION_HEX >= 0x03050000 && (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER) || (defined(__cplusplus) && __cplusplus >= 201402L)) +#if defined(__cplusplus) && __cplusplus >= 201402L +[[deprecated("Use PyImport_AppendInittab(\"constants_inf\", PyInit_constants_inf) instead of calling PyInit_constants_inf directly.")]] inline +#elif defined(__GNUC__) || defined(__clang__) +__attribute__ ((__deprecated__("Use PyImport_AppendInittab(\"constants_inf\", PyInit_constants_inf) instead of calling PyInit_constants_inf directly."), __unused__)) __inline__ +#elif defined(_MSC_VER) +__declspec(deprecated("Use PyImport_AppendInittab(\"constants_inf\", PyInit_constants_inf) instead of calling PyInit_constants_inf directly.")) __inline +#endif +static PyObject* __PYX_WARN_IF_PyInit_constants_inf_INIT_CALLED(PyObject* res) { + return res; +} +#define PyInit_constants_inf() __PYX_WARN_IF_PyInit_constants_inf_INIT_CALLED(PyInit_constants_inf()) +#endif + +#endif /* !__PYX_HAVE__constants_inf */ diff --git a/Azaion.Inference/inference.pxd b/Azaion.Inference/inference.pxd index 781c08a..5799b22 100644 --- a/Azaion.Inference/inference.pxd +++ b/Azaion.Inference/inference.pxd @@ -20,7 +20,7 @@ cdef class Inference: cdef int tile_height cdef build_tensor_engine(self, object updater_callback) - cpdef init_ai(self) + cdef init_ai(self) cdef bint is_building_engine cdef bint is_video(self, str filepath) @@ -28,7 +28,7 @@ cdef class Inference: cdef _process_video(self, RemoteCommand cmd, AIRecognitionConfig ai_config, str video_name) cdef _process_images(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list[str] image_paths) cdef _process_images_inner(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list frame_data) - cpdef split_to_tiles(self, frame, path, overlap_percent) + cdef split_to_tiles(self, frame, path, overlap_percent) cdef stop(self) cdef preprocess(self, frames) diff --git a/Azaion.Inference/inference.pyx b/Azaion.Inference/inference.pyx index 05ddc48..083b86b 100644 --- a/Azaion.Inference/inference.pyx +++ b/Azaion.Inference/inference.pyx @@ -97,7 +97,7 @@ cdef class Inference: except Exception as e: updater_callback(f'Error. {str(e)}') - cpdef init_ai(self): + cdef init_ai(self): if self.engine is not None: return @@ -292,7 +292,7 @@ cdef class Inference: self._process_images_inner(cmd, ai_config, chunk) - cpdef split_to_tiles(self, frame, path, overlap_percent): + cdef split_to_tiles(self, frame, path, overlap_percent): constants_inf.log(f'splitting image {path} to tiles...') img_h, img_w, _ = frame.shape stride_w = int(self.tile_width * (1 - overlap_percent / 100)) diff --git a/Azaion.Inference/onnx_engine.pyx b/Azaion.Inference/onnx_engine.pyx index 15ddd1b..cc64f4c 100644 --- a/Azaion.Inference/onnx_engine.pyx +++ b/Azaion.Inference/onnx_engine.pyx @@ -15,12 +15,12 @@ cdef class OnnxEngine(InferenceEngine): model_meta = self.session.get_modelmeta() constants_inf.log(f"Metadata: {model_meta.custom_metadata_map}") - cpdef tuple get_input_shape(self): + cdef tuple get_input_shape(self): shape = self.input_shape return shape[2], shape[3] - cpdef int get_batch_size(self): + cdef int get_batch_size(self): return self.batch_size - cpdef run(self, input_data): + cdef run(self, input_data): return self.session.run(None, {self.input_name: input_data}) \ No newline at end of file diff --git a/Azaion.Inference/tensorrt_engine.pxd b/Azaion.Inference/tensorrt_engine.pxd index 6fc31bd..5c0f565 100644 --- a/Azaion.Inference/tensorrt_engine.pxd +++ b/Azaion.Inference/tensorrt_engine.pxd @@ -17,8 +17,8 @@ cdef class TensorRTEngine(InferenceEngine): cdef object stream - cpdef tuple get_input_shape(self) + cdef tuple get_input_shape(self) - cpdef int get_batch_size(self) + cdef int get_batch_size(self) - cpdef run(self, input_data) + cdef run(self, input_data) diff --git a/Azaion.Inference/tensorrt_engine.pyx b/Azaion.Inference/tensorrt_engine.pyx index c792340..a0a03e7 100644 --- a/Azaion.Inference/tensorrt_engine.pyx +++ b/Azaion.Inference/tensorrt_engine.pyx @@ -112,13 +112,13 @@ cdef class TensorRTEngine(InferenceEngine): constants_inf.log('conversion done!') return bytes(plan) - cpdef tuple get_input_shape(self): + cdef tuple get_input_shape(self): return self.input_shape[2], self.input_shape[3] - cpdef int get_batch_size(self): + cdef int get_batch_size(self): return self.batch_size - cpdef run(self, input_data): + cdef run(self, input_data): try: cuda.memcpy_htod_async(self.d_input, input_data, self.stream) self.context.set_tensor_address(self.input_name, int(self.d_input)) # input buffer From 16e5853d67d9fcd924256db207b716c29c286f82 Mon Sep 17 00:00:00 2001 From: Oleksandr Bezdieniezhnykh Date: Tue, 12 Aug 2025 14:58:21 +0300 Subject: [PATCH 08/14] put constant tile size temporarily --- Azaion.Inference/inference.pyx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Azaion.Inference/inference.pyx b/Azaion.Inference/inference.pyx index 083b86b..467227b 100644 --- a/Azaion.Inference/inference.pyx +++ b/Azaion.Inference/inference.pyx @@ -118,8 +118,9 @@ cdef class Inference: self.engine = OnnxEngine(res.data) self.model_height, self.model_width = self.engine.get_input_shape() - self.tile_width = self.model_width - self.tile_height = self.model_height + #todo: temporarily, send it from the client + self.tile_width = 550 + self.tile_height = 550 cdef preprocess(self, frames): blobs = [cv2.dnn.blobFromImage(frame, From 4780e8c61ce896005c1efe971cdf49dd50664a9a Mon Sep 17 00:00:00 2001 From: Oleksandr Bezdieniezhnykh Date: Wed, 13 Aug 2025 10:12:25 +0300 Subject: [PATCH 09/14] fix detection label fix schema migrator for enums --- Azaion.Common/Controls/DetectionControl.cs | 5 +++-- .../Controls/DetectionLabelPanel.xaml | 6 +++--- .../Controls/DetectionLabelPanel.xaml.cs | 21 ++++++++++++++++--- Azaion.Common/DTO/Label.cs | 1 + Azaion.Common/Database/Annotation.cs | 11 +++++----- Azaion.Common/Database/SchemaMigrator.cs | 3 +++ Azaion.Common/Extensions/EnumExtensions.cs | 12 +++++++++++ 7 files changed, 45 insertions(+), 14 deletions(-) create mode 100644 Azaion.Common/Extensions/EnumExtensions.cs diff --git a/Azaion.Common/Controls/DetectionControl.cs b/Azaion.Common/Controls/DetectionControl.cs index 39c8340..e3bf6a1 100644 --- a/Azaion.Common/Controls/DetectionControl.cs +++ b/Azaion.Common/Controls/DetectionControl.cs @@ -5,7 +5,7 @@ using System.Windows.Media; using System.Windows.Shapes; using Azaion.Common.DTO; using Azaion.Common.Extensions; -using Label = System.Windows.Controls.Label; +using Annotation = Azaion.Common.Database.Annotation; namespace Azaion.Common.Controls; @@ -94,7 +94,8 @@ public class DetectionControl : Border }; _detectionLabelPanel = new DetectionLabelPanel { - Confidence = canvasLabel.Confidence + Confidence = canvasLabel.Confidence, + DetectionClass = Annotation.DetectionClassesDict[canvasLabel.ClassNumber] }; DetectionLabelContainer.Children.Add(_detectionLabelPanel); diff --git a/Azaion.Common/Controls/DetectionLabelPanel.xaml b/Azaion.Common/Controls/DetectionLabelPanel.xaml index 629bac4..a05cd4a 100644 --- a/Azaion.Common/Controls/DetectionLabelPanel.xaml +++ b/Azaion.Common/Controls/DetectionLabelPanel.xaml @@ -47,13 +47,13 @@ - + - + - + \ No newline at end of file diff --git a/Azaion.Common/Controls/DetectionLabelPanel.xaml.cs b/Azaion.Common/Controls/DetectionLabelPanel.xaml.cs index cb596fb..0be8679 100644 --- a/Azaion.Common/Controls/DetectionLabelPanel.xaml.cs +++ b/Azaion.Common/Controls/DetectionLabelPanel.xaml.cs @@ -1,12 +1,12 @@ using System.Windows.Media; using Azaion.Common.DTO; +using Azaion.Common.Extensions; namespace Azaion.Common.Controls { public partial class DetectionLabelPanel { private AffiliationEnum _affiliation = AffiliationEnum.None; - private double _confidence; public AffiliationEnum Affiliation { @@ -18,18 +18,33 @@ namespace Azaion.Common.Controls } } - public DetectionClass DetectionClass { get; set; } + private DetectionClass _detectionClass = new(); + public DetectionClass DetectionClass { + get => _detectionClass; + set + { + _detectionClass = value; + SetClassName(); + } + } + private double _confidence; public double Confidence { get => _confidence; set { _confidence = value; - + SetClassName(); } } + private void SetClassName() + { + DetectionClassName.Content = _confidence >= 0.995 ? _detectionClass.UIName : $"{_detectionClass.UIName}: {_confidence * 100:F0}%"; + DetectionGrid.Background = new SolidColorBrush(_detectionClass.Color.ToConfidenceColor(_confidence)); + } + public DetectionLabelPanel() { InitializeComponent(); diff --git a/Azaion.Common/DTO/Label.cs b/Azaion.Common/DTO/Label.cs index 0a6130b..4e486c5 100644 --- a/Azaion.Common/DTO/Label.cs +++ b/Azaion.Common/DTO/Label.cs @@ -225,6 +225,7 @@ public class Detection : YoloLabel [JsonProperty(PropertyName = "an")][Key("an")] public string AnnotationName { get; set; } = null!; [JsonProperty(PropertyName = "p")][Key("p")] public double Confidence { get; set; } [JsonProperty(PropertyName = "dn")][Key("dn")] public string Description { get; set; } + [JsonProperty(PropertyName = "af")][Key("af")] public AffiliationEnum Affiliation { get; set; } //For db & serialization public Detection(){} diff --git a/Azaion.Common/Database/Annotation.cs b/Azaion.Common/Database/Annotation.cs index be7970e..9927d1c 100644 --- a/Azaion.Common/Database/Annotation.cs +++ b/Azaion.Common/Database/Annotation.cs @@ -1,7 +1,6 @@ using System.IO; using System.Windows.Media; using Azaion.Common.DTO; -using Azaion.Common.DTO.Config; using Azaion.Common.DTO.Queue; using MessagePack; @@ -13,14 +12,14 @@ public class Annotation private static string _labelsDir = null!; private static string _imagesDir = null!; private static string _thumbDir = null!; - private static Dictionary _detectionClassesDict; + public static Dictionary DetectionClassesDict = null!; public static void Init(DirectoriesConfig config, Dictionary detectionClassesDict) { _labelsDir = config.LabelsDirectory; _imagesDir = config.ImagesDirectory; _thumbDir = config.ThumbnailsDirectory; - _detectionClassesDict = detectionClassesDict; + DetectionClassesDict = detectionClassesDict; } [Key("n")] public string Name { get; set; } = null!; @@ -76,7 +75,7 @@ public class Annotation private List<(Color Color, double Confidence)>? _colors; [IgnoreMember] public List<(Color Color, double Confidence)> Colors => _colors ??= Detections - .Select(d => (_detectionClassesDict[d.ClassNumber].Color, d.Confidence)) + .Select(d => (DetectionClassesDict[d.ClassNumber].Color, d.Confidence)) .ToList(); private string _className; @@ -88,8 +87,8 @@ public class Annotation { var detectionClasses = Detections.Select(x => x.ClassNumber).Distinct().ToList(); _className = detectionClasses.Count > 1 - ? string.Join(", ", detectionClasses.Select(x => _detectionClassesDict[x].UIName)) - : _detectionClassesDict[detectionClasses.FirstOrDefault()].UIName; + ? string.Join(", ", detectionClasses.Select(x => DetectionClassesDict[x].UIName)) + : DetectionClassesDict[detectionClasses.FirstOrDefault()].UIName; } return _className; } diff --git a/Azaion.Common/Database/SchemaMigrator.cs b/Azaion.Common/Database/SchemaMigrator.cs index 1b34996..eb043fb 100644 --- a/Azaion.Common/Database/SchemaMigrator.cs +++ b/Azaion.Common/Database/SchemaMigrator.cs @@ -85,6 +85,9 @@ public static class SchemaMigrator if (underlyingType == typeof(bool)) return $"NOT NULL DEFAULT {(Convert.ToBoolean(defaultValue) ? 1 : 0)}"; + + if (underlyingType.IsEnum) + return $"NOT NULL DEFAULT {(int)defaultValue}"; if (underlyingType.IsValueType && defaultValue is IFormattable f) return $"NOT NULL DEFAULT {f.ToString(null, System.Globalization.CultureInfo.InvariantCulture)}"; diff --git a/Azaion.Common/Extensions/EnumExtensions.cs b/Azaion.Common/Extensions/EnumExtensions.cs new file mode 100644 index 0000000..9b8b32f --- /dev/null +++ b/Azaion.Common/Extensions/EnumExtensions.cs @@ -0,0 +1,12 @@ +namespace Azaion.Common.Extensions; + +public static class EnumExtensions +{ + public static T GetValueOrDefault(this string value, T defaultValue) where T : struct + { + if (string.IsNullOrEmpty(value)) + return defaultValue; + + return Enum.TryParse(value, true, out T result) ? result : defaultValue; + } +} \ No newline at end of file From 61c93e9c88a82d717b751856957f6adfa83caf0a Mon Sep 17 00:00:00 2001 From: Oleksandr Bezdieniezhnykh Date: Thu, 14 Aug 2025 04:22:55 +0300 Subject: [PATCH 10/14] clamp detections to media borders - create, move, resize fix inference start fix config fix resize rectangles show --- Azaion.Common/Controls/CanvasEditor.cs | 135 +++++++++++++++------ Azaion.Common/Controls/DetectionControl.cs | 2 +- Azaion.Common/Services/InferenceClient.cs | 2 +- Azaion.Suite/config.json | 8 +- 4 files changed, 102 insertions(+), 45 deletions(-) diff --git a/Azaion.Common/Controls/CanvasEditor.cs b/Azaion.Common/Controls/CanvasEditor.cs index ff4969e..7e99166 100644 --- a/Azaion.Common/Controls/CanvasEditor.cs +++ b/Azaion.Common/Controls/CanvasEditor.cs @@ -7,7 +7,6 @@ using System.Windows.Media.Imaging; using System.Windows.Shapes; using Azaion.Common.Database; using Azaion.Common.DTO; -using Azaion.Common.Events; using MediatR; using Color = System.Windows.Media.Color; using Image = System.Windows.Controls.Image; @@ -40,7 +39,8 @@ public class CanvasEditor : Canvas private readonly TimeSpan _viewThreshold = TimeSpan.FromMilliseconds(400); public Image BackgroundImage { get; set; } = new() { Stretch = Stretch.Uniform }; - + private RectangleF? _clampedRect; + public static readonly DependencyProperty GetTimeFuncProp = DependencyProperty.Register( nameof(GetTimeFunc), @@ -129,6 +129,7 @@ public class CanvasEditor : Canvas { SetZoom(); BackgroundImage.Source = source; + UpdateClampedRect(); } private void SetZoom(Matrix? matrix = null) @@ -190,7 +191,7 @@ public class CanvasEditor : Canvas private void CanvasMouseMove(object sender, MouseEventArgs e) { - var pos = e.GetPosition(this); + var pos = GetClampedPosition(e); _horizontalLine.Y1 = _horizontalLine.Y2 = pos.Y; _verticalLine.X1 = _verticalLine.X2 = pos.X; SetLeft(_classNameHint, pos.X + 10); @@ -199,24 +200,36 @@ public class CanvasEditor : Canvas switch (SelectionState) { case SelectionState.NewAnnCreating: - NewAnnotationCreatingMove(sender, e); + NewAnnotationCreatingMove(pos); break; case SelectionState.AnnResizing: - AnnotationResizeMove(sender, e); + AnnotationResizeMove(pos); break; case SelectionState.AnnMoving: - AnnotationPositionMove(sender, e); + AnnotationPositionMove(pos); + e.Handled = true; break; case SelectionState.PanZoomMoving: - PanZoomMove(sender, e); + PanZoomMove(pos); break; } } - private void PanZoomMove(object sender, MouseEventArgs e) + private Point GetClampedPosition(MouseEventArgs e) { - var currentPoint = e.GetPosition(this); - var delta = currentPoint - _panStartPoint; + var pos = e.GetPosition(this); + return !_clampedRect.HasValue + ? pos + : new Point + ( + Math.Clamp(pos.X, _clampedRect.Value.X, _clampedRect.Value.Right), + Math.Clamp(pos.Y, _clampedRect.Value.Y, _clampedRect.Value.Bottom) + ); + } + + private void PanZoomMove(Point point) + { + var delta = point - _panStartPoint; var matrix = _matrixTransform.Matrix; matrix.Translate(delta.X, delta.Y); @@ -229,7 +242,7 @@ public class CanvasEditor : Canvas (sender as UIElement)?.ReleaseMouseCapture(); if (SelectionState == SelectionState.NewAnnCreating) { - var endPos = e.GetPosition(this); + var endPos = GetClampedPosition(e); _newAnnotationRect.Width = 0; _newAnnotationRect.Height = 0; var width = Math.Abs(endPos.X - _newAnnotationStartPos.X); @@ -262,14 +275,14 @@ public class CanvasEditor : Canvas var origin = lb.TranslatePoint(new Point(0, 0), this); lb.Children[0].Measure(new Size(double.PositiveInfinity, double.PositiveInfinity)); var size = lb.Children[0].DesiredSize; - var lbRect = new RectangleF((float)origin.X, (float)origin.Y, (float)size.Width, (float)size.Height); + var controlLabel = new RectangleF((float)origin.X, (float)origin.Y, (float)size.Width, (float)size.Height); foreach (var c in CurrentDetections) { if (c == detectionControl) continue; var detRect = new RectangleF((float)GetLeft(c), (float)GetTop(c), (float)c.Width, (float)c.Height); - detRect.Intersect(lbRect); + detRect.Intersect(controlLabel); // var intersect = detections[i].ToRectangle(); @@ -287,8 +300,44 @@ public class CanvasEditor : Canvas _verticalLine.Y2 = e.NewSize.Height; BackgroundImage.Width = e.NewSize.Width; BackgroundImage.Height = e.NewSize.Height; + UpdateClampedRect(); } - + + private void UpdateClampedRect() + { + if (BackgroundImage.Source is not BitmapSource imageSource) + { + _clampedRect = null; + return; + } + + var imgWidth = imageSource.PixelWidth; + var imgHeight = imageSource.PixelHeight; + var canvasWidth = ActualWidth; + var canvasHeight = ActualHeight; + + var imgRatio = imgWidth / (double)imgHeight; + var canvasRatio = canvasWidth / canvasHeight; + + double renderedWidth; + double renderedHeight; + + if (imgRatio > canvasRatio) + { + renderedWidth = canvasWidth; + renderedHeight = canvasWidth / imgRatio; + } + else + { + renderedHeight = canvasHeight; + renderedWidth = canvasHeight * imgRatio; + } + var xOffset = (canvasWidth - renderedWidth) / 2; + var yOffset = (canvasHeight - renderedHeight) / 2; + + _clampedRect = new RectangleF((float)xOffset, (float)yOffset, (float)renderedWidth, (float)renderedHeight); + } + #region Annotation Resizing & Moving private void AnnotationResizeStart(object sender, MouseEventArgs e) @@ -300,17 +349,15 @@ public class CanvasEditor : Canvas e.Handled = true; } - private void AnnotationResizeMove(object sender, MouseEventArgs e) + private void AnnotationResizeMove(Point point) { if (SelectionState != SelectionState.AnnResizing) return; - var currentPos = e.GetPosition(this); - var x = GetLeft(_curAnn); var y = GetTop(_curAnn); - var offsetX = currentPos.X - _lastPos.X; - var offsetY = currentPos.Y - _lastPos.Y; + var offsetX = point.X - _lastPos.X; + var offsetY = point.Y - _lastPos.Y; switch (_curRec.HorizontalAlignment, _curRec.VerticalAlignment) { case (HorizontalAlignment.Left, VerticalAlignment.Top): @@ -350,7 +397,7 @@ public class CanvasEditor : Canvas _curAnn.Height = Math.Max(MIN_SIZE, _curAnn.Height + offsetY); break; } - _lastPos = currentPos; + _lastPos = point; } private void AnnotationPositionStart(object sender, MouseEventArgs e) @@ -367,19 +414,26 @@ public class CanvasEditor : Canvas e.Handled = true; } - private void AnnotationPositionMove(object sender, MouseEventArgs e) + private void AnnotationPositionMove(Point point) { if (SelectionState != SelectionState.AnnMoving) return; - var currentPos = e.GetPosition(this); - var offsetX = currentPos.X - _lastPos.X; - var offsetY = currentPos.Y - _lastPos.Y; - - SetLeft(_curAnn, GetLeft(_curAnn) + offsetX); - SetTop(_curAnn, GetTop(_curAnn) + offsetY); - _lastPos = currentPos; - e.Handled = true; + var offsetX = point.X - _lastPos.X; + var offsetY = point.Y - _lastPos.Y; + + var nextLeft = GetLeft(_curAnn) + offsetX; + var nextTop = GetTop(_curAnn) + offsetY; + + if (_clampedRect.HasValue) + { + nextLeft = Math.Clamp(nextLeft, _clampedRect.Value.X, _clampedRect.Value.Right - _curAnn.Width); + nextTop = Math.Clamp(nextTop, _clampedRect.Value.Y, _clampedRect.Value.Bottom - _curAnn.Height); + } + + SetLeft(_curAnn, nextLeft); + SetTop(_curAnn, nextTop); + _lastPos = point; } #endregion @@ -391,26 +445,29 @@ public class CanvasEditor : Canvas _newAnnotationStartPos = e.GetPosition(this); SetLeft(_newAnnotationRect, _newAnnotationStartPos.X); SetTop(_newAnnotationRect, _newAnnotationStartPos.Y); - _newAnnotationRect.MouseMove += NewAnnotationCreatingMove; + _newAnnotationRect.MouseMove += (sender, e) => + { + var currentPos = e.GetPosition(this); + NewAnnotationCreatingMove(currentPos); + }; SelectionState = SelectionState.NewAnnCreating; } - private void NewAnnotationCreatingMove(object sender, MouseEventArgs e) + private void NewAnnotationCreatingMove(Point point) { if (SelectionState != SelectionState.NewAnnCreating) return; - var currentPos = e.GetPosition(this); - var diff = currentPos - _newAnnotationStartPos; + var diff = point - _newAnnotationStartPos; _newAnnotationRect.Height = Math.Abs(diff.Y); _newAnnotationRect.Width = Math.Abs(diff.X); if (diff.X < 0) - SetLeft(_newAnnotationRect, currentPos.X); + SetLeft(_newAnnotationRect, point.X); if (diff.Y < 0) - SetTop(_newAnnotationRect, currentPos.Y); + SetTop(_newAnnotationRect, point.Y); } public void CreateDetections(Annotation annotation, List detectionClasses, Size mediaSize) @@ -432,7 +489,9 @@ public class CanvasEditor : Canvas canvasLabel = new CanvasLabel(yoloLabel, RenderSize, mediaSize, canvasLabel.Confidence); } - CreateDetectionControl(detectionClass, annotation.Time, canvasLabel); + var control = CreateDetectionControl(detectionClass, annotation.Time, canvasLabel); + control.UpdateLayout(); + CheckLabelBoundaries(control); } } @@ -481,9 +540,7 @@ public class CanvasEditor : Canvas .ToList(); RemoveAnnotations(expiredAnns); } - - public void ResetBackground() => Background = new SolidColorBrush(Color.FromArgb(1, 0, 0, 0)); - + public void ZoomTo(Point point) { SetZoom(); diff --git a/Azaion.Common/Controls/DetectionControl.cs b/Azaion.Common/Controls/DetectionControl.cs index e3bf6a1..bee6a8b 100644 --- a/Azaion.Common/Controls/DetectionControl.cs +++ b/Azaion.Common/Controls/DetectionControl.cs @@ -127,9 +127,9 @@ public class DetectionControl : Border VerticalAlignment = VerticalAlignment.Stretch, Children = { _selectionFrame } }; + _grid.Children.Add(DetectionLabelContainer); foreach (var rect in _resizedRectangles) _grid.Children.Add(rect); - _grid.Children.Add(DetectionLabelContainer); Child = _grid; Cursor = Cursors.SizeAll; diff --git a/Azaion.Common/Services/InferenceClient.cs b/Azaion.Common/Services/InferenceClient.cs index 714311f..7e59620 100644 --- a/Azaion.Common/Services/InferenceClient.cs +++ b/Azaion.Common/Services/InferenceClient.cs @@ -49,7 +49,7 @@ public class InferenceClient : IInferenceClient Arguments = $"-p {_inferenceClientConfig.ZeroMqPort} -lp {_loaderClientConfig.ZeroMqPort} -a {_inferenceClientConfig.ApiUrl}", CreateNoWindow = true }; - //process.Start(); + process.Start(); } catch (Exception e) { diff --git a/Azaion.Suite/config.json b/Azaion.Suite/config.json index c5595a5..748dbb9 100644 --- a/Azaion.Suite/config.json +++ b/Azaion.Suite/config.json @@ -17,10 +17,10 @@ "DirectoriesConfig": { "ApiResourcesDirectory": "stage", "VideosDirectory": "E:\\Azaion6", - "LabelsDirectory": "E:\\labels_test", - "ImagesDirectory": "E:\\images_test", - "ResultsDirectory": "E:\\results_test", - "ThumbnailsDirectory": "E:\\thumbnails_test", + "LabelsDirectory": "E:\\labels", + "ImagesDirectory": "E:\\images", + "ResultsDirectory": "E:\\results", + "ThumbnailsDirectory": "E:\\thumbnails", "GpsSatDirectory": "satellitesDir", "GpsRouteDirectory": "routeDir" }, From 55d8a5cb85e12b94fee8e11fd6bfc6c65fc03a70 Mon Sep 17 00:00:00 2001 From: Oleksandr Bezdieniezhnykh Date: Thu, 14 Aug 2025 04:43:08 +0300 Subject: [PATCH 11/14] small improvements --- Azaion.Common/Controls/CanvasEditor.cs | 1 + Azaion.Common/Controls/DetectionControl.cs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Azaion.Common/Controls/CanvasEditor.cs b/Azaion.Common/Controls/CanvasEditor.cs index 7e99166..4529f38 100644 --- a/Azaion.Common/Controls/CanvasEditor.cs +++ b/Azaion.Common/Controls/CanvasEditor.cs @@ -346,6 +346,7 @@ public class CanvasEditor : Canvas _lastPos = e.GetPosition(this); _curRec = (Rectangle)sender; _curAnn = (DetectionControl)((Grid)_curRec.Parent).Parent; + (sender as UIElement)?.CaptureMouse(); e.Handled = true; } diff --git a/Azaion.Common/Controls/DetectionControl.cs b/Azaion.Common/Controls/DetectionControl.cs index bee6a8b..93e92ca 100644 --- a/Azaion.Common/Controls/DetectionControl.cs +++ b/Azaion.Common/Controls/DetectionControl.cs @@ -16,7 +16,6 @@ public class DetectionControl : Border private readonly Grid _grid; private readonly DetectionLabelPanel _detectionLabelPanel; - //private readonly Label _detectionLabel; public readonly Canvas DetectionLabelContainer; public TimeSpan Time { get; set; } @@ -154,6 +153,7 @@ public class DetectionControl : Border Name = name, }; rect.MouseDown += (sender, args) => _resizeStart(sender, args); + rect.MouseUp += (sender, args) => { (sender as UIElement)?.ReleaseMouseCapture(); }; return rect; } From eb9e2a6f47a0d9a8f7489d60f6e2d84a834e456d Mon Sep 17 00:00:00 2001 From: Oleksandr Bezdieniezhnykh Date: Thu, 14 Aug 2025 10:49:36 +0300 Subject: [PATCH 12/14] don't update loaderconfig.json on each update --- Azaion.LoaderUI/Azaion.LoaderUI.csproj | 5 ++++- .../{loaderconfig.json => loaderconfig.prod.json} | 0 Azaion.LoaderUI/loaderconfig.stage.json | 6 ++++++ 3 files changed, 10 insertions(+), 1 deletion(-) rename Azaion.LoaderUI/{loaderconfig.json => loaderconfig.prod.json} (100%) create mode 100644 Azaion.LoaderUI/loaderconfig.stage.json diff --git a/Azaion.LoaderUI/Azaion.LoaderUI.csproj b/Azaion.LoaderUI/Azaion.LoaderUI.csproj index a61de0e..b130b06 100644 --- a/Azaion.LoaderUI/Azaion.LoaderUI.csproj +++ b/Azaion.LoaderUI/Azaion.LoaderUI.csproj @@ -33,13 +33,16 @@ - + PreserveNewest PreserveNewest + + PreserveNewest + diff --git a/Azaion.LoaderUI/loaderconfig.json b/Azaion.LoaderUI/loaderconfig.prod.json similarity index 100% rename from Azaion.LoaderUI/loaderconfig.json rename to Azaion.LoaderUI/loaderconfig.prod.json diff --git a/Azaion.LoaderUI/loaderconfig.stage.json b/Azaion.LoaderUI/loaderconfig.stage.json new file mode 100644 index 0000000..01edb38 --- /dev/null +++ b/Azaion.LoaderUI/loaderconfig.stage.json @@ -0,0 +1,6 @@ +{ + "DirectoriesConfig": + { + "SuiteInstallerDirectory": "suite-stage" + } +} \ No newline at end of file From d1ce9d9365a528c89544ce6d74a0d27ef1569f16 Mon Sep 17 00:00:00 2001 From: Oleksandr Bezdieniezhnykh Date: Thu, 14 Aug 2025 12:54:32 +0300 Subject: [PATCH 13/14] fix editing tiled images --- Azaion.Annotator/AnnotatorEventHandler.cs | 20 ++++---------------- Azaion.Common/Controls/CanvasEditor.cs | 6 +++--- Azaion.Common/Extensions/BitmapExtensions.cs | 10 ++++++++++ Azaion.Common/Extensions/SizeExtensions.cs | 10 ++++++++++ Azaion.Dataset/DatasetExplorer.xaml | 5 +++-- Azaion.Dataset/DatasetExplorer.xaml.cs | 5 +---- 6 files changed, 31 insertions(+), 25 deletions(-) create mode 100644 Azaion.Common/Extensions/SizeExtensions.cs diff --git a/Azaion.Annotator/AnnotatorEventHandler.cs b/Azaion.Annotator/AnnotatorEventHandler.cs index ee52a7e..f345f6f 100644 --- a/Azaion.Annotator/AnnotatorEventHandler.cs +++ b/Azaion.Annotator/AnnotatorEventHandler.cs @@ -307,15 +307,8 @@ public class AnnotatorEventHandler( if (!File.Exists(imgPath)) { var source = (mainWindow.Editor.BackgroundImage.Source as BitmapSource)!; - if (source.PixelWidth <= Constants.AI_TILE_SIZE * 2 && source.PixelHeight <= Constants.AI_TILE_SIZE * 2) // Allow to be up to 2560*2560 to save to 1280*1280 - { - //Save image - await using var stream = new FileStream(imgPath, FileMode.Create); - var encoder = new JpegBitmapEncoder(); - encoder.Frames.Add(BitmapFrame.Create(source)); - encoder.Save(stream); - await stream.FlushAsync(cancellationToken); - } + if (new Size(source.PixelWidth, source.PixelHeight).FitSizeForAI()) + await source.SaveImage(imgPath, cancellationToken); else { //Tiling @@ -335,14 +328,9 @@ public class AnnotatorEventHandler( var annotationName = $"{formState.MediaName}{Constants.SPLIT_SUFFIX}{res.Tile.Left:0000}_{res.Tile.Top:0000}!".ToTimeName(time); var tileImgPath = Path.Combine(dirConfig.Value.ImagesDirectory, $"{annotationName}{Constants.JPG_EXT}"); - await using var tileStream = new FileStream(tileImgPath, FileMode.Create); var bitmap = new CroppedBitmap(source, new Int32Rect((int)res.Tile.Left, (int)res.Tile.Top, (int)res.Tile.Width, (int)res.Tile.Height)); - - var tileEncoder = new JpegBitmapEncoder { Frames = [BitmapFrame.Create(bitmap)] }; - tileEncoder.Save(tileStream); - await tileStream.FlushAsync(cancellationToken); - tileStream.Close(); - + await bitmap.SaveImage(tileImgPath, cancellationToken); + var frameSize = new Size(res.Tile.Width, res.Tile.Height); var detections = res.Detections .Select(det => det.ReframeToSmall(res.Tile)) diff --git a/Azaion.Common/Controls/CanvasEditor.cs b/Azaion.Common/Controls/CanvasEditor.cs index 4529f38..2a2f11b 100644 --- a/Azaion.Common/Controls/CanvasEditor.cs +++ b/Azaion.Common/Controls/CanvasEditor.cs @@ -7,6 +7,7 @@ using System.Windows.Media.Imaging; using System.Windows.Shapes; using Azaion.Common.Database; using Azaion.Common.DTO; +using Azaion.Common.Extensions; using MediatR; using Color = System.Windows.Media.Color; using Image = System.Windows.Controls.Image; @@ -473,17 +474,16 @@ public class CanvasEditor : Canvas public void CreateDetections(Annotation annotation, List detectionClasses, Size mediaSize) { - var splitTile = annotation.SplitTile; foreach (var detection in annotation.Detections) { var detectionClass = DetectionClass.FromYoloId(detection.ClassNumber, detectionClasses); CanvasLabel canvasLabel; - if (splitTile == null) + if (!annotation.IsSplit || mediaSize.FitSizeForAI()) canvasLabel = new CanvasLabel(detection, RenderSize, mediaSize, detection.Confidence); else { canvasLabel = new CanvasLabel(detection, new Size(Constants.AI_TILE_SIZE, Constants.AI_TILE_SIZE), null, detection.Confidence) - .ReframeFromSmall(splitTile); + .ReframeFromSmall(annotation.SplitTile!); //From CurrentMediaSize to Render Size var yoloLabel = new YoloLabel(canvasLabel, mediaSize); diff --git a/Azaion.Common/Extensions/BitmapExtensions.cs b/Azaion.Common/Extensions/BitmapExtensions.cs index 294a8bb..de23af7 100644 --- a/Azaion.Common/Extensions/BitmapExtensions.cs +++ b/Azaion.Common/Extensions/BitmapExtensions.cs @@ -26,4 +26,14 @@ public static class BitmapExtensions public static Color CreateTransparent(this Color color, byte transparency) => Color.FromArgb(transparency, color.R, color.G, color.B); + + public static async Task SaveImage(this BitmapSource bitmap, string path, CancellationToken ct = default) + { + await using var stream = new FileStream(path, FileMode.Create); + var encoder = new JpegBitmapEncoder(); + + encoder.Frames.Add(BitmapFrame.Create(bitmap)); + encoder.Save(stream); + await stream.FlushAsync(ct); + } } \ No newline at end of file diff --git a/Azaion.Common/Extensions/SizeExtensions.cs b/Azaion.Common/Extensions/SizeExtensions.cs new file mode 100644 index 0000000..c8e62b4 --- /dev/null +++ b/Azaion.Common/Extensions/SizeExtensions.cs @@ -0,0 +1,10 @@ +using System.Windows; + +namespace Azaion.Common.Extensions; + +public static class SizeExtensions +{ + public static bool FitSizeForAI(this Size size) => + // Allow to be up to FullHD to save as 1280*1280 + size.Width <= Constants.AI_TILE_SIZE * 1.5 && size.Height <= Constants.AI_TILE_SIZE * 1.5; +} \ No newline at end of file diff --git a/Azaion.Dataset/DatasetExplorer.xaml b/Azaion.Dataset/DatasetExplorer.xaml index 2791540..5316645 100644 --- a/Azaion.Dataset/DatasetExplorer.xaml +++ b/Azaion.Dataset/DatasetExplorer.xaml @@ -138,8 +138,9 @@ Header="Редактор" Visibility="Collapsed"> + Background="#01000000" + VerticalAlignment="Stretch" + HorizontalAlignment="Stretch" > diff --git a/Azaion.Dataset/DatasetExplorer.xaml.cs b/Azaion.Dataset/DatasetExplorer.xaml.cs index ed687b5..bf6aabb 100644 --- a/Azaion.Dataset/DatasetExplorer.xaml.cs +++ b/Azaion.Dataset/DatasetExplorer.xaml.cs @@ -195,10 +195,7 @@ public partial class DatasetExplorer ThumbnailsView.SelectedIndex = index; var ann = CurrentAnnotation.Annotation; - ExplorerEditor.Background = new ImageBrush - { - ImageSource = await ann.ImagePath.OpenImage() - }; + ExplorerEditor.SetBackground(await ann.ImagePath.OpenImage()); SwitchTab(toEditor: true); ExplorerEditor.RemoveAllAnns(); From 067f02cc63254dc43118585b1ecece57de73757d Mon Sep 17 00:00:00 2001 From: Oleksandr Bezdieniezhnykh Date: Mon, 1 Sep 2025 20:12:13 +0300 Subject: [PATCH 14/14] update AI initializing rework AIAvailabilityStatus events to mediatr --- Azaion.Annotator/Annotator.xaml.cs | 33 +----- Azaion.Annotator/AnnotatorEventHandler.cs | 14 ++- Azaion.Common/DTO/AIAvailabilityStatus.cs | 33 ++++++ Azaion.Common/Services/AnnotationService.cs | 1 + .../GPSMatcherEventHandler.cs | 0 .../{ => GpsMatcher}/GPSMatcherEvents.cs | 0 .../{ => GpsMatcher}/GPSMatcherService.cs | 0 .../{ => GpsMatcher}/GpsMatcherClient.cs | 0 .../{ => Inference}/InferenceClient.cs | 31 +++--- .../Services/Inference/InferenceService.cs | 56 ++++++++++ .../Inference/InferenceServiceEventHandler.cs | 43 ++++++++ .../Inference/InferenceServiceEvents.cs | 9 ++ Azaion.Common/Services/InferenceService.cs | 82 -------------- Azaion.Inference/ai_availability_status.pxd | 14 +++ Azaion.Inference/ai_availability_status.pyx | 36 +++++++ Azaion.Inference/build_inference.cmd | 4 +- Azaion.Inference/inference.pxd | 4 +- Azaion.Inference/inference.pyx | 102 +++++++++--------- Azaion.Inference/main_inference.pyx | 4 +- Azaion.Inference/setup.py | 1 + Azaion.Suite/App.xaml.cs | 1 + Azaion.Suite/MainSuite.xaml.cs | 2 +- Azaion.Suite/config.json | 4 +- 23 files changed, 282 insertions(+), 192 deletions(-) create mode 100644 Azaion.Common/DTO/AIAvailabilityStatus.cs rename Azaion.Common/Services/{ => GpsMatcher}/GPSMatcherEventHandler.cs (100%) rename Azaion.Common/Services/{ => GpsMatcher}/GPSMatcherEvents.cs (100%) rename Azaion.Common/Services/{ => GpsMatcher}/GPSMatcherService.cs (100%) rename Azaion.Common/Services/{ => GpsMatcher}/GpsMatcherClient.cs (100%) rename Azaion.Common/Services/{ => Inference}/InferenceClient.cs (75%) create mode 100644 Azaion.Common/Services/Inference/InferenceService.cs create mode 100644 Azaion.Common/Services/Inference/InferenceServiceEventHandler.cs create mode 100644 Azaion.Common/Services/Inference/InferenceServiceEvents.cs delete mode 100644 Azaion.Common/Services/InferenceService.cs create mode 100644 Azaion.Inference/ai_availability_status.pxd create mode 100644 Azaion.Inference/ai_availability_status.pyx diff --git a/Azaion.Annotator/Annotator.xaml.cs b/Azaion.Annotator/Annotator.xaml.cs index 9feeba4..9e4c0c9 100644 --- a/Azaion.Annotator/Annotator.xaml.cs +++ b/Azaion.Annotator/Annotator.xaml.cs @@ -14,6 +14,7 @@ using Azaion.Common.DTO.Config; using Azaion.Common.Events; using Azaion.Common.Extensions; using Azaion.Common.Services; +using Azaion.Common.Services.Inference; using LibVLCSharp.Shared; using MediatR; using Microsoft.WindowsAPICodePack.Dialogs; @@ -106,38 +107,6 @@ public partial class Annotator _logger.LogError(e, e.Message); } }; - _inferenceClient.AIAvailabilityReceived += (_, command) => - { - Dispatcher.Invoke(() => - { - _logger.LogInformation(command.Message); - var aiEnabled = command.Message == "enabled"; - AIDetectBtn.IsEnabled = aiEnabled; - var aiDisabledText = "Будь ласка, зачекайте, наразі розпізнавання AI недоступне"; - var messagesDict = new Dictionary - { - { "disabled", aiDisabledText }, - { "downloading", "Будь ласка зачекайте, йде завантаження AI для Вашої відеокарти" }, - { "converting", "Будь ласка зачекайте, йде налаштування AI під Ваше залізо. (5-12 хвилин в залежності від моделі відеокарти, до 50 хв на старих GTX1650)" }, - { "uploading", "Будь ласка зачекайте, йде зберігання" }, - { "enabled", "AI готовий для розпізнавання" } - }; - - if (command.Message?.StartsWith("Error") ?? false) - { - _logger.LogError(command.Message); - StatusHelp.Text = command.Message; - } - - else - StatusHelp.Text = messagesDict!.GetValueOrDefault(command.Message, aiDisabledText); - - if (aiEnabled) - StatusHelp.Foreground = aiEnabled ? Brushes.White : Brushes.Red; - }); - }; - _inferenceClient.Send(RemoteCommand.Create(CommandType.AIAvailabilityCheck)); - Editor.GetTimeFunc = () => TimeSpan.FromMilliseconds(_mediaPlayer.Time); MapMatcherComponent.Init(_appConfig, gpsMatcherService); } diff --git a/Azaion.Annotator/AnnotatorEventHandler.cs b/Azaion.Annotator/AnnotatorEventHandler.cs index f345f6f..5a0c4ec 100644 --- a/Azaion.Annotator/AnnotatorEventHandler.cs +++ b/Azaion.Annotator/AnnotatorEventHandler.cs @@ -12,6 +12,7 @@ using Azaion.Common.DTO.Config; using Azaion.Common.Events; using Azaion.Common.Extensions; using Azaion.Common.Services; +using Azaion.Common.Services.Inference; using GMap.NET; using GMap.NET.WindowsPresentation; using LibVLCSharp.Shared; @@ -43,7 +44,8 @@ public class AnnotatorEventHandler( INotificationHandler, INotificationHandler, INotificationHandler, - INotificationHandler + INotificationHandler, + INotificationHandler { private const int STEP = 20; private const int LARGE_STEP = 5000; @@ -472,4 +474,14 @@ public class AnnotatorEventHandler( map.SatelliteMap.Position = pointLatLon; map.SatelliteMap.ZoomAndCenterMarkers(null); } + + public async Task Handle(AIAvailabilityStatusEvent e, CancellationToken cancellationToken) + { + mainWindow.Dispatcher.Invoke(() => + { + logger.LogInformation(e.ToString()); + mainWindow.AIDetectBtn.IsEnabled = e.Status == AIAvailabilityEnum.Enabled; + mainWindow.StatusHelp.Text = e.ToString(); + }); + } } \ No newline at end of file diff --git a/Azaion.Common/DTO/AIAvailabilityStatus.cs b/Azaion.Common/DTO/AIAvailabilityStatus.cs new file mode 100644 index 0000000..b45f865 --- /dev/null +++ b/Azaion.Common/DTO/AIAvailabilityStatus.cs @@ -0,0 +1,33 @@ +using MediatR; +using MessagePack; + +namespace Azaion.Common.DTO; + +public enum AIAvailabilityEnum +{ + None = 0, + Downloading = 10, + Converting = 20, + Uploading = 30, + Enabled = 200, + Error = 500 +} + +[MessagePackObject] +public class AIAvailabilityStatusEvent : INotification +{ + [Key("s")] public AIAvailabilityEnum Status { get; set; } + [Key("m")] public string? ErrorMessage { get; set; } + + public override string ToString() => $"{StatusMessageDict.GetValueOrDefault(Status, "Помилка")} {ErrorMessage}"; + + private static readonly Dictionary StatusMessageDict = new() + { + { AIAvailabilityEnum.Downloading, "Йде завантаження AI для Вашої відеокарти" }, + { AIAvailabilityEnum.Converting, "Йде налаштування AI під Ваше залізо. (5-12 хвилин в залежності від моделі відеокарти, до 50 хв на старих GTX1650)" }, + { AIAvailabilityEnum.Uploading, "Йде зберігання AI" }, + { AIAvailabilityEnum.Enabled, "AI готовий для розпізнавання" }, + { AIAvailabilityEnum.Error, "Помилка під час налаштування AI" } + }; + +} diff --git a/Azaion.Common/Services/AnnotationService.cs b/Azaion.Common/Services/AnnotationService.cs index 190c396..d0381df 100644 --- a/Azaion.Common/Services/AnnotationService.cs +++ b/Azaion.Common/Services/AnnotationService.cs @@ -21,6 +21,7 @@ using RabbitMQ.Stream.Client.Reliable; namespace Azaion.Common.Services; // SHOULD BE ONLY ONE INSTANCE OF AnnotationService. Do not add ANY NotificationHandler to it! +// Queue consumer should be created only once. public class AnnotationService : IAnnotationService { private readonly IDbFactory _dbFactory; diff --git a/Azaion.Common/Services/GPSMatcherEventHandler.cs b/Azaion.Common/Services/GpsMatcher/GPSMatcherEventHandler.cs similarity index 100% rename from Azaion.Common/Services/GPSMatcherEventHandler.cs rename to Azaion.Common/Services/GpsMatcher/GPSMatcherEventHandler.cs diff --git a/Azaion.Common/Services/GPSMatcherEvents.cs b/Azaion.Common/Services/GpsMatcher/GPSMatcherEvents.cs similarity index 100% rename from Azaion.Common/Services/GPSMatcherEvents.cs rename to Azaion.Common/Services/GpsMatcher/GPSMatcherEvents.cs diff --git a/Azaion.Common/Services/GPSMatcherService.cs b/Azaion.Common/Services/GpsMatcher/GPSMatcherService.cs similarity index 100% rename from Azaion.Common/Services/GPSMatcherService.cs rename to Azaion.Common/Services/GpsMatcher/GPSMatcherService.cs diff --git a/Azaion.Common/Services/GpsMatcherClient.cs b/Azaion.Common/Services/GpsMatcher/GpsMatcherClient.cs similarity index 100% rename from Azaion.Common/Services/GpsMatcherClient.cs rename to Azaion.Common/Services/GpsMatcher/GpsMatcherClient.cs diff --git a/Azaion.Common/Services/InferenceClient.cs b/Azaion.Common/Services/Inference/InferenceClient.cs similarity index 75% rename from Azaion.Common/Services/InferenceClient.cs rename to Azaion.Common/Services/Inference/InferenceClient.cs index 7e59620..2e612b9 100644 --- a/Azaion.Common/Services/InferenceClient.cs +++ b/Azaion.Common/Services/Inference/InferenceClient.cs @@ -1,18 +1,17 @@ using System.Diagnostics; using System.Text; using Azaion.Common.DTO; +using MediatR; using MessagePack; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using NetMQ; using NetMQ.Sockets; -namespace Azaion.Common.Services; +namespace Azaion.Common.Services.Inference; public interface IInferenceClient : IDisposable { - event EventHandler? InferenceDataReceived; - event EventHandler? AIAvailabilityReceived; void Send(RemoteCommand create); void Stop(); } @@ -20,21 +19,22 @@ public interface IInferenceClient : IDisposable public class InferenceClient : IInferenceClient { private readonly ILogger _logger; - public event EventHandler? BytesReceived; - public event EventHandler? InferenceDataReceived; - public event EventHandler? AIAvailabilityReceived; private readonly DealerSocket _dealer = new(); private readonly NetMQPoller _poller = new(); private readonly Guid _clientId = Guid.NewGuid(); private readonly InferenceClientConfig _inferenceClientConfig; private readonly LoaderClientConfig _loaderClientConfig; + private readonly IMediator _mediator; - public InferenceClient(ILogger logger, IOptions inferenceConfig, IOptions loaderConfig) + public InferenceClient(ILogger logger, IOptions inferenceConfig, + IMediator mediator, + IOptions loaderConfig) { _logger = logger; _inferenceClientConfig = inferenceConfig.Value; _loaderClientConfig = loaderConfig.Value; + _mediator = mediator; Start(); } @@ -59,32 +59,31 @@ public class InferenceClient : IInferenceClient _dealer.Options.Identity = Encoding.UTF8.GetBytes(_clientId.ToString("N")); _dealer.Connect($"tcp://{_inferenceClientConfig.ZeroMqHost}:{_inferenceClientConfig.ZeroMqPort}"); - _dealer.ReceiveReady += (_, e) => ProcessClientCommand(e.Socket); + _dealer.ReceiveReady += async (_, e) => await ProcessClientCommand(e.Socket); _poller.Add(_dealer); _ = Task.Run(() => _poller.RunAsync()); } - private void ProcessClientCommand(NetMQSocket socket, CancellationToken ct = default) + private async Task ProcessClientCommand(NetMQSocket socket, CancellationToken ct = default) { while (socket.TryReceiveFrameBytes(TimeSpan.Zero, out var bytes)) { - if (bytes?.Length == 0) + if (bytes.Length == 0) continue; var remoteCommand = MessagePackSerializer.Deserialize(bytes, cancellationToken: ct); switch (remoteCommand.CommandType) { - case CommandType.DataBytes: - BytesReceived?.Invoke(this, remoteCommand); - break; case CommandType.InferenceData: - InferenceDataReceived?.Invoke(this, remoteCommand); + await _mediator.Publish(new InferenceDataEvent(remoteCommand), ct); break; case CommandType.AIAvailabilityResult: - AIAvailabilityReceived?.Invoke(this, remoteCommand); + var aiAvailabilityStatus = MessagePackSerializer.Deserialize(remoteCommand.Data, cancellationToken: ct); + await _mediator.Publish(aiAvailabilityStatus, ct); break; + default: + throw new ArgumentOutOfRangeException(); } - } } diff --git a/Azaion.Common/Services/Inference/InferenceService.cs b/Azaion.Common/Services/Inference/InferenceService.cs new file mode 100644 index 0000000..d94ae4d --- /dev/null +++ b/Azaion.Common/Services/Inference/InferenceService.cs @@ -0,0 +1,56 @@ +using Azaion.Common.DTO; +using Azaion.Common.DTO.Config; +using Azaion.Common.Extensions; +using Microsoft.Extensions.Options; + +namespace Azaion.Common.Services.Inference; + +public interface IInferenceService +{ + Task RunInference(List mediaPaths, CancellationToken ct = default); + CancellationTokenSource InferenceCancelTokenSource { get; set; } + void StopInference(); +} + +// SHOULD BE ONLY ONE INSTANCE OF InferenceService. Do not add ANY NotificationHandler to it! +// _inferenceCancelTokenSource should be created only once. +public class InferenceService : IInferenceService +{ + private readonly IInferenceClient _client; + private readonly IAzaionApi _azaionApi; + private readonly IOptions _aiConfigOptions; + public CancellationTokenSource InferenceCancelTokenSource { get; set; } = new(); + public CancellationTokenSource CheckAIAvailabilityTokenSource { get; set; } = new(); + + public InferenceService(IInferenceClient client, IAzaionApi azaionApi, IOptions aiConfigOptions) + { + _client = client; + _azaionApi = azaionApi; + _aiConfigOptions = aiConfigOptions; + } + + public async Task CheckAIAvailabilityStatus() + { + CheckAIAvailabilityTokenSource = new CancellationTokenSource(); + while (!CheckAIAvailabilityTokenSource.IsCancellationRequested) + { + _client.Send(RemoteCommand.Create(CommandType.AIAvailabilityCheck)); + await Task.Delay(10000, CheckAIAvailabilityTokenSource.Token); + } + } + + public async Task RunInference(List mediaPaths, CancellationToken ct = default) + { + InferenceCancelTokenSource = new CancellationTokenSource(); + _client.Send(RemoteCommand.Create(CommandType.Login, _azaionApi.Credentials)); + + var aiConfig = _aiConfigOptions.Value; + aiConfig.Paths = mediaPaths; + _client.Send(RemoteCommand.Create(CommandType.Inference, aiConfig)); + + using var combinedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(ct, InferenceCancelTokenSource.Token); + await combinedTokenSource.Token.AsTask(); + } + + public void StopInference() => _client.Stop(); +} \ No newline at end of file diff --git a/Azaion.Common/Services/Inference/InferenceServiceEventHandler.cs b/Azaion.Common/Services/Inference/InferenceServiceEventHandler.cs new file mode 100644 index 0000000..092edfb --- /dev/null +++ b/Azaion.Common/Services/Inference/InferenceServiceEventHandler.cs @@ -0,0 +1,43 @@ +using Azaion.Common.Database; +using Azaion.Common.DTO; +using Azaion.Common.Events; +using MediatR; +using MessagePack; +using Microsoft.Extensions.Logging; + +namespace Azaion.Common.Services.Inference; + +public class InferenceServiceEventHandler(IInferenceService inferenceService, + IAnnotationService annotationService, + IMediator mediator, + ILogger logger) : + INotificationHandler, + INotificationHandler +{ + public async Task Handle(InferenceDataEvent e, CancellationToken ct) + { + try + { + if (e.Command.Message == "DONE") + { + await inferenceService.InferenceCancelTokenSource.CancelAsync(); + return; + } + + var annImage = MessagePackSerializer.Deserialize(e.Command.Data, cancellationToken: ct); + var annotation = await annotationService.SaveAnnotation(annImage, ct); + await mediator.Publish(new AnnotationAddedEvent(annotation), ct); + } + catch (Exception ex) + { + logger.LogError(ex, ex.Message); + } + } + + public async Task Handle(AIAvailabilityStatusEvent e, CancellationToken ct) + { + + e.Status = AIAvailabilityEnum.Enabled; + + } +} \ No newline at end of file diff --git a/Azaion.Common/Services/Inference/InferenceServiceEvents.cs b/Azaion.Common/Services/Inference/InferenceServiceEvents.cs new file mode 100644 index 0000000..94aff5c --- /dev/null +++ b/Azaion.Common/Services/Inference/InferenceServiceEvents.cs @@ -0,0 +1,9 @@ +using Azaion.Common.DTO; +using MediatR; + +namespace Azaion.Common.Services.Inference; + +public class InferenceDataEvent(RemoteCommand command) : INotification +{ + public RemoteCommand Command { get; set; } = command; +} diff --git a/Azaion.Common/Services/InferenceService.cs b/Azaion.Common/Services/InferenceService.cs deleted file mode 100644 index 8ae9949..0000000 --- a/Azaion.Common/Services/InferenceService.cs +++ /dev/null @@ -1,82 +0,0 @@ -using Azaion.Common.Database; -using Azaion.Common.DTO; -using Azaion.Common.DTO.Config; -using Azaion.Common.Events; -using Azaion.Common.Extensions; -using MediatR; -using MessagePack; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; - -namespace Azaion.Common.Services; - -public interface IInferenceService -{ - Task RunInference(List mediaPaths, CancellationToken ct = default); - void StopInference(); -} - -public class InferenceService : IInferenceService -{ - private readonly IInferenceClient _client; - private readonly IAzaionApi _azaionApi; - private readonly IOptions _aiConfigOptions; - private readonly IAnnotationService _annotationService; - private readonly IMediator _mediator; - private CancellationTokenSource _inferenceCancelTokenSource = new(); - - public InferenceService( - ILogger logger, - IInferenceClient client, - IAzaionApi azaionApi, - IOptions aiConfigOptions, - IAnnotationService annotationService, - IMediator mediator) - { - _client = client; - _azaionApi = azaionApi; - _aiConfigOptions = aiConfigOptions; - _annotationService = annotationService; - _mediator = mediator; - - client.InferenceDataReceived += async (sender, command) => - { - try - { - if (command.Message == "DONE") - { - _inferenceCancelTokenSource?.Cancel(); - return; - } - - var annImage = MessagePackSerializer.Deserialize(command.Data); - await ProcessDetection(annImage); - } - catch (Exception e) - { - logger.LogError(e, e.Message); - } - }; - } - - private async Task ProcessDetection(AnnotationImage annotationImage, CancellationToken ct = default) - { - var annotation = await _annotationService.SaveAnnotation(annotationImage, ct); - await _mediator.Publish(new AnnotationAddedEvent(annotation), ct); - } - - public async Task RunInference(List mediaPaths, CancellationToken ct = default) - { - _inferenceCancelTokenSource = new CancellationTokenSource(); - _client.Send(RemoteCommand.Create(CommandType.Login, _azaionApi.Credentials)); - - var aiConfig = _aiConfigOptions.Value; - aiConfig.Paths = mediaPaths; - _client.Send(RemoteCommand.Create(CommandType.Inference, aiConfig)); - - using var combinedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(ct, _inferenceCancelTokenSource.Token); - await combinedTokenSource.Token.AsTask(); - } - - public void StopInference() => _client.Stop(); -} \ No newline at end of file diff --git a/Azaion.Inference/ai_availability_status.pxd b/Azaion.Inference/ai_availability_status.pxd new file mode 100644 index 0000000..5501113 --- /dev/null +++ b/Azaion.Inference/ai_availability_status.pxd @@ -0,0 +1,14 @@ +cdef enum AIAvailabilityEnum: + NONE = 0 + DOWNLOADING = 10 + CONVERTING = 20 + UPLOADING = 30 + ENABLED = 200 + ERROR = 500 + +cdef class AIAvailabilityStatus: + cdef AIAvailabilityEnum status + cdef str error_message + + cdef bytes serialize(self) + cdef set_status(self, AIAvailabilityEnum status, str error_message=*) \ No newline at end of file diff --git a/Azaion.Inference/ai_availability_status.pyx b/Azaion.Inference/ai_availability_status.pyx new file mode 100644 index 0000000..d467682 --- /dev/null +++ b/Azaion.Inference/ai_availability_status.pyx @@ -0,0 +1,36 @@ +cimport constants_inf +import msgpack + +AIStatus2Text = { + AIAvailabilityEnum.NONE: "None", + AIAvailabilityEnum.DOWNLOADING: "Downloading", + AIAvailabilityEnum.CONVERTING: "Converting", + AIAvailabilityEnum.UPLOADING: "Uploading", + AIAvailabilityEnum.ENABLED: "Enabled", + AIAvailabilityEnum.ERROR: "Error", +} + +cdef class AIAvailabilityStatus: + def __init__(self): + self.status = AIAvailabilityEnum.NONE + self.error_message = None + + def __str__(self): + status_text = AIStatus2Text.get(self.status, "Unknown") + error_text = self.error_message if self.error_message else "" + return f"{status_text} {error_text}" + + cdef bytes serialize(self): + return msgpack.packb({ + "s": self.status, + "m": self.error_message + }) + + cdef set_status(self, AIAvailabilityEnum status, str error_message=None): + self.status = status + self.error_message = error_message + if error_message is not None: + constants_inf.logerror(error_message) + else: + constants_inf.log(str(self)) + diff --git a/Azaion.Inference/build_inference.cmd b/Azaion.Inference/build_inference.cmd index a880c9b..dc99d3e 100644 --- a/Azaion.Inference/build_inference.cmd +++ b/Azaion.Inference/build_inference.cmd @@ -35,6 +35,7 @@ venv\Scripts\pyinstaller --name=azaion-inference ^ --collect-all jwt ^ --collect-all loguru ^ --hidden-import constants_inf ^ +--hidden-import ai_availability_status ^ --hidden-import file_data ^ --hidden-import remote_command_inf ^ --hidden-import remote_command_handler_inf ^ @@ -49,8 +50,9 @@ start.py robocopy "dist\azaion-inference\_internal" "..\dist-azaion\_internal" "ai_config.cp312-win_amd64.pyd" "annotation.cp312-win_amd64.pyd" robocopy "dist\azaion-inference\_internal" "..\dist-azaion\_internal" "constants_inf.cp312-win_amd64.pyd" "file_data.cp312-win_amd64.pyd" +robocopy "dist\azaion-inference\_internal" "..\dist-azaion\_internal" "ai_availability_status.pyd" robocopy "dist\azaion-inference\_internal" "..\dist-azaion\_internal" "remote_command_inf.cp312-win_amd64.pyd" "remote_command_handler_inf.cp312-win_amd64.pyd" -robocopy "dist\azaion-inference\_internal" "..\dist-azaion\_internal" "inference.cp312-win_amd64.pyd" "inference_engine.cp312-win_amd64.pyd" +robocopy "dist\azaion-inference\_internal" "..\dist-azaion\_internal" "inference.cp312-win_amd64.py=d" "inference_engine.cp312-win_amd64.pyd" robocopy "dist\azaion-inference\_internal" "..\dist-azaion\_internal" "loader_client.cp312-win_amd64.pyd" "tensorrt_engine.cp312-win_amd64.pyd" robocopy "dist\azaion-inference\_internal" "..\dist-azaion\_internal" "onnx_engine.cp312-win_amd64.pyd" "main_inference.cp312-win_amd64.pyd" diff --git a/Azaion.Inference/inference.pxd b/Azaion.Inference/inference.pxd index 5799b22..c540595 100644 --- a/Azaion.Inference/inference.pxd +++ b/Azaion.Inference/inference.pxd @@ -1,3 +1,4 @@ +from ai_availability_status cimport AIAvailabilityStatus from remote_command_inf cimport RemoteCommand from annotation cimport Annotation, Detection from ai_config cimport AIRecognitionConfig @@ -12,6 +13,7 @@ cdef class Inference: cdef dict[str, list(Detection)] _tile_detections cdef AIRecognitionConfig ai_config cdef bint stop_signal + cdef AIAvailabilityStatus ai_availability_status cdef str model_input cdef int model_width @@ -19,7 +21,7 @@ cdef class Inference: cdef int tile_width cdef int tile_height - cdef build_tensor_engine(self, object updater_callback) + cdef bytes get_onnx_engine_bytes(self) cdef init_ai(self) cdef bint is_building_engine cdef bint is_video(self, str filepath) diff --git a/Azaion.Inference/inference.pyx b/Azaion.Inference/inference.pyx index 467227b..1bdd3f5 100644 --- a/Azaion.Inference/inference.pyx +++ b/Azaion.Inference/inference.pyx @@ -5,6 +5,8 @@ from pathlib import Path import cv2 import numpy as np cimport constants_inf + +from ai_availability_status cimport AIAvailabilityEnum, AIAvailabilityStatus from remote_command_inf cimport RemoteCommand from annotation cimport Detection, Annotation from ai_config cimport AIRecognitionConfig @@ -60,67 +62,59 @@ cdef class Inference: self.tile_height = 0 self.engine = None self.is_building_engine = False + self.ai_availability_status = AIAvailabilityStatus() + self.init_ai() - cdef build_tensor_engine(self, object updater_callback): - if tensor_gpu_index == -1: - return - - try: - engine_filename = TensorRTEngine.get_engine_filename(0) - models_dir = constants_inf.MODELS_FOLDER - - self.is_building_engine = True - updater_callback('downloading') - - res = self.loader_client.load_big_small_resource(engine_filename, models_dir) - if res.err is None: - constants_inf.log('tensor rt engine is here, no need to build') - self.is_building_engine = False - updater_callback('enabled') - return - - constants_inf.logerror(res.err) - # time.sleep(8) # prevent simultaneously loading dll and models - updater_callback('converting') - constants_inf.log('try to load onnx') - res = self.loader_client.load_big_small_resource(constants_inf.AI_ONNX_MODEL_FILE, models_dir) - if res.err is not None: - updater_callback(f'Error. {res.err}') - model_bytes = TensorRTEngine.convert_from_onnx(res.data) - updater_callback('uploading') - res = self.loader_client.upload_big_small_resource(model_bytes, engine_filename, models_dir) - if res.err is not None: - updater_callback(f'Error. {res.err}') - constants_inf.log(f'uploaded {engine_filename} to CDN and API') - self.is_building_engine = False - updater_callback('enabled') - except Exception as e: - updater_callback(f'Error. {str(e)}') + cdef bytes get_onnx_engine_bytes(self): + models_dir = constants_inf.MODELS_FOLDER + self.ai_availability_status.set_status(AIAvailabilityEnum.DOWNLOADING) + res = self.loader_client.load_big_small_resource(constants_inf.AI_ONNX_MODEL_FILE, models_dir) + if res.err is not None: + raise Exception(res.err) + return res.data cdef init_ai(self): - if self.engine is not None: - return - - models_dir = constants_inf.MODELS_FOLDER - if tensor_gpu_index > -1: + constants_inf.log( 'init AI...') + try: while self.is_building_engine: time.sleep(1) - engine_filename = TensorRTEngine.get_engine_filename(0) + if self.engine is not None: + return + + self.is_building_engine = True + models_dir = constants_inf.MODELS_FOLDER + if tensor_gpu_index > -1: + try: + engine_filename = TensorRTEngine.get_engine_filename(0) + self.ai_availability_status.set_status(AIAvailabilityEnum.DOWNLOADING) + res = self.loader_client.load_big_small_resource(engine_filename, models_dir) + if res.err is not None: + raise Exception(res.err) + self.engine = TensorRTEngine(res.data) + self.ai_availability_status.set_status(AIAvailabilityEnum.ENABLED) + except Exception as e: + self.ai_availability_status.set_status(AIAvailabilityEnum.ERROR, str(e)) + onnx_engine_bytes = self.get_onnx_engine_bytes() + self.ai_availability_status.set_status(AIAvailabilityEnum.CONVERTING) + model_bytes = TensorRTEngine.convert_from_onnx(res.data) + self.ai_availability_status.set_status(AIAvailabilityEnum.UPLOADING) + res = self.loader_client.upload_big_small_resource(model_bytes, engine_filename, models_dir) + if res.err is not None: + self.ai_availability_status.set_status(AIAvailabilityEnum.ERROR, res.err) + self.ai_availability_status.set_status(AIAvailabilityEnum.ENABLED) + else: + self.engine = OnnxEngine(self.get_onnx_engine_bytes()) + self.is_building_engine = False + + self.model_height, self.model_width = self.engine.get_input_shape() + #todo: temporarily, send it from the client + self.tile_width = 550 + self.tile_height = 550 + except Exception as e: + self.ai_availability_status.set_status(AIAvailabilityEnum.ERROR, str(e)) + self.is_building_engine = False - res = self.loader_client.load_big_small_resource(engine_filename, models_dir) - if res.err is not None: - raise Exception(res.err) - self.engine = TensorRTEngine(res.data) - else: - res = self.loader_client.load_big_small_resource(constants_inf.AI_ONNX_MODEL_FILE, models_dir) - if res.err is not None: - raise Exception(res.err) - self.engine = OnnxEngine(res.data) - self.model_height, self.model_width = self.engine.get_input_shape() - #todo: temporarily, send it from the client - self.tile_width = 550 - self.tile_height = 550 cdef preprocess(self, frames): blobs = [cv2.dnn.blobFromImage(frame, diff --git a/Azaion.Inference/main_inference.pyx b/Azaion.Inference/main_inference.pyx index 6978fd8..6d27280 100644 --- a/Azaion.Inference/main_inference.pyx +++ b/Azaion.Inference/main_inference.pyx @@ -44,8 +44,8 @@ cdef class CommandProcessor: if command.command_type == CommandType.INFERENCE: self.inference_queue.put(command) elif command.command_type == CommandType.AI_AVAILABILITY_CHECK: - self.inference.build_tensor_engine(lambda status: self.remote_handler.send(command.client_id, - RemoteCommand(CommandType.AI_AVAILABILITY_RESULT, None, status).serialize())) + status = self.inference.ai_availability_status.serialize() + self.remote_handler.send(command.client_id, RemoteCommand(CommandType.AI_AVAILABILITY_RESULT, status).serialize()) elif command.command_type == CommandType.STOP_INFERENCE: self.inference.stop() elif command.command_type == CommandType.EXIT: diff --git a/Azaion.Inference/setup.py b/Azaion.Inference/setup.py index 54901f1..b7aebbf 100644 --- a/Azaion.Inference/setup.py +++ b/Azaion.Inference/setup.py @@ -14,6 +14,7 @@ trace_line = False extensions = [ Extension('constants_inf', ['constants_inf.pyx'], **debug_args), + Extension('ai_availability_status', ['ai_availability_status.pyx'], **debug_args), Extension('file_data', ['file_data.pyx'], **debug_args), Extension('remote_command_inf', ['remote_command_inf.pyx'], **debug_args), Extension('remote_command_handler_inf', ['remote_command_handler_inf.pyx'], **debug_args), diff --git a/Azaion.Suite/App.xaml.cs b/Azaion.Suite/App.xaml.cs index 0f7b671..865d5c9 100644 --- a/Azaion.Suite/App.xaml.cs +++ b/Azaion.Suite/App.xaml.cs @@ -11,6 +11,7 @@ using Azaion.Common.DTO.Config; using Azaion.Common.Events; using Azaion.Common.Extensions; using Azaion.Common.Services; +using Azaion.Common.Services.Inference; using Azaion.Dataset; using CommandLine; using LibVLCSharp.Shared; diff --git a/Azaion.Suite/MainSuite.xaml.cs b/Azaion.Suite/MainSuite.xaml.cs index 912c66d..d0272d5 100644 --- a/Azaion.Suite/MainSuite.xaml.cs +++ b/Azaion.Suite/MainSuite.xaml.cs @@ -6,8 +6,8 @@ using System.Windows.Media; using Azaion.Common.Database; using Azaion.Common.DTO; using Azaion.Common.DTO.Config; -using Azaion.Common.Extensions; using Azaion.Common.Services; +using Azaion.Common.Services.Inference; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; using SharpVectors.Converters; diff --git a/Azaion.Suite/config.json b/Azaion.Suite/config.json index 748dbb9..711fdb7 100644 --- a/Azaion.Suite/config.json +++ b/Azaion.Suite/config.json @@ -1,12 +1,12 @@ { "LoaderClientConfig": { "ZeroMqHost": "127.0.0.1", - "ZeroMqPort": 5024, + "ZeroMqPort": 5025, "ApiUrl": "https://api.azaion.com" }, "InferenceClientConfig": { "ZeroMqHost": "127.0.0.1", - "ZeroMqPort": 5126, + "ZeroMqPort": 5127, "ApiUrl": "https://api.azaion.com" }, "GpsDeniedClientConfig": {