From fc6e5db795f029c579969043a2bb679c9cab0b61 Mon Sep 17 00:00:00 2001 From: Oleksandr Bezdieniezhnykh Date: Mon, 28 Jul 2025 12:39:52 +0300 Subject: [PATCH] add manual Tile Processor zoom on video on pause (temp image) --- .gitignore | 3 +- Azaion.Annotator/Annotator.xaml.cs | 23 +-- Azaion.Annotator/AnnotatorEventHandler.cs | 184 ++++++++++++------ .../{SecurityConstants.cs => Constants.cs} | 30 ++- Azaion.Common/Controls/CanvasEditor.cs | 25 ++- Azaion.Common/Controls/DetectionControl.cs | 42 ++-- .../Controls/DetectionLabelPanel.xaml | 59 ++++++ .../Controls/DetectionLabelPanel.xaml.cs | 55 ++++++ Azaion.Common/DTO/AffiliationEnum.cs | 9 + .../DTO/Config/AIRecognitionConfig.cs | 1 + Azaion.Common/DTO/FormState.cs | 4 +- Azaion.Common/DTO/Label.cs | 39 +++- Azaion.Common/Database/DbFactory.cs | 4 +- Azaion.Common/Database/SchemaMigrator.cs | 94 +++++++++ Azaion.Common/Services/TileProcessor.cs | 82 ++++++++ Azaion.Dataset/DatasetExplorerEventHandler.cs | 2 +- Azaion.Inference/README.md | 24 +-- Azaion.Inference/ai_config.pxd | 4 +- Azaion.Inference/ai_config.pyx | 4 + Azaion.Inference/annotation.pxd | 2 +- Azaion.Inference/annotation.pyx | 4 +- Azaion.Inference/inference.pxd | 6 +- Azaion.Inference/inference.pyx | 73 +++++-- Azaion.Inference/requirements.txt | 7 +- Azaion.Inference/setup.py | 42 ++-- Azaion.Inference/setup_old.py | 37 ++++ Azaion.Inference/test/test_inference.py | 8 + Azaion.Loader/requirements.txt | 2 +- Azaion.LoaderUI/App.xaml.cs | 5 +- Azaion.LoaderUI/Constants.cs | 13 -- Azaion.LoaderUI/ConstantsLoader.cs | 7 + Azaion.LoaderUI/Login.xaml.cs | 20 +- Azaion.Suite/App.xaml.cs | 8 +- Azaion.Suite/config.system.json | 3 +- 34 files changed, 716 insertions(+), 209 deletions(-) rename Azaion.Common/{SecurityConstants.cs => Constants.cs} (89%) create mode 100644 Azaion.Common/Controls/DetectionLabelPanel.xaml create mode 100644 Azaion.Common/Controls/DetectionLabelPanel.xaml.cs create mode 100644 Azaion.Common/DTO/AffiliationEnum.cs create mode 100644 Azaion.Common/Database/SchemaMigrator.cs create mode 100644 Azaion.Common/Services/TileProcessor.cs create mode 100644 Azaion.Inference/setup_old.py create mode 100644 Azaion.Inference/test/test_inference.py delete mode 100644 Azaion.LoaderUI/Constants.cs create mode 100644 Azaion.LoaderUI/ConstantsLoader.cs diff --git a/.gitignore b/.gitignore index 5f9d1be..fe8a470 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,5 @@ Azaion*.bin azaion\.*\.big _internal *.spec -dist \ No newline at end of file +dist +*.jpg diff --git a/Azaion.Annotator/Annotator.xaml.cs b/Azaion.Annotator/Annotator.xaml.cs index bc2091f..13bfb44 100644 --- a/Azaion.Annotator/Annotator.xaml.cs +++ b/Azaion.Annotator/Annotator.xaml.cs @@ -56,6 +56,7 @@ public partial class Annotator public Dictionary MediaFilesDict = new(); public IntervalTree TimedAnnotations { get; set; } = new(); + public string MainTitle { get; set; } public Annotator( IConfigUpdater configUpdater, @@ -72,7 +73,9 @@ public partial class Annotator IGpsMatcherService gpsMatcherService) { InitializeComponent(); - + + MainTitle = $"Azaion Annotator {Constants.GetLocalVersion()}"; + Title = MainTitle; _appConfig = appConfig.Value; _configUpdater = configUpdater; _libVLC = libVLC; @@ -194,7 +197,7 @@ public partial class Annotator _formState.CurrentMrl = _mediaPlayer.Media?.Mrl ?? ""; uint vw = 0, vh = 0; _mediaPlayer.Size(0, ref vw, ref vh); - _formState.CurrentVideoSize = new Size(vw, vh); + _formState.CurrentMediaSize = new Size(vw, vh); _formState.CurrentVideoLength = TimeSpan.FromMilliseconds(_mediaPlayer.Length); }; @@ -289,27 +292,23 @@ public partial class Annotator StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}"; Editor.ClearExpiredAnnotations(time); }); - - ShowAnnotation(TimedAnnotations.Query(time).FirstOrDefault(), showImage); + var annotation = TimedAnnotations.Query(time).FirstOrDefault(); + if (annotation != null) ShowAnnotation(annotation, showImage); } - private void ShowAnnotation(Annotation? annotation, bool showImage = false) + private void ShowAnnotation(Annotation annotation, bool showImage = false) { - if (annotation == null) - return; Dispatcher.Invoke(async () => { - var videoSize = _formState.CurrentVideoSize; if (showImage) { if (File.Exists(annotation.ImagePath)) { Editor.SetBackground(await annotation.ImagePath.OpenImage()); _formState.BackgroundTime = annotation.Time; - videoSize = Editor.RenderSize; } } - Editor.CreateDetections(annotation.Time, annotation.Detections, _appConfig.AnnotationConfig.DetectionClasses, videoSize); + Editor.CreateDetections(annotation.Time, annotation.Detections, _appConfig.AnnotationConfig.DetectionClasses, _formState.CurrentMediaSize); }); } @@ -321,7 +320,7 @@ public partial class Annotator var annotations = await _dbFactory.Run(async db => await db.Annotations.LoadWith(x => x.Detections) - .Where(x => x.OriginalMediaName == _formState.VideoName) + .Where(x => x.OriginalMediaName == _formState.MediaName) .OrderBy(x => x.Time) .ToListAsync(token: MainCancellationSource.Token)); @@ -583,13 +582,11 @@ public partial class Annotator private void SoundDetections(object sender, RoutedEventArgs e) { MessageBox.Show("Функція Аудіоаналіз знаходиться в стадії розробки","Система", MessageBoxButton.OK, MessageBoxImage.Information); - _logger.LogInformation("Denys wishes #1. To be implemented"); } private void RunDroneMaintenance(object sender, RoutedEventArgs e) { MessageBox.Show("Функція Аналіз стану БПЛА знаходиться в стадії розробки","Система", MessageBoxButton.OK, MessageBoxImage.Information); - _logger.LogInformation("Denys wishes #2. To be implemented"); } #endregion diff --git a/Azaion.Annotator/AnnotatorEventHandler.cs b/Azaion.Annotator/AnnotatorEventHandler.cs index 92dc415..0607530 100644 --- a/Azaion.Annotator/AnnotatorEventHandler.cs +++ b/Azaion.Annotator/AnnotatorEventHandler.cs @@ -2,6 +2,7 @@ using System.Windows; using System.Windows.Input; using System.Windows.Media; +using System.Windows.Media.Imaging; using Azaion.Annotator.Controls; using Azaion.Annotator.DTO; using Azaion.Common; @@ -47,7 +48,8 @@ public class AnnotatorEventHandler( private const int STEP = 20; private const int LARGE_STEP = 5000; private const int RESULT_WIDTH = 1280; - + private readonly string tempImgPath = Path.Combine(dirConfig.Value.ImagesDirectory, "___temp___.jpg"); + private readonly Dictionary _keysControlEnumDict = new() { { Key.Space, PlaybackControlEnum.Pause }, @@ -139,12 +141,21 @@ public class AnnotatorEventHandler( await Play(cancellationToken); break; case PlaybackControlEnum.Pause: - mediaPlayer.Pause(); - - if (formState.BackgroundTime.HasValue) + if (mediaPlayer.IsPlaying) { - mainWindow.Editor.SetBackground(null); - formState.BackgroundTime = null; + mediaPlayer.Pause(); + mediaPlayer.TakeSnapshot(0, tempImgPath, 0, 0); + mainWindow.Editor.SetBackground(await tempImgPath.OpenImage()); + formState.BackgroundTime = TimeSpan.FromMilliseconds(mediaPlayer.Time); + } + else + { + mediaPlayer.Play(); + if (formState.BackgroundTime.HasValue) + { + mainWindow.Editor.SetBackground(null); + formState.BackgroundTime = null; + } } break; case PlaybackControlEnum.Stop: @@ -159,7 +170,7 @@ public class AnnotatorEventHandler( mainWindow.SeekTo(mediaPlayer.Time + step); break; case PlaybackControlEnum.SaveAnnotations: - await SaveAnnotations(cancellationToken); + await SaveAnnotation(cancellationToken); break; case PlaybackControlEnum.RemoveSelectedAnns: @@ -226,63 +237,125 @@ public class AnnotatorEventHandler( if (mainWindow.LvFiles.SelectedItem == null) return; var mediaInfo = (MediaFileInfo)mainWindow.LvFiles.SelectedItem; - mainWindow.Editor.SetBackground(null); - + formState.CurrentMedia = mediaInfo; - mainWindow.Title = $"Azaion Annotator - {mediaInfo.Name}"; - - - //need to wait a bit for correct VLC playback event handling - await Task.Delay(100, ct); - mediaPlayer.Stop(); - mediaPlayer.Play(new Media(libVLC, mediaInfo.Path)); + mainWindow.Title = $"{mainWindow.MainTitle} - {mediaInfo.Name}"; + + if (mediaInfo.MediaType == MediaTypes.Video) + { + mainWindow.Editor.SetBackground(null); + //need to wait a bit for correct VLC playback event handling + await Task.Delay(100, ct); + mediaPlayer.Stop(); + mediaPlayer.Play(new Media(libVLC, mediaInfo.Path)); + } + else + { + formState.BackgroundTime = TimeSpan.Zero; + var image = await mediaInfo.Path.OpenImage(); + formState.CurrentMediaSize = new Size(image.PixelWidth, image.PixelHeight); + mainWindow.Editor.SetBackground(image); + mediaPlayer.Stop(); + } } //SAVE: MANUAL - private async Task SaveAnnotations(CancellationToken cancellationToken = default) + private async Task SaveAnnotation(CancellationToken cancellationToken = default) { if (formState.CurrentMedia == null) return; var time = formState.BackgroundTime ?? TimeSpan.FromMilliseconds(mediaPlayer.Time); - var originalMediaName = formState.VideoName; - var fName = originalMediaName.ToTimeName(time); - - var currentDetections = mainWindow.Editor.CurrentDetections - .Select(x => new Detection(fName, x.GetLabel(mainWindow.Editor.RenderSize, formState.BackgroundTime.HasValue ? mainWindow.Editor.RenderSize : formState.CurrentVideoSize))) - .ToList(); - - formState.CurrentMedia.HasAnnotations = currentDetections.Count != 0; - mainWindow.LvFiles.Items.Refresh(); - mainWindow.Editor.RemoveAllAnns(); - + var timeName = formState.MediaName.ToTimeName(time); var isVideo = formState.CurrentMedia.MediaType == MediaTypes.Video; - var imgPath = Path.Combine(dirConfig.Value.ImagesDirectory, $"{fName}{Constants.JPG_EXT}"); + var imgPath = Path.Combine(dirConfig.Value.ImagesDirectory, $"{timeName}{Constants.JPG_EXT}"); - if (formState.BackgroundTime.HasValue) + formState.CurrentMedia.HasAnnotations = mainWindow.Editor.CurrentDetections.Count != 0; + var annotations = await SaveAnnotationInner(imgPath, cancellationToken); + if (isVideo) { - //no need to save image, it's already there, just remove background - mainWindow.Editor.SetBackground(null); - formState.BackgroundTime = null; - - //next item - var annGrid = mainWindow.DgAnnotations; - annGrid.SelectedIndex = Math.Min(annGrid.Items.Count, annGrid.SelectedIndex + 1); - mainWindow.OpenAnnotationResult((AnnotationResult)annGrid.SelectedItem); + foreach (var annotation in annotations) + mainWindow.AddAnnotation(annotation); + mediaPlayer.Play(); + + // next item. Probably not needed + // var annGrid = mainWindow.DgAnnotations; + // annGrid.SelectedIndex = Math.Min(annGrid.Items.Count, annGrid.SelectedIndex + 1); + // mainWindow.OpenAnnotationResult((AnnotationResult)annGrid.SelectedItem); } else { - var resultHeight = (uint)Math.Round(RESULT_WIDTH / formState.CurrentVideoSize.Width * formState.CurrentVideoSize.Height); - mediaPlayer.TakeSnapshot(0, imgPath, RESULT_WIDTH, resultHeight); - if (isVideo) - mediaPlayer.Play(); - else - await NextMedia(ct: cancellationToken); + await NextMedia(ct: cancellationToken); } + mainWindow.Editor.SetBackground(null); + formState.BackgroundTime = null; + + mainWindow.LvFiles.Items.Refresh(); + mainWindow.Editor.RemoveAllAnns(); + } - var annotation = await annotationService.SaveAnnotation(originalMediaName, time, currentDetections, token: cancellationToken); - if (isVideo) - mainWindow.AddAnnotation(annotation); + private async Task> SaveAnnotationInner(string imgPath, CancellationToken cancellationToken = default) + { + var canvasDetections = mainWindow.Editor.CurrentDetections.Select(x => x.ToCanvasLabel()).ToList(); + var annotationsResult = new List(); + if (!File.Exists(imgPath)) + { + var source = (mainWindow.Editor.BackgroundImage.Source as BitmapSource)!; + if (source.PixelWidth <= RESULT_WIDTH * 2 && source.PixelHeight <= RESULT_WIDTH * 2) // Allow to be up to 2560*2560 to save to 1280*1280 + { + //Save image + await using var stream = new FileStream(imgPath, FileMode.Create); + var encoder = new JpegBitmapEncoder(); + encoder.Frames.Add(BitmapFrame.Create(source)); + encoder.Save(stream); + await stream.FlushAsync(cancellationToken); + } + else + { + //Tiling + + //1. Restore original picture coordinates + var pictureCoordinatesDetections = canvasDetections.Select(x => new CanvasLabel( + new YoloLabel(x, mainWindow.Editor.RenderSize, formState.CurrentMediaSize), formState.CurrentMediaSize, null, x.Confidence)) + .ToList(); + + //2. Split to 1280*1280 frames + var results = TileProcessor.Split(formState.CurrentMediaSize, pictureCoordinatesDetections, cancellationToken); + + //3. Save each frame as a separate annotation + BitmapEncoder tileEncoder = new JpegBitmapEncoder(); + foreach (var res in results) + { + var mediaName = $"{formState.MediaName}!split!{res.Tile.X}_{res.Tile.Y}!"; + var time = TimeSpan.Zero; + var annotationName = mediaName.ToTimeName(time); + + var tileImgPath = Path.Combine(dirConfig.Value.ImagesDirectory, $"{annotationName}{Constants.JPG_EXT}"); + await using var tileStream = new FileStream(tileImgPath, FileMode.Create); + var bitmap = new CroppedBitmap(source, new Int32Rect((int)res.Tile.X, (int)res.Tile.Y, (int)res.Tile.Width, (int)res.Tile.Height)); + tileEncoder.Frames.Add(BitmapFrame.Create(bitmap)); + tileEncoder.Save(tileStream); + await tileStream.FlushAsync(cancellationToken); + + var frameSize = new Size(res.Tile.Width, res.Tile.Height); + var detections = res.Detections + .Select(det => det.ReframeToSmall(res.Tile)) + .Select(x => new Detection(annotationName, new YoloLabel(x, frameSize))) + .ToList(); + + annotationsResult.Add(await annotationService.SaveAnnotation(mediaName, time, detections, token: cancellationToken)); + } + return annotationsResult; + } + } + + var timeImg = formState.BackgroundTime ?? TimeSpan.FromMilliseconds(mediaPlayer.Time); + var timeName = formState.MediaName.ToTimeName(timeImg); + var currentDetections = canvasDetections.Select(x => + new Detection(timeName, new YoloLabel(x, mainWindow.Editor.RenderSize))) + .ToList(); + var annotation = await annotationService.SaveAnnotation(formState.MediaName, timeImg, currentDetections, token: cancellationToken); + return [annotation]; } public async Task Handle(AnnotationsDeletedEvent notification, CancellationToken ct) @@ -316,21 +389,20 @@ public class AnnotatorEventHandler( }); await dbFactory.DeleteAnnotations(notification.AnnotationNames, ct); - - try + + foreach (var name in notification.AnnotationNames) { - foreach (var name in notification.AnnotationNames) + try { File.Delete(Path.Combine(dirConfig.Value.ImagesDirectory, $"{name}{Constants.JPG_EXT}")); File.Delete(Path.Combine(dirConfig.Value.LabelsDirectory, $"{name}{Constants.TXT_EXT}")); File.Delete(Path.Combine(dirConfig.Value.ThumbnailsDirectory, $"{name}{Constants.THUMBNAIL_PREFIX}{Constants.JPG_EXT}")); File.Delete(Path.Combine(dirConfig.Value.ResultsDirectory, $"{name}{Constants.RESULT_PREFIX}{Constants.JPG_EXT}")); } - } - catch (Exception e) - { - logger.LogError(e, e.Message); - throw; + catch (Exception e) + { + logger.LogError(e, e.Message); + } } //Only validators can send Delete to the queue @@ -403,4 +475,4 @@ public class AnnotatorEventHandler( map.SatelliteMap.Position = pointLatLon; map.SatelliteMap.ZoomAndCenterMarkers(null); } -} +} \ No newline at end of file diff --git a/Azaion.Common/SecurityConstants.cs b/Azaion.Common/Constants.cs similarity index 89% rename from Azaion.Common/SecurityConstants.cs rename to Azaion.Common/Constants.cs index eaa82d0..b92b8aa 100644 --- a/Azaion.Common/SecurityConstants.cs +++ b/Azaion.Common/Constants.cs @@ -1,4 +1,5 @@ -using System.IO; +using System.Diagnostics; +using System.IO; using Azaion.Common.DTO; using Azaion.Common.DTO.Config; using Azaion.Common.Extensions; @@ -11,9 +12,10 @@ namespace Azaion.Common; public class Constants { public const string CONFIG_PATH = "config.json"; - - private const string DEFAULT_API_URL = "https://api.azaion.com"; - + public const string LOADER_CONFIG_PATH = "loaderconfig.json"; + public const string DEFAULT_API_URL = "https://api.azaion.com"; + public const string AZAION_SUITE_EXE = "Azaion.Suite.exe"; + #region ExternalClientsConfig private const string DEFAULT_ZMQ_LOADER_HOST = "127.0.0.1"; @@ -103,14 +105,16 @@ public class Constants TrackingDistanceConfidence = TRACKING_DISTANCE_CONFIDENCE, TrackingProbabilityIncrease = TRACKING_PROBABILITY_INCREASE, TrackingIntersectionThreshold = TRACKING_INTERSECTION_THRESHOLD, + BigImageTileOverlapPercent = DEFAULT_BIG_IMAGE_TILE_OVERLAP_PERCENT, FramePeriodRecognition = DEFAULT_FRAME_PERIOD_RECOGNITION }; - public const double DEFAULT_FRAME_RECOGNITION_SECONDS = 2; - public const double TRACKING_DISTANCE_CONFIDENCE = 0.15; - public const double TRACKING_PROBABILITY_INCREASE = 15; - public const double TRACKING_INTERSECTION_THRESHOLD = 0.8; - public const int DEFAULT_FRAME_PERIOD_RECOGNITION = 4; + public const double DEFAULT_FRAME_RECOGNITION_SECONDS = 2; + public const double TRACKING_DISTANCE_CONFIDENCE = 0.15; + public const double TRACKING_PROBABILITY_INCREASE = 15; + public const double TRACKING_INTERSECTION_THRESHOLD = 0.8; + public const int DEFAULT_BIG_IMAGE_TILE_OVERLAP_PERCENT = 20; + public const int DEFAULT_FRAME_PERIOD_RECOGNITION = 4; # endregion AIRecognitionConfig @@ -251,4 +255,12 @@ public class Constants return DefaultInitConfig; } } + + public static Version GetLocalVersion() + { + var localFileInfo = FileVersionInfo.GetVersionInfo(AZAION_SUITE_EXE); + if (string.IsNullOrWhiteSpace(localFileInfo.ProductVersion)) + throw new Exception($"Can't find {AZAION_SUITE_EXE} and its version"); + return new Version(localFileInfo.FileVersion!); + } } \ No newline at end of file diff --git a/Azaion.Common/Controls/CanvasEditor.cs b/Azaion.Common/Controls/CanvasEditor.cs index 15ab798..48ea36c 100644 --- a/Azaion.Common/Controls/CanvasEditor.cs +++ b/Azaion.Common/Controls/CanvasEditor.cs @@ -6,6 +6,7 @@ using System.Windows.Media; using System.Windows.Media.Imaging; using System.Windows.Shapes; using Azaion.Common.DTO; +using Azaion.Common.Events; using MediatR; using Color = System.Windows.Media.Color; using Image = System.Windows.Controls.Image; @@ -34,10 +35,10 @@ public class CanvasEditor : Canvas private Point _panStartPoint; private bool _isZoomedIn; - private const int MIN_SIZE = 20; + private const int MIN_SIZE = 12; private readonly TimeSpan _viewThreshold = TimeSpan.FromMilliseconds(400); - private Image _backgroundImage { get; set; } = new() { Stretch = Stretch.Fill }; + public Image BackgroundImage { get; set; } = new() { Stretch = Stretch.Uniform }; public IMediator Mediator { get; set; } = null!; public static readonly DependencyProperty GetTimeFuncProp = @@ -113,7 +114,7 @@ public class CanvasEditor : Canvas MouseUp += CanvasMouseUp; SizeChanged += CanvasResized; Cursor = Cursors.Cross; - Children.Insert(0, _backgroundImage); + Children.Insert(0, BackgroundImage); Children.Add(_newAnnotationRect); Children.Add(_horizontalLine); Children.Add(_verticalLine); @@ -127,7 +128,7 @@ public class CanvasEditor : Canvas public void SetBackground(ImageSource? source) { SetZoom(); - _backgroundImage.Source = source; + BackgroundImage.Source = source; } private void SetZoom(Matrix? matrix = null) @@ -142,8 +143,8 @@ public class CanvasEditor : Canvas _matrixTransform.Matrix = matrix.Value; _isZoomedIn = true; } - foreach (var detection in CurrentDetections) - detection.UpdateAdornerScale(scale: _matrixTransform.Matrix.M11); + // foreach (var detection in CurrentDetections) + // detection.UpdateAdornerScale(scale: _matrixTransform.Matrix.M11); } private void CanvasWheel(object sender, MouseWheelEventArgs e) @@ -175,6 +176,8 @@ public class CanvasEditor : Canvas private void CanvasMouseDown(object sender, MouseButtonEventArgs e) { ClearSelections(); + if (e.LeftButton != MouseButtonState.Pressed) + return; if (Keyboard.Modifiers == ModifierKeys.Control && _isZoomedIn) { _panStartPoint = e.GetPosition(this); @@ -182,11 +185,13 @@ public class CanvasEditor : Canvas } else NewAnnotationStart(sender, e); + (sender as UIElement)?.CaptureMouse(); } private void CanvasMouseMove(object sender, MouseEventArgs e) { var pos = e.GetPosition(this); + Mediator.Publish(new SetStatusTextEvent($"Mouse Coordinates: {pos.X}, {pos.Y}")); _horizontalLine.Y1 = _horizontalLine.Y2 = pos.Y; _verticalLine.X1 = _verticalLine.X2 = pos.X; SetLeft(_classNameHint, pos.X + 10); @@ -216,11 +221,14 @@ public class CanvasEditor : Canvas var matrix = _matrixTransform.Matrix; matrix.Translate(delta.X, delta.Y); + _matrixTransform.Matrix = matrix; + Mediator.Publish(new SetStatusTextEvent(_matrixTransform.Matrix.ToString())); } private void CanvasMouseUp(object sender, MouseButtonEventArgs e) { + (sender as UIElement)?.ReleaseMouseCapture(); if (SelectionState == SelectionState.NewAnnCreating) { var endPos = e.GetPosition(this); @@ -279,8 +287,8 @@ public class CanvasEditor : Canvas { _horizontalLine.X2 = e.NewSize.Width; _verticalLine.Y2 = e.NewSize.Height; - _backgroundImage.Width = e.NewSize.Width; - _backgroundImage.Height = e.NewSize.Height; + BackgroundImage.Width = e.NewSize.Width; + BackgroundImage.Height = e.NewSize.Height; } #region Annotation Resizing & Moving @@ -383,7 +391,6 @@ public class CanvasEditor : Canvas private void NewAnnotationStart(object sender, MouseButtonEventArgs e) { _newAnnotationStartPos = e.GetPosition(this); - SetLeft(_newAnnotationRect, _newAnnotationStartPos.X); SetTop(_newAnnotationRect, _newAnnotationStartPos.Y); _newAnnotationRect.MouseMove += NewAnnotationCreatingMove; diff --git a/Azaion.Common/Controls/DetectionControl.cs b/Azaion.Common/Controls/DetectionControl.cs index 4c6bfe8..9974464 100644 --- a/Azaion.Common/Controls/DetectionControl.cs +++ b/Azaion.Common/Controls/DetectionControl.cs @@ -12,15 +12,15 @@ namespace Azaion.Common.Controls; public class DetectionControl : Border { private readonly Action _resizeStart; - private const double RESIZE_RECT_SIZE = 12; + private const double RESIZE_RECT_SIZE = 10; private readonly Grid _grid; - private readonly Label _detectionLabel; + private readonly DetectionLabelPanel _detectionLabelPanel; + //private readonly Label _detectionLabel; public readonly Canvas DetectionLabelContainer; public TimeSpan Time { get; set; } - private readonly double _confidence; - private List _resizedRectangles = new(); + private readonly List _resizedRectangles = new(); private DetectionClass _detectionClass = null!; public DetectionClass DetectionClass @@ -33,9 +33,8 @@ public class DetectionControl : Border BorderThickness = new Thickness(3); foreach (var rect in _resizedRectangles) rect.Stroke = brush; - - _detectionLabel.Background = new SolidColorBrush(value.Color.ToConfidenceColor(_confidence)); - _detectionLabel.Content = _detectionLabelText(value.UIName); + + _detectionLabelPanel.DetectionClass = value; _detectionClass = value; } } @@ -78,10 +77,7 @@ public class DetectionControl : Border DetectionLabelContainer.VerticalAlignment = value.Vertical; } } - - private string _detectionLabelText(string detectionClassName) => - _confidence >= 0.995 ? detectionClassName : $"{detectionClassName}: {_confidence * 100:F0}%"; //double - + public DetectionControl(DetectionClass detectionClass, TimeSpan time, Action resizeStart, CanvasLabel canvasLabel) { @@ -89,7 +85,6 @@ public class DetectionControl : Border Height = canvasLabel.Height; Time = time; _resizeStart = resizeStart; - _confidence = canvasLabel.Confidence; DetectionLabelContainer = new Canvas { @@ -97,16 +92,16 @@ public class DetectionControl : Border VerticalAlignment = VerticalAlignment.Top, ClipToBounds = false, }; - _detectionLabel = new Label + _detectionLabelPanel = new DetectionLabelPanel { - Content = _detectionLabelText(detectionClass.Name), - FontSize = 16, - Visibility = Visibility.Visible + Confidence = canvasLabel.Confidence }; - DetectionLabelContainer.Children.Add(_detectionLabel); + + DetectionLabelContainer.Children.Add(_detectionLabelPanel); _selectionFrame = new Rectangle { + Margin = new Thickness(-3), HorizontalAlignment = HorizontalAlignment.Stretch, VerticalAlignment = VerticalAlignment.Stretch, Stroke = new SolidColorBrush(Colors.Black), @@ -146,12 +141,13 @@ public class DetectionControl : Border var rect = new Rectangle() // small rectangles at the corners and sides { ClipToBounds = false, - Margin = new Thickness(-RESIZE_RECT_SIZE * 0.7), + Margin = new Thickness(-RESIZE_RECT_SIZE), HorizontalAlignment = ha, VerticalAlignment = va, Width = RESIZE_RECT_SIZE, Height = RESIZE_RECT_SIZE, Stroke = new SolidColorBrush(Color.FromArgb(230, 20, 20, 20)), // small rectangles color + StrokeThickness = 0.8, Fill = new SolidColorBrush(Color.FromArgb(150, 80, 80, 80)), Cursor = crs, Name = name, @@ -160,9 +156,9 @@ public class DetectionControl : Border return rect; } - public YoloLabel GetLabel(Size canvasSize, Size? videoSize = null) - { - var label = new CanvasLabel(DetectionClass.YoloId, Canvas.GetLeft(this), Canvas.GetTop(this), Width, Height); - return new YoloLabel(label, canvasSize, videoSize); - } + public CanvasLabel ToCanvasLabel() => + new(DetectionClass.YoloId, Canvas.GetLeft(this), Canvas.GetTop(this), Width, Height); + + public YoloLabel ToYoloLabel(Size canvasSize, Size? videoSize = null) => + new(ToCanvasLabel(), canvasSize, videoSize); } diff --git a/Azaion.Common/Controls/DetectionLabelPanel.xaml b/Azaion.Common/Controls/DetectionLabelPanel.xaml new file mode 100644 index 0000000..629bac4 --- /dev/null +++ b/Azaion.Common/Controls/DetectionLabelPanel.xaml @@ -0,0 +1,59 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Azaion.Common/Controls/DetectionLabelPanel.xaml.cs b/Azaion.Common/Controls/DetectionLabelPanel.xaml.cs new file mode 100644 index 0000000..cb596fb --- /dev/null +++ b/Azaion.Common/Controls/DetectionLabelPanel.xaml.cs @@ -0,0 +1,55 @@ +using System.Windows.Media; +using Azaion.Common.DTO; + +namespace Azaion.Common.Controls +{ + public partial class DetectionLabelPanel + { + private AffiliationEnum _affiliation = AffiliationEnum.None; + private double _confidence; + + public AffiliationEnum Affiliation + { + get => _affiliation; + set + { + _affiliation = value; + UpdateAffiliationImage(); + } + } + + public DetectionClass DetectionClass { get; set; } + + public double Confidence + { + get => _confidence; + set + { + _confidence = value; + + } + } + + public DetectionLabelPanel() + { + InitializeComponent(); + } + + private string _detectionLabelText(string detectionClassName) => + _confidence >= 0.98 ? detectionClassName : $"{detectionClassName}: {_confidence * 100:F0}%"; + + private void UpdateAffiliationImage() + { + if (_affiliation == AffiliationEnum.None) + { + AffiliationImage.Source = null; + return; + } + + if (TryFindResource(_affiliation.ToString()) is DrawingImage drawingImage) + AffiliationImage.Source = drawingImage; + else + AffiliationImage.Source = null; + } + } +} \ No newline at end of file diff --git a/Azaion.Common/DTO/AffiliationEnum.cs b/Azaion.Common/DTO/AffiliationEnum.cs new file mode 100644 index 0000000..9fc6343 --- /dev/null +++ b/Azaion.Common/DTO/AffiliationEnum.cs @@ -0,0 +1,9 @@ +namespace Azaion.Common.DTO; + +public enum AffiliationEnum +{ + None = 0, + Friendly = 10, + Hostile = 20, + Unknown = 30 +} \ No newline at end of file diff --git a/Azaion.Common/DTO/Config/AIRecognitionConfig.cs b/Azaion.Common/DTO/Config/AIRecognitionConfig.cs index 94394ed..c3eff14 100644 --- a/Azaion.Common/DTO/Config/AIRecognitionConfig.cs +++ b/Azaion.Common/DTO/Config/AIRecognitionConfig.cs @@ -12,6 +12,7 @@ public class AIRecognitionConfig [Key("t_dc")] public double TrackingDistanceConfidence { get; set; } [Key("t_pi")] public double TrackingProbabilityIncrease { get; set; } [Key("t_it")] public double TrackingIntersectionThreshold { get; set; } + [Key("ov_p")] public double BigImageTileOverlapPercent { get; set; } [Key("d")] public byte[] Data { get; set; } = null!; [Key("p")] public List Paths { get; set; } = null!; diff --git a/Azaion.Common/DTO/FormState.cs b/Azaion.Common/DTO/FormState.cs index 07e1ba7..fcac093 100644 --- a/Azaion.Common/DTO/FormState.cs +++ b/Azaion.Common/DTO/FormState.cs @@ -6,10 +6,10 @@ namespace Azaion.Common.DTO; public class FormState { public MediaFileInfo? CurrentMedia { get; set; } - public string VideoName => CurrentMedia?.FName ?? ""; + public string MediaName => CurrentMedia?.FName ?? ""; public string CurrentMrl { get; set; } = null!; - public Size CurrentVideoSize { get; set; } + public Size CurrentMediaSize { get; set; } public TimeSpan CurrentVideoLength { get; set; } public TimeSpan? BackgroundTime { get; set; } diff --git a/Azaion.Common/DTO/Label.cs b/Azaion.Common/DTO/Label.cs index 341ba1a..65baec7 100644 --- a/Azaion.Common/DTO/Label.cs +++ b/Azaion.Common/DTO/Label.cs @@ -22,16 +22,36 @@ public abstract class Label public class CanvasLabel : Label { - public double X { get; set; } - public double Y { get; set; } + public double X { get; set; } //left + public double Y { get; set; } //top public double Width { get; set; } public double Height { get; set; } public double Confidence { get; set; } - public CanvasLabel() + public double Bottom { + get => Y + Height; + set => Height = value - Y; } + public double Right + { + get => X + Width; + set => Width = value - X; + } + + public CanvasLabel() { } + + public CanvasLabel(double left, double right, double top, double bottom) + { + X = left; + Y = top; + Width = right - left; + Height = bottom - top; + Confidence = 1; + ClassNumber = -1; + } + public CanvasLabel(int classNumber, double x, double y, double width, double height, double confidence = 1) : base(classNumber) { X = x; @@ -77,6 +97,13 @@ public class CanvasLabel : Label } Confidence = confidence; } + + public CanvasLabel ReframeToSmall(CanvasLabel smallTile) => + new(ClassNumber, X - smallTile.X, Y - smallTile.Y, Width, Height, Confidence); + + public CanvasLabel ReframeFromSmall(CanvasLabel smallTile) => + new(ClassNumber, X + smallTile.X, Y + smallTile.Y, Width, Height, Confidence); + } [MessagePackObject] @@ -193,13 +220,15 @@ public class Detection : YoloLabel { [JsonProperty(PropertyName = "an")][Key("an")] public string AnnotationName { get; set; } = null!; [JsonProperty(PropertyName = "p")][Key("p")] public double Confidence { get; set; } - + [JsonProperty(PropertyName = "dn")][Key("dn")] public string Description { get; set; } + //For db & serialization public Detection(){} - public Detection(string annotationName, YoloLabel label, double confidence = 1) + public Detection(string annotationName, YoloLabel label, string description = "", double confidence = 1) { AnnotationName = annotationName; + Description = description; ClassNumber = label.ClassNumber; CenterX = label.CenterX; CenterY = label.CenterY; diff --git a/Azaion.Common/Database/DbFactory.cs b/Azaion.Common/Database/DbFactory.cs index 72875ae..1b85c50 100644 --- a/Azaion.Common/Database/DbFactory.cs +++ b/Azaion.Common/Database/DbFactory.cs @@ -60,8 +60,10 @@ public class DbFactory : IDbFactory if (!File.Exists(_annConfig.AnnotationsDbFile)) SQLiteConnection.CreateFile(_annConfig.AnnotationsDbFile); RecreateTables(); - + _fileConnection.Open(); + using var db = new AnnotationsDb(_fileDataOptions); + SchemaMigrator.EnsureSchemaUpdated(db, typeof(Annotation), typeof(Detection)); _fileConnection.BackupDatabase(_memoryConnection, "main", "main", -1, null, -1); } diff --git a/Azaion.Common/Database/SchemaMigrator.cs b/Azaion.Common/Database/SchemaMigrator.cs new file mode 100644 index 0000000..1b34996 --- /dev/null +++ b/Azaion.Common/Database/SchemaMigrator.cs @@ -0,0 +1,94 @@ +using System.Data; +using LinqToDB.Data; +using LinqToDB.Mapping; + +namespace Azaion.Common.Database; + +public static class SchemaMigrator +{ + public static void EnsureSchemaUpdated(DataConnection dbConnection, params Type[] entityTypes) + { + var connection = dbConnection.Connection; + var mappingSchema = dbConnection.MappingSchema; + + if (connection.State == ConnectionState.Closed) + { + connection.Open(); + } + + foreach (var type in entityTypes) + { + var entityDescriptor = mappingSchema.GetEntityDescriptor(type); + var tableName = entityDescriptor.Name.Name; + var existingColumns = GetTableColumns(connection, tableName); + + foreach (var column in entityDescriptor.Columns) + { + if (existingColumns.Contains(column.ColumnName, StringComparer.OrdinalIgnoreCase)) + continue; + + var columnDefinition = GetColumnDefinition(column); + dbConnection.Execute($"ALTER TABLE {tableName} ADD COLUMN {columnDefinition}"); + } + } + } + + private static HashSet GetTableColumns(IDbConnection connection, string tableName) + { + var columns = new HashSet(StringComparer.OrdinalIgnoreCase); + using var cmd = connection.CreateCommand(); + cmd.CommandText = $"PRAGMA table_info({tableName})"; + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + columns.Add(reader.GetString(1)); // "name" is in the second column + + return columns; + } + + private static string GetColumnDefinition(ColumnDescriptor column) + { + var type = column.MemberType; + var underlyingType = Nullable.GetUnderlyingType(type) ?? type; + var sqliteType = GetSqliteType(underlyingType); + var defaultClause = GetSqlDefaultValue(type, underlyingType); + + return $"\"{column.ColumnName}\" {sqliteType} {defaultClause}"; + } + + private static string GetSqliteType(Type type) => + type switch + { + _ when type == typeof(int) + || type == typeof(long) + || type == typeof(bool) + || type.IsEnum + => "INTEGER", + + _ when type == typeof(double) + || type == typeof(float) + || type == typeof(decimal) + => "REAL", + + _ when type == typeof(byte[]) + => "BLOB", + + _ => "TEXT" + }; + + private static string GetSqlDefaultValue(Type originalType, Type underlyingType) + { + var isNullable = originalType.IsClass || Nullable.GetUnderlyingType(originalType) != null; + if (isNullable) + return "NULL"; + + var defaultValue = Activator.CreateInstance(underlyingType); + + if (underlyingType == typeof(bool)) + return $"NOT NULL DEFAULT {(Convert.ToBoolean(defaultValue) ? 1 : 0)}"; + + if (underlyingType.IsValueType && defaultValue is IFormattable f) + return $"NOT NULL DEFAULT {f.ToString(null, System.Globalization.CultureInfo.InvariantCulture)}"; + + return $"NOT NULL DEFAULT '{defaultValue}'"; + } +} \ No newline at end of file diff --git a/Azaion.Common/Services/TileProcessor.cs b/Azaion.Common/Services/TileProcessor.cs new file mode 100644 index 0000000..c534595 --- /dev/null +++ b/Azaion.Common/Services/TileProcessor.cs @@ -0,0 +1,82 @@ +using System.Windows; +using System.Windows.Media.Imaging; +using Azaion.Common.DTO; + +namespace Azaion.Common.Services; + +public class TileResult +{ + public CanvasLabel Tile { get; set; } + public List Detections { get; set; } + + public TileResult(CanvasLabel tile, List detections) + { + Tile = tile; + Detections = detections; + } +} + +public static class TileProcessor +{ + private const int MaxTileWidth = 1280; + private const int MaxTileHeight = 1280; + private const int Border = 10; + + public static List Split(Size originalSize, List detections, CancellationToken cancellationToken) + { + var results = new List(); + var processingDetectionList = new List(detections); + + while (processingDetectionList.Count > 0 && !cancellationToken.IsCancellationRequested) + { + var topMostDetection = processingDetectionList + .OrderBy(d => d.Y) + .First(); + + var result = GetDetectionsInTile(originalSize, topMostDetection, processingDetectionList); + processingDetectionList.RemoveAll(x => result.Detections.Contains(x)); + results.Add(result); + } + return results; + } + + private static TileResult GetDetectionsInTile(Size originalSize, CanvasLabel startDet, List allDetections) + { + var tile = new CanvasLabel( + left: Math.Max(startDet.X - Border, 0), + right: Math.Min(startDet.Right + Border, originalSize.Width), + top: Math.Max(startDet.Y - Border, 0), + bottom: Math.Min(startDet.Bottom + Border, originalSize.Height)); + var selectedDetections = new List{startDet}; + + foreach (var det in allDetections) + { + if (det == startDet) + continue; + + var commonTile = new CanvasLabel( + left: Math.Max(Math.Min(tile.X, det.X) - Border, 0), + right: Math.Min(Math.Max(tile.Right, det.Right) + Border, originalSize.Width), + top: Math.Max(Math.Min(tile.Y, det.Y) - Border, 0), + bottom: Math.Min(Math.Max(tile.Bottom, det.Bottom) + Border, originalSize.Height) + ); + + if (commonTile.Width > MaxTileWidth || commonTile.Height > MaxTileHeight) + continue; + + tile = commonTile; + selectedDetections.Add(det); + } + + //normalization, width and height should be at least half of 1280px + tile.Width = Math.Max(tile.Width, MaxTileWidth / 2.0); + tile.Height = Math.Max(tile.Height, MaxTileHeight / 2.0); + + //boundaries check after normalization + tile.Right = Math.Min(tile.Right, originalSize.Width); + tile.Bottom = Math.Min(tile.Bottom, originalSize.Height); + + return new TileResult(tile, selectedDetections); + } + +} \ No newline at end of file diff --git a/Azaion.Dataset/DatasetExplorerEventHandler.cs b/Azaion.Dataset/DatasetExplorerEventHandler.cs index 2a39822..4d66a4e 100644 --- a/Azaion.Dataset/DatasetExplorerEventHandler.cs +++ b/Azaion.Dataset/DatasetExplorerEventHandler.cs @@ -67,7 +67,7 @@ public class DatasetExplorerEventHandler( var a = datasetExplorer.CurrentAnnotation!.Annotation; var detections = datasetExplorer.ExplorerEditor.CurrentDetections - .Select(x => new Detection(a.Name, x.GetLabel(datasetExplorer.ExplorerEditor.RenderSize))) + .Select(x => new Detection(a.Name, x.ToYoloLabel(datasetExplorer.ExplorerEditor.RenderSize))) .ToList(); var index = datasetExplorer.ThumbnailsView.SelectedIndex; var annotation = await annotationService.SaveAnnotation(a.OriginalMediaName, a.Time, detections, token: token); diff --git a/Azaion.Inference/README.md b/Azaion.Inference/README.md index 116a353..842b2ce 100644 --- a/Azaion.Inference/README.md +++ b/Azaion.Inference/README.md @@ -13,23 +13,14 @@ Results (file or annotations) is putted to the other queue, or the same socket,

Installation

-Prepare correct onnx model from YOLO: -```python -from ultralytics import YOLO -import netron - -model = YOLO("azaion.pt") -model.export(format="onnx", imgsz=1280, nms=True, batch=4) -netron.start('azaion.onnx') -``` -Read carefully about [export arguments](https://docs.ultralytics.com/modes/export/), you have to use nms=True, and batching with a proper batch size -

Install libs

https://www.python.org/downloads/ Windows - [Install CUDA](https://developer.nvidia.com/cuda-12-1-0-download-archive) +- [Install Visual Studio Build Tools 2019](https://visualstudio.microsoft.com/downloads/?q=build+tools) + Linux ``` @@ -44,6 +35,17 @@ Linux nvcc --version ``` +Prepare correct onnx model from YOLO: +```python +from ultralytics import YOLO +import netron + +model = YOLO("azaion.pt") +model.export(format="onnx", imgsz=1280, nms=True, batch=4) +netron.start('azaion.onnx') +``` +Read carefully about [export arguments](https://docs.ultralytics.com/modes/export/), you have to use nms=True, and batching with a proper batch size +

Install dependencies

1. Install python with max version 3.11. Pytorch for now supports 3.11 max diff --git a/Azaion.Inference/ai_config.pxd b/Azaion.Inference/ai_config.pxd index b5b19c5..90ebae8 100644 --- a/Azaion.Inference/ai_config.pxd +++ b/Azaion.Inference/ai_config.pxd @@ -7,9 +7,11 @@ cdef class AIRecognitionConfig: cdef public double tracking_probability_increase cdef public double tracking_intersection_threshold + cdef public int big_image_tile_overlap_percent + cdef public bytes file_data cdef public list[str] paths cdef public int model_batch_size @staticmethod - cdef from_msgpack(bytes data) \ No newline at end of file + cdef from_msgpack(bytes data) diff --git a/Azaion.Inference/ai_config.pyx b/Azaion.Inference/ai_config.pyx index 28af40e..acbdf8b 100644 --- a/Azaion.Inference/ai_config.pyx +++ b/Azaion.Inference/ai_config.pyx @@ -9,6 +9,7 @@ cdef class AIRecognitionConfig: tracking_distance_confidence, tracking_probability_increase, tracking_intersection_threshold, + big_image_tile_overlap_percent, file_data, paths, @@ -21,6 +22,7 @@ cdef class AIRecognitionConfig: self.tracking_distance_confidence = tracking_distance_confidence self.tracking_probability_increase = tracking_probability_increase self.tracking_intersection_threshold = tracking_intersection_threshold + self.big_image_tile_overlap_percent = big_image_tile_overlap_percent self.file_data = file_data self.paths = paths @@ -31,6 +33,7 @@ cdef class AIRecognitionConfig: f'probability_increase : {self.tracking_probability_increase}, ' f'intersection_threshold : {self.tracking_intersection_threshold}, ' f'frame_period_recognition : {self.frame_period_recognition}, ' + f'big_image_tile_overlap_percent: {self.big_image_tile_overlap_percent}, ' f'paths: {self.paths}, ' f'model_batch_size: {self.model_batch_size}') @@ -45,6 +48,7 @@ cdef class AIRecognitionConfig: unpacked.get("t_dc", 0.0), unpacked.get("t_pi", 0.0), unpacked.get("t_it", 0.0), + unpacked.get("ov_p", 20), unpacked.get("d", b''), unpacked.get("p", []), diff --git a/Azaion.Inference/annotation.pxd b/Azaion.Inference/annotation.pxd index b8b1b34..932e969 100644 --- a/Azaion.Inference/annotation.pxd +++ b/Azaion.Inference/annotation.pxd @@ -3,7 +3,7 @@ cdef class Detection: cdef public str annotation_name cdef public int cls - cdef public overlaps(self, Detection det2) + cdef public overlaps(self, Detection det2, float confidence_threshold) cdef class Annotation: cdef public str name diff --git a/Azaion.Inference/annotation.pyx b/Azaion.Inference/annotation.pyx index 1d4f481..454eda5 100644 --- a/Azaion.Inference/annotation.pyx +++ b/Azaion.Inference/annotation.pyx @@ -14,13 +14,13 @@ cdef class Detection: def __str__(self): return f'{self.cls}: {self.x:.2f} {self.y:.2f} {self.w:.2f} {self.h:.2f}, prob: {(self.confidence*100):.1f}%' - cdef overlaps(self, Detection det2): + cdef overlaps(self, Detection det2, float confidence_threshold): cdef double overlap_x = 0.5 * (self.w + det2.w) - abs(self.x - det2.x) cdef double overlap_y = 0.5 * (self.h + det2.h) - abs(self.y - det2.y) cdef double overlap_area = max(0.0, overlap_x) * max(0.0, overlap_y) cdef double min_area = min(self.w * self.h, det2.w * det2.h) - return overlap_area / min_area > 0.6 + return overlap_area / min_area > confidence_threshold cdef class Annotation: def __init__(self, str name, long ms, list[Detection] detections): diff --git a/Azaion.Inference/inference.pxd b/Azaion.Inference/inference.pxd index 9e69a25..45952f4 100644 --- a/Azaion.Inference/inference.pxd +++ b/Azaion.Inference/inference.pxd @@ -23,11 +23,13 @@ cdef class Inference: cdef run_inference(self, RemoteCommand cmd) cdef _process_video(self, RemoteCommand cmd, AIRecognitionConfig ai_config, str video_name) - cdef _process_images(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list[str] image_paths) + cpdef _process_images(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list[str] image_paths) + cpdef _process_images_inner(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list frame_data) + cpdef split_to_tiles(self, frame, path, img_w, img_h, overlap_percent) cdef stop(self) cdef preprocess(self, frames) - cdef remove_overlapping_detections(self, list[Detection] detections) + cdef remove_overlapping_detections(self, list[Detection] detections, float confidence_threshold=?) cdef postprocess(self, output, ai_config) cdef split_list_extend(self, lst, chunk_size) diff --git a/Azaion.Inference/inference.pyx b/Azaion.Inference/inference.pyx index d6ff306..5e6b16e 100644 --- a/Azaion.Inference/inference.pyx +++ b/Azaion.Inference/inference.pyx @@ -150,13 +150,13 @@ cdef class Inference: h = y2 - y1 if conf >= ai_config.probability_threshold: detections.append(Detection(x, y, w, h, class_id, conf)) - filtered_detections = self.remove_overlapping_detections(detections) + filtered_detections = self.remove_overlapping_detections(detections, ai_config.tracking_intersection_threshold) results.append(filtered_detections) return results except Exception as e: raise RuntimeError(f"Failed to postprocess: {str(e)}") - cdef remove_overlapping_detections(self, list[Detection] detections): + cdef remove_overlapping_detections(self, list[Detection] detections, float confidence_threshold=0.6): cdef Detection det1, det2 filtered_output = [] filtered_out_indexes = [] @@ -168,7 +168,7 @@ cdef class Inference: res = det1_index for det2_index in range(det1_index + 1, len(detections)): det2 = detections[det2_index] - if det1.overlaps(det2): + if det1.overlaps(det2, confidence_threshold): if det1.confidence > det2.confidence or ( det1.confidence == det2.confidence and det1.cls < det2.cls): # det1 has higher confidence or lower class_id filtered_out_indexes.append(det2_index) @@ -211,9 +211,8 @@ cdef class Inference: images.append(m) # images first, it's faster if len(images) > 0: - for chunk in self.split_list_extend(images, self.engine.get_batch_size()): - constants_inf.log(f'run inference on {" ".join(chunk)}...') - self._process_images(cmd, ai_config, chunk) + constants_inf.log(f'run inference on {" ".join(images)}...') + self._process_images(cmd, ai_config, images) if len(videos) > 0: for v in videos: constants_inf.log(f'run inference on {v}...') @@ -250,8 +249,6 @@ cdef class Inference: _, image = cv2.imencode('.jpg', batch_frames[i]) annotation.image = image.tobytes() self._previous_annotation = annotation - - print(annotation) self.on_annotation(cmd, annotation) batch_frames.clear() @@ -259,15 +256,53 @@ cdef class Inference: v_input.release() - cdef _process_images(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list[str] image_paths): - cdef list frames = [] - cdef list timestamps = [] - self._previous_annotation = None - for image in image_paths: - frame = cv2.imread(image) - frames.append(frame) - timestamps.append(0) + cpdef _process_images(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list[str] image_paths): + cdef list frame_data = [] + for path in image_paths: + frame = cv2.imread(path) + if frame is None: + constants_inf.logerror(f'Failed to read image {path}') + continue + img_h, img_w, _ = frame.shape + if img_h <= 1.5 * self.model_height and img_w <= 1.5 * self.model_width: + frame_data.append((frame, path)) + else: + (split_frames, split_pats) = self.split_to_tiles(frame, path, img_w, img_h, ai_config.big_image_tile_overlap_percent) + frame_data.extend(zip(split_frames, split_pats)) + for chunk in self.split_list_extend(frame_data, self.engine.get_batch_size()): + self._process_images_inner(cmd, ai_config, chunk) + + + cpdef split_to_tiles(self, frame, path, img_w, img_h, overlap_percent): + stride_w = self.model_width * (1 - overlap_percent / 100) + stride_h = self.model_height * (1 - overlap_percent / 100) + n_tiles_x = int(np.ceil((img_w - self.model_width) / stride_w)) + 1 + n_tiles_y = int(np.ceil((img_h - self.model_height) / stride_h)) + 1 + + results = [] + for y_idx in range(n_tiles_y): + for x_idx in range(n_tiles_x): + y_start = y_idx * stride_w + x_start = x_idx * stride_h + + # Ensure the tile doesn't go out of bounds + y_end = min(y_start + self.model_width, img_h) + x_end = min(x_start + self.model_height, img_w) + + # We need to re-calculate start if we are at the edge to get a full 1280x1280 tile + if y_end == img_h: + y_start = img_h - self.model_height + if x_end == img_w: + x_start = img_w - self.model_width + + tile = frame[y_start:y_end, x_start:x_end] + name = path.stem + f'.tile_{x_start}_{y_start}' + path.suffix + results.append((tile, name)) + return results + + cpdef _process_images_inner(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list frame_data): + frames = [frame for frame, _ in frame_data] input_blob = self.preprocess(frames) outputs = self.engine.run(input_blob) @@ -275,7 +310,7 @@ cdef class Inference: list_detections = self.postprocess(outputs, ai_config) for i in range(len(list_detections)): detections = list_detections[i] - annotation = Annotation(image_paths[i], timestamps[i], detections) + annotation = Annotation(frame_data[i][1], 0, detections) _, image = cv2.imencode('.jpg', frames[i]) annotation.image = image.tobytes() self.on_annotation(cmd, annotation) @@ -322,7 +357,9 @@ cdef class Inference: closest_det = prev_det # Check if beyond tracking distance - if min_distance_sq > ai_config.tracking_distance_confidence: + dist_px = ai_config.tracking_distance_confidence * self.model_width + dist_px_sq = dist_px * dist_px + if min_distance_sq > dist_px_sq: return True # Check probability increase diff --git a/Azaion.Inference/requirements.txt b/Azaion.Inference/requirements.txt index 6c9f1c9..61e3564 100644 --- a/Azaion.Inference/requirements.txt +++ b/Azaion.Inference/requirements.txt @@ -7,11 +7,12 @@ cryptography==44.0.2 psutil msgpack pyjwt -zmq +pyzmq requests pyyaml pycuda -tensorrt +tensorrt==10.11.0.33 pynvml boto3 -loguru \ No newline at end of file +loguru +pytest \ No newline at end of file diff --git a/Azaion.Inference/setup.py b/Azaion.Inference/setup.py index 91157a4..9e3edcc 100644 --- a/Azaion.Inference/setup.py +++ b/Azaion.Inference/setup.py @@ -2,19 +2,30 @@ from setuptools import setup, Extension from Cython.Build import cythonize import numpy as np +# debug_args = {} +# trace_line = False + +debug_args = { + 'extra_compile_args': ['-O0', '-g'], + 'extra_link_args': ['-g'], + 'define_macros': [('CYTHON_TRACE_NOGIL', '1')] +} +trace_line = True + extensions = [ - Extension('constants_inf', ['constants_inf.pyx']), - Extension('file_data', ['file_data.pyx']), - Extension('remote_command_inf', ['remote_command_inf.pyx']), - Extension('remote_command_handler_inf', ['remote_command_handler_inf.pyx']), - Extension('annotation', ['annotation.pyx']), - Extension('loader_client', ['loader_client.pyx']), - Extension('ai_config', ['ai_config.pyx']), - Extension('tensorrt_engine', ['tensorrt_engine.pyx'], include_dirs=[np.get_include()]), - Extension('onnx_engine', ['onnx_engine.pyx'], include_dirs=[np.get_include()]), - Extension('inference_engine', ['inference_engine.pyx'], include_dirs=[np.get_include()]), - Extension('inference', ['inference.pyx'], include_dirs=[np.get_include()]), - Extension('main_inference', ['main_inference.pyx']), + Extension('constants_inf', ['constants_inf.pyx'], **debug_args), + Extension('file_data', ['file_data.pyx'], **debug_args), + Extension('remote_command_inf', ['remote_command_inf.pyx'], **debug_args), + Extension('remote_command_handler_inf', ['remote_command_handler_inf.pyx'], **debug_args), + Extension('annotation', ['annotation.pyx'], **debug_args), + Extension('loader_client', ['loader_client.pyx'], **debug_args), + Extension('ai_config', ['ai_config.pyx'], **debug_args), + Extension('tensorrt_engine', ['tensorrt_engine.pyx'], include_dirs=[np.get_include()], **debug_args), + Extension('onnx_engine', ['onnx_engine.pyx'], include_dirs=[np.get_include()], **debug_args), + Extension('inference_engine', ['inference_engine.pyx'], include_dirs=[np.get_include()], **debug_args), + Extension('inference', ['inference.pyx'], include_dirs=[np.get_include()], **debug_args), + Extension('main_inference', ['main_inference.pyx'], **debug_args), + ] setup( @@ -23,10 +34,11 @@ setup( extensions, compiler_directives={ "language_level": 3, - "emit_code_comments" : False, + "emit_code_comments": False, "binding": True, 'boundscheck': False, - 'wraparound': False + 'wraparound': False, + 'linetrace': trace_line } ), install_requires=[ @@ -34,4 +46,4 @@ setup( 'pywin32; platform_system=="Windows"' ], zip_safe=False -) \ No newline at end of file +) diff --git a/Azaion.Inference/setup_old.py b/Azaion.Inference/setup_old.py new file mode 100644 index 0000000..3dec931 --- /dev/null +++ b/Azaion.Inference/setup_old.py @@ -0,0 +1,37 @@ +from setuptools import setup, Extension +from Cython.Build import cythonize +import numpy as np + +extensions = [ + Extension('constants_inf', ['constants_inf.pyx']), + Extension('file_data', ['file_data.pyx']), + Extension('remote_command_inf', ['remote_command_inf.pyx']), + Extension('remote_command_handler_inf', ['remote_command_handler_inf.pyx']), + Extension('annotation', ['annotation.pyx']), + Extension('loader_client', ['loader_client.pyx']), + Extension('ai_config', ['ai_config.pyx']), + Extension('tensorrt_engine', ['tensorrt_engine.pyx'], include_dirs=[np.get_include()]), + Extension('onnx_engine', ['onnx_engine.pyx'], include_dirs=[np.get_include()]), + Extension('inference_engine', ['inference_engine.pyx'], include_dirs=[np.get_include()]), + Extension('inference', ['inference.pyx'], include_dirs=[np.get_include()]), + Extension('main_inference', ['main_inference.pyx']) +] + +setup( + name="azaion.ai", + ext_modules=cythonize( + extensions, + compiler_directives={ + "language_level": 3, + "emit_code_comments" : False, + "binding": True, + 'boundscheck': False, + 'wraparound': False + } + ), + install_requires=[ + 'ultralytics>=8.0.0', + 'pywin32; platform_system=="Windows"' + ], + zip_safe=False +) \ No newline at end of file diff --git a/Azaion.Inference/test/test_inference.py b/Azaion.Inference/test/test_inference.py new file mode 100644 index 0000000..6407ad2 --- /dev/null +++ b/Azaion.Inference/test/test_inference.py @@ -0,0 +1,8 @@ +import inference +from ai_config import AIRecognitionConfig +from remote_command_inf import RemoteCommand + + +def test_process_images(): + inf = inference.Inference(None, None) + inf._process_images(RemoteCommand(30), AIRecognitionConfig(4, 2, 15, 0.15, 15, 0.8, 20, b'test', [], 4), ['test_img01.JPG', 'test_img02.jpg']) \ No newline at end of file diff --git a/Azaion.Loader/requirements.txt b/Azaion.Loader/requirements.txt index 77008ff..2dd5417 100644 --- a/Azaion.Loader/requirements.txt +++ b/Azaion.Loader/requirements.txt @@ -3,7 +3,7 @@ Cython psutil msgpack pyjwt -zmq +pyzmq requests pyyaml boto3 diff --git a/Azaion.LoaderUI/App.xaml.cs b/Azaion.LoaderUI/App.xaml.cs index ff41625..7da8d9e 100644 --- a/Azaion.LoaderUI/App.xaml.cs +++ b/Azaion.LoaderUI/App.xaml.cs @@ -1,4 +1,5 @@ using System.Windows; +using Azaion.Common; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -28,7 +29,7 @@ public partial class App var host = Host.CreateDefaultBuilder() .ConfigureAppConfiguration((_, config) => config .AddCommandLine(Environment.GetCommandLineArgs()) - .AddJsonFile(Constants.CONFIG_JSON_FILE, optional: true)) + .AddJsonFile(Constants.LOADER_CONFIG_PATH, optional: true)) .UseSerilog() .ConfigureServices((context, services) => { @@ -36,7 +37,7 @@ public partial class App services.Configure(context.Configuration.GetSection(nameof(DirectoriesConfig))); services.AddHttpClient((sp, client) => { - client.BaseAddress = new Uri(Constants.API_URL); + client.BaseAddress = new Uri(Constants.DEFAULT_API_URL); client.DefaultRequestHeaders.Add("Accept", "application/json"); client.DefaultRequestHeaders.Add("User-Agent", "Azaion.LoaderUI"); }); diff --git a/Azaion.LoaderUI/Constants.cs b/Azaion.LoaderUI/Constants.cs deleted file mode 100644 index 1171109..0000000 --- a/Azaion.LoaderUI/Constants.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace Azaion.LoaderUI; - -public static class Constants -{ - public const string CONFIG_JSON_FILE = "loaderconfig.json"; - public const string API_URL = "https://api.azaion.com"; - public const string AZAION_SUITE_EXE = "Azaion.Suite.exe"; - public const string SUITE_FOLDER = "suite"; - public const string INFERENCE_EXE = "azaion-inference"; - public const string EXTERNAL_LOADER_PATH = "azaion-loader.exe"; - public const int EXTERNAL_LOADER_PORT = 5020; - public const string EXTERNAL_LOADER_HOST = "127.0.0.1"; -} \ No newline at end of file diff --git a/Azaion.LoaderUI/ConstantsLoader.cs b/Azaion.LoaderUI/ConstantsLoader.cs new file mode 100644 index 0000000..7f38ef6 --- /dev/null +++ b/Azaion.LoaderUI/ConstantsLoader.cs @@ -0,0 +1,7 @@ +namespace Azaion.LoaderUI; + +public static class ConstantsLoader +{ + public const string SUITE_FOLDER = "suite"; + public const int EXTERNAL_LOADER_PORT = 5020; +} \ No newline at end of file diff --git a/Azaion.LoaderUI/Login.xaml.cs b/Azaion.LoaderUI/Login.xaml.cs index f42bdf0..80436e4 100644 --- a/Azaion.LoaderUI/Login.xaml.cs +++ b/Azaion.LoaderUI/Login.xaml.cs @@ -57,7 +57,7 @@ public partial class Login TbStatus.Foreground = Brushes.Black; var installerVersion = await GetInstallerVer(); - var localVersion = GetLocalVer(); + var localVersion = Constants.GetLocalVersion(); var credsEncrypted = Security.Encrypt(creds); if (installerVersion > localVersion) @@ -81,7 +81,7 @@ public partial class Login Process.Start(Constants.AZAION_SUITE_EXE, $"-c {credsEncrypted}"); await Task.Delay(800); TbStatus.Text = "Loading..."; - while (!Process.GetProcessesByName(Constants.INFERENCE_EXE).Any()) + while (!Process.GetProcessesByName(Path.GetFileNameWithoutExtension(Constants.EXTERNAL_INFERENCE_PATH)).Any()) await Task.Delay(500); await Task.Delay(1500); } @@ -106,12 +106,12 @@ public partial class Login process.StartInfo = new ProcessStartInfo { FileName = Constants.EXTERNAL_LOADER_PATH, - Arguments = $"--port {Constants.EXTERNAL_LOADER_PORT} --api {Constants.API_URL}", + Arguments = $"--port {ConstantsLoader.EXTERNAL_LOADER_PORT} --api {Constants.DEFAULT_API_URL}", CreateNoWindow = true }; process.Start(); dealer.Options.Identity = Encoding.UTF8.GetBytes(Guid.NewGuid().ToString("N")); - dealer.Connect($"tcp://{Constants.EXTERNAL_LOADER_HOST}:{Constants.EXTERNAL_LOADER_PORT}"); + dealer.Connect($"tcp://{Constants.DEFAULT_ZMQ_INFERENCE_HOST}:{ConstantsLoader.EXTERNAL_LOADER_PORT}"); var result = SendCommand(dealer, RemoteCommand.Create(CommandType.Login, creds)); if (result.CommandType != CommandType.Ok) @@ -164,7 +164,7 @@ public partial class Login { TbStatus.Text = "Checking for the newer version..."; var installerDir = string.IsNullOrWhiteSpace(_dirConfig?.SuiteInstallerDirectory) - ? Constants.SUITE_FOLDER + ? ConstantsLoader.SUITE_FOLDER : _dirConfig.SuiteInstallerDirectory; var installerName = await _azaionApi.GetLastInstallerName(installerDir); var match = Regex.Match(installerName, @"\d+(\.\d+)+"); @@ -172,15 +172,7 @@ public partial class Login throw new Exception($"Can't find version in {installerName}"); return new Version(match.Value); } - - private Version GetLocalVer() - { - var localFileInfo = FileVersionInfo.GetVersionInfo(Constants.AZAION_SUITE_EXE); - if (string.IsNullOrWhiteSpace(localFileInfo.ProductVersion)) - throw new Exception($"Can't find {Constants.AZAION_SUITE_EXE} and its version"); - return new Version(localFileInfo.FileVersion!); - } - + private void CloseClick(object sender, RoutedEventArgs e) => Close(); private void MainMouseMove(object sender, MouseEventArgs e) diff --git a/Azaion.Suite/App.xaml.cs b/Azaion.Suite/App.xaml.cs index d5dfc5f..1b62fbc 100644 --- a/Azaion.Suite/App.xaml.cs +++ b/Azaion.Suite/App.xaml.cs @@ -153,12 +153,12 @@ public partial class App typeof(Annotator.Annotator).Assembly, typeof(DatasetExplorer).Assembly, typeof(AnnotationService).Assembly)); - services.AddSingleton(_ => new LibVLC()); + services.AddSingleton(_ => new LibVLC("--no-osd", "--no-video-title-show", "--no-snapshot-preview")); services.AddSingleton(); services.AddSingleton(sp => { - var libVLC = sp.GetRequiredService(); - return new MediaPlayer(libVLC); + var libVlc = sp.GetRequiredService(); + return new MediaPlayer(libVlc); }); services.AddSingleton(); services.AddSingleton(); @@ -177,8 +177,6 @@ public partial class App Annotation.InitializeDirs(_host.Services.GetRequiredService>().Value); _host.Services.GetRequiredService(); - // datasetExplorer.Show(); - // datasetExplorer.Hide(); _mediator = _host.Services.GetRequiredService(); diff --git a/Azaion.Suite/config.system.json b/Azaion.Suite/config.system.json index c6f777f..c740a46 100644 --- a/Azaion.Suite/config.system.json +++ b/Azaion.Suite/config.system.json @@ -30,7 +30,8 @@ "TrackingDistanceConfidence": 0.15, "TrackingProbabilityIncrease": 15.0, - "TrackingIntersectionThreshold": 0.8, + "TrackingIntersectionThreshold": 0.6, + "BigImageTileOverlapPercent": 20, "ModelBatchSize": 4 },