add manual Tile Processor

zoom on video on pause (temp image)

.gitignore

@@ -23,3 +23,4 @@ azaion\.*\.big
 _internal
 *.spec
 dist
+*.jpg

Azaion.Annotator/Annotator.xaml.cs

@@ -56,6 +56,7 @@ public partial class Annotator
     public Dictionary<string, MediaFileInfo> MediaFilesDict = new();
 
     public IntervalTree<TimeSpan, Annotation> TimedAnnotations { get; set; } = new();
+    public string MainTitle { get; set; }
 
     public Annotator(
         IConfigUpdater configUpdater,
@@ -73,6 +74,8 @@ public partial class Annotator
     {
         InitializeComponent();
         
+        MainTitle = $"Azaion Annotator {Constants.GetLocalVersion()}";
+        Title = MainTitle;
         _appConfig = appConfig.Value;
         _configUpdater = configUpdater;
         _libVLC = libVLC;
@@ -194,7 +197,7 @@ public partial class Annotator
             _formState.CurrentMrl = _mediaPlayer.Media?.Mrl ?? "";
             uint vw = 0, vh = 0;
             _mediaPlayer.Size(0, ref vw, ref vh);
-            _formState.CurrentVideoSize = new Size(vw, vh);
+            _formState.CurrentMediaSize = new Size(vw, vh);
             _formState.CurrentVideoLength = TimeSpan.FromMilliseconds(_mediaPlayer.Length);
         };
 
@@ -289,27 +292,23 @@ public partial class Annotator
             StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}";
             Editor.ClearExpiredAnnotations(time);
         });
-
+        var annotation = TimedAnnotations.Query(time).FirstOrDefault();
-        ShowAnnotation(TimedAnnotations.Query(time).FirstOrDefault(), showImage);
+        if (annotation != null) ShowAnnotation(annotation, showImage);
     }
 
-    private void ShowAnnotation(Annotation? annotation, bool showImage = false)
+    private void ShowAnnotation(Annotation annotation, bool showImage = false)
     {
-        if (annotation == null)
-            return;
         Dispatcher.Invoke(async () =>
         {
-            var videoSize = _formState.CurrentVideoSize;
             if (showImage)
             {
                 if (File.Exists(annotation.ImagePath))
                 {
                     Editor.SetBackground(await annotation.ImagePath.OpenImage());
                     _formState.BackgroundTime = annotation.Time;
-                    videoSize = Editor.RenderSize;
                 }
             }
-            Editor.CreateDetections(annotation.Time, annotation.Detections, _appConfig.AnnotationConfig.DetectionClasses, videoSize);
+            Editor.CreateDetections(annotation.Time, annotation.Detections, _appConfig.AnnotationConfig.DetectionClasses, _formState.CurrentMediaSize);
         });
     }
 
@@ -321,7 +320,7 @@ public partial class Annotator
 
         var annotations = await _dbFactory.Run(async db =>
             await db.Annotations.LoadWith(x => x.Detections)
-                .Where(x => x.OriginalMediaName == _formState.VideoName)
+                .Where(x => x.OriginalMediaName == _formState.MediaName)
                 .OrderBy(x => x.Time)
                 .ToListAsync(token: MainCancellationSource.Token));
 
@@ -583,13 +582,11 @@ public partial class Annotator
     private void SoundDetections(object sender, RoutedEventArgs e)
     {
         MessageBox.Show("Функція Аудіоаналіз знаходиться в стадії розробки","Система", MessageBoxButton.OK, MessageBoxImage.Information);
-        _logger.LogInformation("Denys wishes #1. To be implemented");
     }
 
     private void RunDroneMaintenance(object sender, RoutedEventArgs e)
     {
         MessageBox.Show("Функція Аналіз стану БПЛА знаходиться в стадії розробки","Система", MessageBoxButton.OK, MessageBoxImage.Information);
-        _logger.LogInformation("Denys wishes #2. To be implemented");
     }
 
     #endregion

Azaion.Annotator/AnnotatorEventHandler.cs

@@ -2,6 +2,7 @@
 using System.Windows;
 using System.Windows.Input;
 using System.Windows.Media;
+using System.Windows.Media.Imaging;
 using Azaion.Annotator.Controls;
 using Azaion.Annotator.DTO;
 using Azaion.Common;
@@ -47,6 +48,7 @@ public class AnnotatorEventHandler(
     private const int STEP = 20;
     private const int LARGE_STEP = 5000;
     private const int RESULT_WIDTH = 1280;
+    private readonly string tempImgPath = Path.Combine(dirConfig.Value.ImagesDirectory, "___temp___.jpg"); 
     
     private readonly Dictionary<Key, PlaybackControlEnum> _keysControlEnumDict = new()
     {
@@ -139,12 +141,21 @@ public class AnnotatorEventHandler(
                     await Play(cancellationToken);
                     break;
                 case PlaybackControlEnum.Pause:
-                    mediaPlayer.Pause();
+                    if (mediaPlayer.IsPlaying) 
-
-                    if (formState.BackgroundTime.HasValue)
                     {
-                        mainWindow.Editor.SetBackground(null);
+                        mediaPlayer.Pause();
-                        formState.BackgroundTime = null;
+                        mediaPlayer.TakeSnapshot(0, tempImgPath, 0, 0);
+                        mainWindow.Editor.SetBackground(await tempImgPath.OpenImage());
+                        formState.BackgroundTime = TimeSpan.FromMilliseconds(mediaPlayer.Time);
+                    }
+                    else
+                    {
+                        mediaPlayer.Play();
+                        if (formState.BackgroundTime.HasValue)
+                        {
+                            mainWindow.Editor.SetBackground(null);
+                            formState.BackgroundTime = null;
+                        }   
                     }
                     break;
                 case PlaybackControlEnum.Stop:
@@ -159,7 +170,7 @@ public class AnnotatorEventHandler(
                     mainWindow.SeekTo(mediaPlayer.Time + step);
                     break;
                 case PlaybackControlEnum.SaveAnnotations:
-                    await SaveAnnotations(cancellationToken);
+                    await SaveAnnotation(cancellationToken);
                     break;
                 case PlaybackControlEnum.RemoveSelectedAnns:
 
@@ -226,63 +237,125 @@ public class AnnotatorEventHandler(
         if (mainWindow.LvFiles.SelectedItem == null)
             return;
         var mediaInfo = (MediaFileInfo)mainWindow.LvFiles.SelectedItem;
-        mainWindow.Editor.SetBackground(null);
         
         formState.CurrentMedia = mediaInfo;
-        mainWindow.Title = $"Azaion Annotator - {mediaInfo.Name}";
+        mainWindow.Title =  $"{mainWindow.MainTitle} - {mediaInfo.Name}";
 
-        
+        if (mediaInfo.MediaType == MediaTypes.Video)
-        //need to wait a bit for correct VLC playback event handling
+        {
-        await Task.Delay(100, ct);
+            mainWindow.Editor.SetBackground(null);
-        mediaPlayer.Stop();
+            //need to wait a bit for correct VLC playback event handling
-        mediaPlayer.Play(new Media(libVLC, mediaInfo.Path));
+            await Task.Delay(100, ct);
+            mediaPlayer.Stop();
+            mediaPlayer.Play(new Media(libVLC, mediaInfo.Path));   
+        }
+        else
+        {
+            formState.BackgroundTime = TimeSpan.Zero;
+            var image = await mediaInfo.Path.OpenImage();
+            formState.CurrentMediaSize = new Size(image.PixelWidth, image.PixelHeight);
+            mainWindow.Editor.SetBackground(image);
+            mediaPlayer.Stop();
+        }
     }
 
     //SAVE: MANUAL
-    private async Task SaveAnnotations(CancellationToken cancellationToken = default)
+    private async Task SaveAnnotation(CancellationToken cancellationToken = default)
     {
         if (formState.CurrentMedia == null)
             return;
 
         var time = formState.BackgroundTime ?? TimeSpan.FromMilliseconds(mediaPlayer.Time);
-        var originalMediaName = formState.VideoName;
+        var timeName = formState.MediaName.ToTimeName(time);
-        var fName = originalMediaName.ToTimeName(time);
-
-        var currentDetections = mainWindow.Editor.CurrentDetections
-            .Select(x => new Detection(fName, x.GetLabel(mainWindow.Editor.RenderSize, formState.BackgroundTime.HasValue ? mainWindow.Editor.RenderSize : formState.CurrentVideoSize)))
-            .ToList();
-
-        formState.CurrentMedia.HasAnnotations = currentDetections.Count != 0;
-        mainWindow.LvFiles.Items.Refresh();
-        mainWindow.Editor.RemoveAllAnns();
-
         var isVideo = formState.CurrentMedia.MediaType == MediaTypes.Video;
-        var imgPath = Path.Combine(dirConfig.Value.ImagesDirectory, $"{fName}{Constants.JPG_EXT}");
+        var imgPath = Path.Combine(dirConfig.Value.ImagesDirectory, $"{timeName}{Constants.JPG_EXT}");
 
-        if (formState.BackgroundTime.HasValue)
+        formState.CurrentMedia.HasAnnotations = mainWindow.Editor.CurrentDetections.Count != 0;
+        var annotations = await SaveAnnotationInner(imgPath, cancellationToken);
+        if (isVideo)
         {
-            //no need to save image, it's already there, just remove background
+            foreach (var annotation in annotations) 
-            mainWindow.Editor.SetBackground(null);
+                mainWindow.AddAnnotation(annotation);
-            formState.BackgroundTime = null;
+            mediaPlayer.Play();
             
-            //next item
+            // next item. Probably not needed
-            var annGrid = mainWindow.DgAnnotations;
+            // var annGrid = mainWindow.DgAnnotations;
-            annGrid.SelectedIndex = Math.Min(annGrid.Items.Count, annGrid.SelectedIndex + 1);
+            // annGrid.SelectedIndex = Math.Min(annGrid.Items.Count, annGrid.SelectedIndex + 1);
-            mainWindow.OpenAnnotationResult((AnnotationResult)annGrid.SelectedItem);
+            // mainWindow.OpenAnnotationResult((AnnotationResult)annGrid.SelectedItem);
         }
         else
         {
-            var resultHeight = (uint)Math.Round(RESULT_WIDTH / formState.CurrentVideoSize.Width * formState.CurrentVideoSize.Height);
+            await NextMedia(ct: cancellationToken);
-            mediaPlayer.TakeSnapshot(0, imgPath, RESULT_WIDTH, resultHeight);
+        }
-            if (isVideo)
+        mainWindow.Editor.SetBackground(null);
-                mediaPlayer.Play();
+        formState.BackgroundTime = null;
+        
+        mainWindow.LvFiles.Items.Refresh();
+        mainWindow.Editor.RemoveAllAnns();
+    }
+
+    private async Task<List<Annotation>> SaveAnnotationInner(string imgPath, CancellationToken cancellationToken = default)
+    {
+        var canvasDetections = mainWindow.Editor.CurrentDetections.Select(x => x.ToCanvasLabel()).ToList();
+        var annotationsResult = new List<Annotation>();
+        if (!File.Exists(imgPath))
+        {
+            var source = (mainWindow.Editor.BackgroundImage.Source as BitmapSource)!;
+            if (source.PixelWidth <= RESULT_WIDTH * 2 && source.PixelHeight <= RESULT_WIDTH * 2) // Allow to be up to 2560*2560 to save to 1280*1280
+            {
+                //Save image
+                await using var stream = new FileStream(imgPath, FileMode.Create);
+                var encoder = new JpegBitmapEncoder();
+                encoder.Frames.Add(BitmapFrame.Create(source));
+                encoder.Save(stream);
+                await stream.FlushAsync(cancellationToken);
+            }
             else
-                await NextMedia(ct: cancellationToken);
+            {
+                //Tiling
+                
+                //1. Restore original picture coordinates
+                var pictureCoordinatesDetections = canvasDetections.Select(x => new CanvasLabel(
+                    new YoloLabel(x, mainWindow.Editor.RenderSize, formState.CurrentMediaSize), formState.CurrentMediaSize, null, x.Confidence))
+                    .ToList();
+                
+                //2. Split to 1280*1280 frames
+                var results = TileProcessor.Split(formState.CurrentMediaSize, pictureCoordinatesDetections, cancellationToken);
+                
+                //3. Save each frame as a separate annotation
+                BitmapEncoder tileEncoder = new JpegBitmapEncoder();
+                foreach (var res in results)
+                {
+                    var mediaName = $"{formState.MediaName}!split!{res.Tile.X}_{res.Tile.Y}!";
+                    var time = TimeSpan.Zero;
+                    var annotationName = mediaName.ToTimeName(time);
+                    
+                    var tileImgPath = Path.Combine(dirConfig.Value.ImagesDirectory, $"{annotationName}{Constants.JPG_EXT}");
+                    await using var tileStream = new FileStream(tileImgPath, FileMode.Create);
+                    var bitmap = new CroppedBitmap(source, new Int32Rect((int)res.Tile.X, (int)res.Tile.Y, (int)res.Tile.Width, (int)res.Tile.Height));
+                    tileEncoder.Frames.Add(BitmapFrame.Create(bitmap));
+                    tileEncoder.Save(tileStream);
+                    await tileStream.FlushAsync(cancellationToken);
+                    
+                    var frameSize = new Size(res.Tile.Width, res.Tile.Height);
+                    var detections = res.Detections
+                        .Select(det => det.ReframeToSmall(res.Tile))
+                        .Select(x => new Detection(annotationName, new YoloLabel(x, frameSize)))
+                        .ToList();
+
+                    annotationsResult.Add(await annotationService.SaveAnnotation(mediaName, time, detections, token: cancellationToken));
+                }
+                return annotationsResult;
+            }
         }
         
-        var annotation = await annotationService.SaveAnnotation(originalMediaName, time, currentDetections, token: cancellationToken);
+        var timeImg = formState.BackgroundTime ?? TimeSpan.FromMilliseconds(mediaPlayer.Time);
-        if (isVideo)
+        var timeName = formState.MediaName.ToTimeName(timeImg);
-            mainWindow.AddAnnotation(annotation);
+        var currentDetections = canvasDetections.Select(x => 
+                new Detection(timeName, new YoloLabel(x, mainWindow.Editor.RenderSize)))
+            .ToList();
+        var annotation = await annotationService.SaveAnnotation(formState.MediaName, timeImg, currentDetections, token: cancellationToken);
+        return [annotation];
     }
 
     public async Task Handle(AnnotationsDeletedEvent notification, CancellationToken ct)
@@ -317,20 +390,19 @@ public class AnnotatorEventHandler(
 
             await dbFactory.DeleteAnnotations(notification.AnnotationNames, ct);
             
-            try
+            foreach (var name in notification.AnnotationNames)
             {
-                foreach (var name in notification.AnnotationNames)
+                try
                 {
                     File.Delete(Path.Combine(dirConfig.Value.ImagesDirectory, $"{name}{Constants.JPG_EXT}"));
                     File.Delete(Path.Combine(dirConfig.Value.LabelsDirectory, $"{name}{Constants.TXT_EXT}"));
                     File.Delete(Path.Combine(dirConfig.Value.ThumbnailsDirectory, $"{name}{Constants.THUMBNAIL_PREFIX}{Constants.JPG_EXT}"));
                     File.Delete(Path.Combine(dirConfig.Value.ResultsDirectory, $"{name}{Constants.RESULT_PREFIX}{Constants.JPG_EXT}"));
                 }
-            }
+                catch (Exception e)
-            catch (Exception e)
+                {
-            {
+                    logger.LogError(e, e.Message);
-                logger.LogError(e, e.Message);
+                }
-                throw;
             }
 
             //Only validators can send Delete to the queue

Azaion.Common/Constants.cs

@@ -1,4 +1,5 @@
-using System.IO;
+using System.Diagnostics;
+using System.IO;
 using Azaion.Common.DTO;
 using Azaion.Common.DTO.Config;
 using Azaion.Common.Extensions;
@@ -11,8 +12,9 @@ namespace Azaion.Common;
 public class Constants
 {
     public const string CONFIG_PATH = "config.json";
-
+    public const string LOADER_CONFIG_PATH = "loaderconfig.json";
-    private const string DEFAULT_API_URL = "https://api.azaion.com";
+    public const string DEFAULT_API_URL = "https://api.azaion.com";
+    public const string AZAION_SUITE_EXE = "Azaion.Suite.exe";
     
     #region ExternalClientsConfig
 
@@ -103,14 +105,16 @@ public class Constants
         TrackingDistanceConfidence    = TRACKING_DISTANCE_CONFIDENCE,
         TrackingProbabilityIncrease   = TRACKING_PROBABILITY_INCREASE,
         TrackingIntersectionThreshold = TRACKING_INTERSECTION_THRESHOLD,
+        BigImageTileOverlapPercent    = DEFAULT_BIG_IMAGE_TILE_OVERLAP_PERCENT, 
         FramePeriodRecognition        = DEFAULT_FRAME_PERIOD_RECOGNITION
     };
 
-    public const double DEFAULT_FRAME_RECOGNITION_SECONDS = 2;
+    public const double DEFAULT_FRAME_RECOGNITION_SECONDS      = 2;
-    public const double TRACKING_DISTANCE_CONFIDENCE      = 0.15;
+    public const double TRACKING_DISTANCE_CONFIDENCE           = 0.15;
-    public const double TRACKING_PROBABILITY_INCREASE     = 15;
+    public const double TRACKING_PROBABILITY_INCREASE          = 15;
-    public const double TRACKING_INTERSECTION_THRESHOLD   = 0.8;
+    public const double TRACKING_INTERSECTION_THRESHOLD        = 0.8;
-    public const int    DEFAULT_FRAME_PERIOD_RECOGNITION  = 4;
+    public const int    DEFAULT_BIG_IMAGE_TILE_OVERLAP_PERCENT = 20;
+    public const int    DEFAULT_FRAME_PERIOD_RECOGNITION       = 4;
 
     # endregion AIRecognitionConfig
 
@@ -251,4 +255,12 @@ public class Constants
             return DefaultInitConfig;
         }
     }
+    
+    public static Version GetLocalVersion()
+    {
+        var localFileInfo = FileVersionInfo.GetVersionInfo(AZAION_SUITE_EXE);
+        if (string.IsNullOrWhiteSpace(localFileInfo.ProductVersion))
+            throw new Exception($"Can't find {AZAION_SUITE_EXE} and its version");
+        return new Version(localFileInfo.FileVersion!);
+    }
 }

Azaion.Common/Controls/CanvasEditor.cs

@@ -6,6 +6,7 @@ using System.Windows.Media;
 using System.Windows.Media.Imaging;
 using System.Windows.Shapes;
 using Azaion.Common.DTO;
+using Azaion.Common.Events;
 using MediatR;
 using Color = System.Windows.Media.Color;
 using Image = System.Windows.Controls.Image;
@@ -34,10 +35,10 @@ public class CanvasEditor : Canvas
     private Point _panStartPoint;
     private bool _isZoomedIn;
 
-    private const int MIN_SIZE = 20;
+    private const int MIN_SIZE = 12;
     private readonly TimeSpan _viewThreshold = TimeSpan.FromMilliseconds(400);
 
-    private Image _backgroundImage { get; set; } = new() { Stretch = Stretch.Fill };
+    public Image BackgroundImage { get; set; } = new() { Stretch = Stretch.Uniform };
     public IMediator Mediator { get; set; } = null!;
 
     public static readonly DependencyProperty GetTimeFuncProp =
@@ -113,7 +114,7 @@ public class CanvasEditor : Canvas
         MouseUp += CanvasMouseUp;
         SizeChanged += CanvasResized;
         Cursor = Cursors.Cross;
-        Children.Insert(0, _backgroundImage);
+        Children.Insert(0, BackgroundImage);
         Children.Add(_newAnnotationRect);
         Children.Add(_horizontalLine);
         Children.Add(_verticalLine);
@@ -127,7 +128,7 @@ public class CanvasEditor : Canvas
     public void SetBackground(ImageSource? source)
     {
         SetZoom();
-        _backgroundImage.Source = source;
+        BackgroundImage.Source = source;
     }
 
     private void SetZoom(Matrix? matrix = null)
@@ -142,8 +143,8 @@ public class CanvasEditor : Canvas
             _matrixTransform.Matrix = matrix.Value;
             _isZoomedIn = true;
         }
-        foreach (var detection in CurrentDetections) 
+        // foreach (var detection in CurrentDetections) 
-            detection.UpdateAdornerScale(scale: _matrixTransform.Matrix.M11);
+        //     detection.UpdateAdornerScale(scale: _matrixTransform.Matrix.M11);
     }
     
     private void CanvasWheel(object sender, MouseWheelEventArgs e)
@@ -175,6 +176,8 @@ public class CanvasEditor : Canvas
     private void CanvasMouseDown(object sender, MouseButtonEventArgs e)
     {
         ClearSelections();
+        if (e.LeftButton != MouseButtonState.Pressed)
+            return;
         if (Keyboard.Modifiers == ModifierKeys.Control && _isZoomedIn)
         {
             _panStartPoint = e.GetPosition(this);
@@ -182,11 +185,13 @@ public class CanvasEditor : Canvas
         }
         else
             NewAnnotationStart(sender, e);
+        (sender as UIElement)?.CaptureMouse();
     }
 
     private void CanvasMouseMove(object sender, MouseEventArgs e)
     {
         var pos = e.GetPosition(this);
+        Mediator.Publish(new SetStatusTextEvent($"Mouse Coordinates: {pos.X}, {pos.Y}"));
         _horizontalLine.Y1 = _horizontalLine.Y2 = pos.Y;
         _verticalLine.X1 = _verticalLine.X2 = pos.X;
         SetLeft(_classNameHint, pos.X + 10);
@@ -216,11 +221,14 @@ public class CanvasEditor : Canvas
 
         var matrix = _matrixTransform.Matrix;
         matrix.Translate(delta.X, delta.Y);
+        
         _matrixTransform.Matrix = matrix;
+        Mediator.Publish(new SetStatusTextEvent(_matrixTransform.Matrix.ToString()));
     }
 
     private void CanvasMouseUp(object sender, MouseButtonEventArgs e)
     {
+        (sender as UIElement)?.ReleaseMouseCapture();
         if (SelectionState == SelectionState.NewAnnCreating)
         {
             var endPos = e.GetPosition(this);
@@ -279,8 +287,8 @@ public class CanvasEditor : Canvas
     {
         _horizontalLine.X2 = e.NewSize.Width;
         _verticalLine.Y2 = e.NewSize.Height;
-        _backgroundImage.Width = e.NewSize.Width;
+        BackgroundImage.Width = e.NewSize.Width;
-        _backgroundImage.Height = e.NewSize.Height;
+        BackgroundImage.Height = e.NewSize.Height;
     }
     
     #region Annotation Resizing & Moving
@@ -383,7 +391,6 @@ public class CanvasEditor : Canvas
     private void NewAnnotationStart(object sender, MouseButtonEventArgs e)
     {
         _newAnnotationStartPos = e.GetPosition(this);
-        
         SetLeft(_newAnnotationRect, _newAnnotationStartPos.X);
         SetTop(_newAnnotationRect, _newAnnotationStartPos.Y);
         _newAnnotationRect.MouseMove += NewAnnotationCreatingMove;

Azaion.Common/Controls/DetectionControl.cs

@@ -12,15 +12,15 @@ namespace Azaion.Common.Controls;
 public class DetectionControl : Border
 {
     private readonly Action<object, MouseButtonEventArgs> _resizeStart;
-    private const double RESIZE_RECT_SIZE = 12;
+    private const double RESIZE_RECT_SIZE = 10;
 
     private readonly Grid _grid;
-    private readonly Label _detectionLabel;
+    private readonly DetectionLabelPanel _detectionLabelPanel;
+    //private readonly Label _detectionLabel;
     public readonly Canvas DetectionLabelContainer;
 
     public TimeSpan Time { get; set; }
-    private readonly double _confidence;
+    private readonly List<Rectangle> _resizedRectangles = new();
-    private List<Rectangle> _resizedRectangles = new();
 
     private DetectionClass _detectionClass = null!;
     public DetectionClass DetectionClass
@@ -34,8 +34,7 @@ public class DetectionControl : Border
             foreach (var rect in _resizedRectangles)
                 rect.Stroke = brush;
             
-            _detectionLabel.Background = new SolidColorBrush(value.Color.ToConfidenceColor(_confidence));
+            _detectionLabelPanel.DetectionClass = value;
-            _detectionLabel.Content = _detectionLabelText(value.UIName);
             _detectionClass = value;
         }
     }
@@ -79,9 +78,6 @@ public class DetectionControl : Border
         }
     }
     
-    private string _detectionLabelText(string detectionClassName) =>
-        _confidence >= 0.995 ? detectionClassName : $"{detectionClassName}: {_confidence * 100:F0}%"; //double
-
     public DetectionControl(DetectionClass detectionClass, TimeSpan time, Action<object,
         MouseButtonEventArgs> resizeStart, CanvasLabel canvasLabel)
     {
@@ -89,7 +85,6 @@ public class DetectionControl : Border
         Height = canvasLabel.Height;
         Time = time;
         _resizeStart = resizeStart;
-        _confidence = canvasLabel.Confidence;
         
         DetectionLabelContainer = new Canvas
         {
@@ -97,16 +92,16 @@ public class DetectionControl : Border
             VerticalAlignment = VerticalAlignment.Top,
             ClipToBounds = false,
         };
-        _detectionLabel = new Label
+        _detectionLabelPanel = new DetectionLabelPanel
         {
-            Content = _detectionLabelText(detectionClass.Name),
+            Confidence = canvasLabel.Confidence
-            FontSize = 16,
-            Visibility = Visibility.Visible
         };
-        DetectionLabelContainer.Children.Add(_detectionLabel);
+        
+        DetectionLabelContainer.Children.Add(_detectionLabelPanel);
 
         _selectionFrame = new Rectangle
         {
+            Margin = new Thickness(-3),
             HorizontalAlignment = HorizontalAlignment.Stretch,
             VerticalAlignment = VerticalAlignment.Stretch,
             Stroke = new SolidColorBrush(Colors.Black),
@@ -146,12 +141,13 @@ public class DetectionControl : Border
         var rect = new Rectangle()   // small rectangles at the corners and sides
         {
             ClipToBounds = false,
-            Margin = new Thickness(-RESIZE_RECT_SIZE * 0.7),
+            Margin = new Thickness(-RESIZE_RECT_SIZE),
             HorizontalAlignment = ha,
             VerticalAlignment = va,
             Width = RESIZE_RECT_SIZE,
             Height = RESIZE_RECT_SIZE,
             Stroke = new SolidColorBrush(Color.FromArgb(230, 20, 20, 20)), // small rectangles color
+            StrokeThickness = 0.8,
             Fill = new SolidColorBrush(Color.FromArgb(150, 80, 80, 80)),
             Cursor = crs,
             Name = name,
@@ -160,9 +156,9 @@ public class DetectionControl : Border
         return rect;
     }
 
-    public YoloLabel GetLabel(Size canvasSize, Size? videoSize = null)
+    public CanvasLabel ToCanvasLabel() => 
-    {
+        new(DetectionClass.YoloId, Canvas.GetLeft(this), Canvas.GetTop(this), Width, Height);
-        var label = new CanvasLabel(DetectionClass.YoloId, Canvas.GetLeft(this), Canvas.GetTop(this), Width, Height);
+
-        return new YoloLabel(label, canvasSize, videoSize);
+    public YoloLabel ToYoloLabel(Size canvasSize, Size? videoSize = null) => 
-    }
+        new(ToCanvasLabel(), canvasSize, videoSize);
 }

Azaion.Common/Controls/DetectionLabelPanel.xaml

@@ -0,0 +1,59 @@
+<UserControl x:Class="Azaion.Common.Controls.DetectionLabelPanel"
+             xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
+             xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+             xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
+             xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
+             mc:Ignorable="d">
+    
+    <UserControl.Resources>
+        <!-- Friendly (Light Blue Square) -->
+        <DrawingImage x:Key="Friendly">
+            <DrawingImage.Drawing>
+                <DrawingGroup ClipGeometry="M0,0 V320 H320 V0 H0 Z">
+                    <GeometryDrawing Brush="LightBlue" Geometry="M25,50 l150,0 0,100 -150,0 z">
+                        <GeometryDrawing.Pen>
+                            <Pen Brush="Black" Thickness="8"/>    
+                        </GeometryDrawing.Pen>
+                    </GeometryDrawing>
+                </DrawingGroup>
+            </DrawingImage.Drawing>
+        </DrawingImage>
+        
+        <!-- Hostile (Red Diamond) -->
+        <DrawingImage x:Key="Hostile">
+            <DrawingImage.Drawing>
+                <DrawingGroup ClipGeometry="M0,0 V320 H320 V0 H0 Z">
+                    <GeometryDrawing Brush="Red" Geometry="M 100,28 L172,100 100,172 28,100 100,28 Z">
+                        <GeometryDrawing.Pen>
+                            <Pen Brush="Black" Thickness="8"/>    
+                        </GeometryDrawing.Pen>
+                    </GeometryDrawing>
+                </DrawingGroup>
+            </DrawingImage.Drawing>
+        </DrawingImage>
+        
+        <!-- Unknown (Yellow Quatrefoil) -->
+        <DrawingImage x:Key="Unknown">
+            <DrawingImage.Drawing>
+                <DrawingGroup ClipGeometry="M0,0 V320 H320 V0 H0 Z">
+                    <GeometryDrawing Brush="Yellow" Geometry="M63,63 C63,20 137,20 137,63 C180,63 180,137 137,137 C137,180 
+                        63,180 63,137 C20,137 20,63 63,63 Z">
+                        <GeometryDrawing.Pen>
+                            <Pen Brush="Black" Thickness="8"/>    
+                        </GeometryDrawing.Pen>
+                    </GeometryDrawing>
+                </DrawingGroup>
+            </DrawingImage.Drawing>
+        </DrawingImage>
+    </UserControl.Resources>
+
+    <Grid>
+        <Grid.ColumnDefinitions>
+            <ColumnDefinition Width="28"></ColumnDefinition>
+            <ColumnDefinition Width="Auto"></ColumnDefinition>
+        </Grid.ColumnDefinitions>
+        <Image Grid.Column="0" x:Name="AffiliationImage">
+        </Image>
+        <Label Grid.Column="1" FontSize="16"></Label>
+    </Grid>
+</UserControl>

Azaion.Common/Controls/DetectionLabelPanel.xaml.cs

@@ -0,0 +1,55 @@
+using System.Windows.Media;
+using Azaion.Common.DTO;
+
+namespace Azaion.Common.Controls
+{
+    public partial class DetectionLabelPanel
+    {
+        private AffiliationEnum _affiliation = AffiliationEnum.None;
+        private double _confidence;
+        
+        public AffiliationEnum Affiliation
+        {
+            get => _affiliation;
+            set
+            {
+                _affiliation = value;
+                UpdateAffiliationImage();
+            }
+        }
+
+        public DetectionClass DetectionClass { get; set; }
+
+        public double Confidence
+        {
+            get => _confidence;
+            set
+            {
+                _confidence = value;
+                
+            }
+        }
+
+        public DetectionLabelPanel()
+        {
+            InitializeComponent();
+        }
+        
+        private string _detectionLabelText(string detectionClassName) =>
+            _confidence >= 0.98 ? detectionClassName : $"{detectionClassName}: {_confidence * 100:F0}%";
+        
+        private void UpdateAffiliationImage()
+        {
+            if (_affiliation == AffiliationEnum.None)
+            {
+                AffiliationImage.Source = null;
+                return;
+            }
+
+            if (TryFindResource(_affiliation.ToString()) is DrawingImage drawingImage)
+                AffiliationImage.Source = drawingImage;
+            else
+                AffiliationImage.Source = null;
+        }
+    }
+}

Azaion.Common/DTO/AffiliationEnum.cs

@@ -0,0 +1,9 @@
+namespace Azaion.Common.DTO;
+
+public enum AffiliationEnum
+{
+    None = 0,
+    Friendly = 10,
+    Hostile = 20,
+    Unknown = 30
+}

Azaion.Common/DTO/Config/AIRecognitionConfig.cs

@@ -12,6 +12,7 @@ public class AIRecognitionConfig
     [Key("t_dc")] public double TrackingDistanceConfidence { get; set; }
     [Key("t_pi")] public double TrackingProbabilityIncrease { get; set; }
     [Key("t_it")] public double TrackingIntersectionThreshold { get; set; }
+    [Key("ov_p")] public double BigImageTileOverlapPercent { get; set; }
 
     [Key("d")] public byte[] Data { get; set; } = null!;
     [Key("p")] public List<string> Paths { get; set; } = null!;

Azaion.Common/DTO/FormState.cs

@@ -6,10 +6,10 @@ namespace Azaion.Common.DTO;
 public class FormState
 {
     public MediaFileInfo? CurrentMedia { get; set; }
-    public string VideoName => CurrentMedia?.FName ?? "";
+    public string MediaName => CurrentMedia?.FName ?? "";
 
     public string CurrentMrl { get; set; } = null!;
-    public Size CurrentVideoSize { get; set; }
+    public Size CurrentMediaSize { get; set; }
     public TimeSpan CurrentVideoLength { get; set; }
 
     public TimeSpan? BackgroundTime { get; set; }

Azaion.Common/DTO/Label.cs

@@ -22,14 +22,34 @@ public abstract class Label
 
 public class CanvasLabel : Label
 {
-    public double X { get; set; }
+    public double X { get; set; } //left
-    public double Y { get; set; }
+    public double Y { get; set; } //top
     public double Width { get; set; }
     public double Height { get; set; }
     public double Confidence { get; set;  }
 
-    public CanvasLabel()
+    public double Bottom
     {
+        get => Y + Height;
+        set => Height = value - Y;
+    }
+
+    public double Right
+    {
+        get => X + Width;
+        set => Width = value - X;
+    }
+
+    public CanvasLabel() { }
+
+    public CanvasLabel(double left, double right, double top, double bottom)
+    {
+        X = left;
+        Y = top;
+        Width = right - left;
+        Height = bottom - top;
+        Confidence = 1;
+        ClassNumber = -1;
     }
     
     public CanvasLabel(int classNumber, double x, double y, double width, double height, double confidence = 1) : base(classNumber)
@@ -77,6 +97,13 @@ public class CanvasLabel : Label
         }
         Confidence = confidence;
     }
+
+    public CanvasLabel ReframeToSmall(CanvasLabel smallTile) => 
+        new(ClassNumber, X - smallTile.X, Y - smallTile.Y, Width, Height, Confidence);
+    
+    public CanvasLabel ReframeFromSmall(CanvasLabel smallTile) => 
+        new(ClassNumber, X + smallTile.X, Y + smallTile.Y, Width, Height, Confidence);
+    
 }
 
 [MessagePackObject]
@@ -193,13 +220,15 @@ public class Detection : YoloLabel
 {
     [JsonProperty(PropertyName = "an")][Key("an")] public string AnnotationName { get; set; } = null!;
     [JsonProperty(PropertyName = "p")][Key("p")]  public double  Confidence { get; set; }
+    [JsonProperty(PropertyName = "dn")][Key("dn")]  public string Description { get; set; }
     
     //For db & serialization
     public Detection(){}
 
-    public Detection(string annotationName, YoloLabel label, double confidence = 1)
+    public Detection(string annotationName, YoloLabel label, string description = "", double confidence = 1)
     {
         AnnotationName = annotationName;
+        Description = description;
         ClassNumber = label.ClassNumber;
         CenterX = label.CenterX;
         CenterY = label.CenterY;

Azaion.Common/Database/DbFactory.cs

@@ -62,6 +62,8 @@ public class DbFactory : IDbFactory
         RecreateTables();
         
         _fileConnection.Open();
+        using var db = new AnnotationsDb(_fileDataOptions);
+        SchemaMigrator.EnsureSchemaUpdated(db, typeof(Annotation), typeof(Detection));
         _fileConnection.BackupDatabase(_memoryConnection, "main", "main", -1, null, -1);
     }
 

Azaion.Common/Database/SchemaMigrator.cs

@@ -0,0 +1,94 @@
+using System.Data;
+using LinqToDB.Data;
+using LinqToDB.Mapping;
+
+namespace Azaion.Common.Database;
+
+public static class SchemaMigrator
+{
+    public static void EnsureSchemaUpdated(DataConnection dbConnection, params Type[] entityTypes)
+    {
+        var connection = dbConnection.Connection;
+        var mappingSchema = dbConnection.MappingSchema;
+
+        if (connection.State == ConnectionState.Closed)
+        {
+            connection.Open();
+        }
+
+        foreach (var type in entityTypes)
+        {
+            var entityDescriptor = mappingSchema.GetEntityDescriptor(type);
+            var tableName = entityDescriptor.Name.Name;
+            var existingColumns = GetTableColumns(connection, tableName);
+
+            foreach (var column in entityDescriptor.Columns)
+            {
+                if (existingColumns.Contains(column.ColumnName, StringComparer.OrdinalIgnoreCase)) 
+                    continue;
+                
+                var columnDefinition = GetColumnDefinition(column);
+                dbConnection.Execute($"ALTER TABLE {tableName} ADD COLUMN {columnDefinition}");
+            }
+        }
+    }
+
+    private static HashSet<string> GetTableColumns(IDbConnection connection, string tableName)
+    {
+        var columns = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+        using var cmd = connection.CreateCommand();
+        cmd.CommandText = $"PRAGMA table_info({tableName})";
+        using var reader = cmd.ExecuteReader();
+        while (reader.Read()) 
+            columns.Add(reader.GetString(1)); // "name" is in the second column
+
+        return columns;
+    }
+
+    private static string GetColumnDefinition(ColumnDescriptor column)
+    {
+        var type = column.MemberType;
+        var underlyingType = Nullable.GetUnderlyingType(type) ?? type;
+        var sqliteType = GetSqliteType(underlyingType);
+        var defaultClause = GetSqlDefaultValue(type, underlyingType);
+
+        return $"\"{column.ColumnName}\" {sqliteType} {defaultClause}";
+    }
+
+    private static string GetSqliteType(Type type) =>
+        type switch
+        {
+            _ when type == typeof(int) 
+                   || type == typeof(long) 
+                   || type == typeof(bool) 
+                   || type.IsEnum 
+                => "INTEGER",
+
+            _ when type == typeof(double) 
+                   || type == typeof(float) 
+                   || type == typeof(decimal) 
+                => "REAL",
+
+            _ when type == typeof(byte[]) 
+                => "BLOB",
+
+            _ => "TEXT"
+        };
+
+    private static string GetSqlDefaultValue(Type originalType, Type underlyingType)
+    {
+        var isNullable = originalType.IsClass || Nullable.GetUnderlyingType(originalType) != null;
+        if (isNullable)
+            return "NULL";
+
+        var defaultValue = Activator.CreateInstance(underlyingType);
+        
+        if (underlyingType == typeof(bool))
+            return $"NOT NULL DEFAULT {(Convert.ToBoolean(defaultValue) ? 1 : 0)}";
+        
+        if (underlyingType.IsValueType && defaultValue is IFormattable f)
+            return $"NOT NULL DEFAULT {f.ToString(null, System.Globalization.CultureInfo.InvariantCulture)}";
+        
+        return $"NOT NULL DEFAULT '{defaultValue}'";
+    }
+}

Azaion.Common/Services/TileProcessor.cs

@@ -0,0 +1,82 @@
+using System.Windows;
+using System.Windows.Media.Imaging;
+using Azaion.Common.DTO;
+
+namespace Azaion.Common.Services;
+
+public class TileResult
+{
+    public CanvasLabel Tile { get; set; }
+    public List<CanvasLabel> Detections { get; set; }
+
+    public TileResult(CanvasLabel tile, List<CanvasLabel> detections)
+    {
+        Tile = tile;
+        Detections = detections;
+    }
+}
+
+public static class TileProcessor
+{
+    private const int MaxTileWidth = 1280;
+    private const int MaxTileHeight = 1280;
+    private const int Border = 10;
+
+    public static List<TileResult> Split(Size originalSize, List<CanvasLabel> detections, CancellationToken cancellationToken)
+    {
+        var results = new List<TileResult>();
+        var processingDetectionList = new List<CanvasLabel>(detections);
+        
+        while (processingDetectionList.Count > 0 && !cancellationToken.IsCancellationRequested)
+        {
+            var topMostDetection = processingDetectionList
+                .OrderBy(d => d.Y)
+                .First();
+            
+            var result = GetDetectionsInTile(originalSize, topMostDetection, processingDetectionList);
+            processingDetectionList.RemoveAll(x => result.Detections.Contains(x));
+            results.Add(result);
+        }
+        return results;
+    }
+    
+    private static TileResult GetDetectionsInTile(Size originalSize, CanvasLabel startDet, List<CanvasLabel> allDetections)
+    {
+        var tile = new CanvasLabel(
+            left: Math.Max(startDet.X - Border, 0),
+            right: Math.Min(startDet.Right + Border, originalSize.Width),
+            top: Math.Max(startDet.Y - Border, 0),
+            bottom: Math.Min(startDet.Bottom + Border, originalSize.Height));
+        var selectedDetections = new List<CanvasLabel>{startDet};
+        
+        foreach (var det in allDetections)
+        {
+            if (det == startDet) 
+                continue;
+            
+            var commonTile = new CanvasLabel(
+                left: Math.Max(Math.Min(tile.X, det.X) - Border, 0),
+                right: Math.Min(Math.Max(tile.Right, det.Right) + Border, originalSize.Width),
+                top: Math.Max(Math.Min(tile.Y, det.Y) - Border, 0),
+                bottom: Math.Min(Math.Max(tile.Bottom, det.Bottom) + Border, originalSize.Height)
+            );
+
+            if (commonTile.Width > MaxTileWidth || commonTile.Height > MaxTileHeight) 
+                continue;
+            
+            tile = commonTile;
+            selectedDetections.Add(det);
+        }
+
+        //normalization, width and height should be at least half of 1280px
+        tile.Width = Math.Max(tile.Width, MaxTileWidth / 2.0);
+        tile.Height = Math.Max(tile.Height, MaxTileHeight / 2.0);
+        
+        //boundaries check after normalization
+        tile.Right = Math.Min(tile.Right, originalSize.Width);
+        tile.Bottom = Math.Min(tile.Bottom, originalSize.Height);
+        
+        return new TileResult(tile, selectedDetections);
+    }
+
+}

Azaion.Dataset/DatasetExplorerEventHandler.cs

@@ -67,7 +67,7 @@ public class DatasetExplorerEventHandler(
                 var a = datasetExplorer.CurrentAnnotation!.Annotation;
 
                 var detections = datasetExplorer.ExplorerEditor.CurrentDetections
-                    .Select(x => new Detection(a.Name, x.GetLabel(datasetExplorer.ExplorerEditor.RenderSize)))
+                    .Select(x => new Detection(a.Name, x.ToYoloLabel(datasetExplorer.ExplorerEditor.RenderSize)))
                     .ToList();
                 var index = datasetExplorer.ThumbnailsView.SelectedIndex;
                 var annotation = await annotationService.SaveAnnotation(a.OriginalMediaName, a.Time, detections, token: token);

Azaion.Inference/README.md

@@ -13,23 +13,14 @@ Results (file or annotations) is putted to the other queue, or the same socket,
 
 <h2>Installation</h2>
 
-Prepare correct onnx model from YOLO:
-```python
-from ultralytics import YOLO
-import netron
-
-model = YOLO("azaion.pt")
-model.export(format="onnx", imgsz=1280, nms=True, batch=4)
-netron.start('azaion.onnx')
-```    
-Read carefully about [export arguments](https://docs.ultralytics.com/modes/export/), you have to use nms=True, and batching with a proper batch size  
-
 <h3>Install libs</h3>
 https://www.python.org/downloads/
 
 Windows
 
 - [Install CUDA](https://developer.nvidia.com/cuda-12-1-0-download-archive)
+- [Install Visual Studio Build Tools 2019](https://visualstudio.microsoft.com/downloads/?q=build+tools)
+
 
 Linux
 ```
@@ -44,6 +35,17 @@ Linux
     nvcc --version
 ```
     
+Prepare correct onnx model from YOLO:
+```python
+from ultralytics import YOLO
+import netron
+
+model = YOLO("azaion.pt")
+model.export(format="onnx", imgsz=1280, nms=True, batch=4)
+netron.start('azaion.onnx')
+```    
+Read carefully about [export arguments](https://docs.ultralytics.com/modes/export/), you have to use nms=True, and batching with a proper batch size  
+
 
 <h3>Install dependencies</h3>
 1. Install python with max version 3.11. Pytorch for now supports 3.11 max  

Azaion.Inference/ai_config.pxd

@@ -7,6 +7,8 @@ cdef class AIRecognitionConfig:
     cdef public double tracking_probability_increase
     cdef public double tracking_intersection_threshold
 
+    cdef public int big_image_tile_overlap_percent
+
     cdef public bytes file_data
     cdef public list[str] paths
     cdef public int model_batch_size

Azaion.Inference/ai_config.pyx

@@ -9,6 +9,7 @@ cdef class AIRecognitionConfig:
                  tracking_distance_confidence,
                  tracking_probability_increase,
                  tracking_intersection_threshold,
+                 big_image_tile_overlap_percent,
 
                  file_data,
                  paths,
@@ -21,6 +22,7 @@ cdef class AIRecognitionConfig:
         self.tracking_distance_confidence = tracking_distance_confidence
         self.tracking_probability_increase = tracking_probability_increase
         self.tracking_intersection_threshold = tracking_intersection_threshold
+        self.big_image_tile_overlap_percent = big_image_tile_overlap_percent
 
         self.file_data = file_data
         self.paths = paths
@@ -31,6 +33,7 @@ cdef class AIRecognitionConfig:
                 f'probability_increase : {self.tracking_probability_increase}, '
                 f'intersection_threshold : {self.tracking_intersection_threshold}, '
                 f'frame_period_recognition : {self.frame_period_recognition}, '
+                f'big_image_tile_overlap_percent: {self.big_image_tile_overlap_percent}, '
                 f'paths: {self.paths}, '
                 f'model_batch_size: {self.model_batch_size}')
 
@@ -45,6 +48,7 @@ cdef class AIRecognitionConfig:
             unpacked.get("t_dc", 0.0),
             unpacked.get("t_pi", 0.0),
             unpacked.get("t_it", 0.0),
+            unpacked.get("ov_p", 20),
 
             unpacked.get("d", b''),
             unpacked.get("p", []),

Azaion.Inference/annotation.pxd

@@ -3,7 +3,7 @@ cdef class Detection:
     cdef public str annotation_name
     cdef public int cls
 
-    cdef public overlaps(self, Detection det2)
+    cdef public overlaps(self, Detection det2, float confidence_threshold)
 
 cdef class Annotation:
     cdef public str name

Azaion.Inference/annotation.pyx

@@ -14,13 +14,13 @@ cdef class Detection:
     def __str__(self):
         return f'{self.cls}: {self.x:.2f} {self.y:.2f} {self.w:.2f} {self.h:.2f}, prob: {(self.confidence*100):.1f}%'
 
-    cdef overlaps(self, Detection det2):
+    cdef overlaps(self, Detection det2, float confidence_threshold):
         cdef double overlap_x = 0.5 * (self.w + det2.w) - abs(self.x - det2.x)
         cdef double overlap_y = 0.5 * (self.h + det2.h) - abs(self.y - det2.y)
         cdef double overlap_area = max(0.0, overlap_x) * max(0.0, overlap_y)
         cdef double min_area = min(self.w * self.h, det2.w * det2.h)
 
-        return overlap_area / min_area > 0.6
+        return overlap_area / min_area > confidence_threshold
 
 cdef class Annotation:
     def __init__(self, str name, long ms, list[Detection] detections):

Azaion.Inference/inference.pxd

@@ -23,11 +23,13 @@ cdef class Inference:
 
     cdef run_inference(self, RemoteCommand cmd)
     cdef _process_video(self, RemoteCommand cmd, AIRecognitionConfig ai_config, str video_name)
-    cdef _process_images(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list[str] image_paths)
+    cpdef _process_images(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list[str] image_paths)
+    cpdef _process_images_inner(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list frame_data)
+    cpdef split_to_tiles(self, frame, path, img_w, img_h, overlap_percent)
     cdef stop(self)
 
     cdef preprocess(self, frames)
-    cdef remove_overlapping_detections(self, list[Detection] detections)
+    cdef remove_overlapping_detections(self, list[Detection] detections, float confidence_threshold=?)
     cdef postprocess(self, output, ai_config)
     cdef split_list_extend(self, lst, chunk_size)
 

Azaion.Inference/inference.pyx

@@ -150,13 +150,13 @@ cdef class Inference:
                     h = y2 - y1
                     if conf >= ai_config.probability_threshold:
                         detections.append(Detection(x, y, w, h, class_id, conf))
-                filtered_detections = self.remove_overlapping_detections(detections)
+                filtered_detections = self.remove_overlapping_detections(detections, ai_config.tracking_intersection_threshold)
                 results.append(filtered_detections)
             return results
         except Exception as e:
             raise RuntimeError(f"Failed to postprocess: {str(e)}")
 
-    cdef remove_overlapping_detections(self, list[Detection] detections):
+    cdef remove_overlapping_detections(self, list[Detection] detections, float confidence_threshold=0.6):
         cdef Detection det1, det2
         filtered_output = []
         filtered_out_indexes = []
@@ -168,7 +168,7 @@ cdef class Inference:
             res = det1_index
             for det2_index in range(det1_index + 1, len(detections)):
                 det2 = detections[det2_index]
-                if det1.overlaps(det2):
+                if det1.overlaps(det2, confidence_threshold):
                     if det1.confidence > det2.confidence or (
                             det1.confidence == det2.confidence and det1.cls < det2.cls):  # det1 has higher confidence or lower class_id
                         filtered_out_indexes.append(det2_index)
@@ -211,9 +211,8 @@ cdef class Inference:
                 images.append(m)
         # images first, it's faster
         if len(images) > 0:
-            for chunk in self.split_list_extend(images, self.engine.get_batch_size()):
+            constants_inf.log(f'run inference on {" ".join(images)}...')
-                constants_inf.log(f'run inference on {" ".join(chunk)}...')
+            self._process_images(cmd, ai_config, images)
-                self._process_images(cmd, ai_config, chunk)
         if len(videos) > 0:
             for v in videos:
                 constants_inf.log(f'run inference on {v}...')
@@ -250,8 +249,6 @@ cdef class Inference:
                         _, image = cv2.imencode('.jpg', batch_frames[i])
                         annotation.image = image.tobytes()
                         self._previous_annotation = annotation
-
-                        print(annotation)
                         self.on_annotation(cmd, annotation)
 
                 batch_frames.clear()
@@ -259,15 +256,53 @@ cdef class Inference:
         v_input.release()
 
 
-    cdef _process_images(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list[str] image_paths):
+    cpdef _process_images(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list[str] image_paths):
-        cdef list frames = []
+        cdef list frame_data = []
-        cdef list timestamps = []
+        for path in image_paths:
-        self._previous_annotation = None
+            frame = cv2.imread(<str>path)
-        for image in image_paths:
+            if frame is None:
-            frame = cv2.imread(image)
+                constants_inf.logerror(<str>f'Failed to read image {path}')
-            frames.append(frame)
+                continue
-            timestamps.append(0)
+            img_h, img_w, _ = frame.shape
+            if img_h <= 1.5 * self.model_height and img_w <= 1.5 * self.model_width:
+                frame_data.append((frame, path))
+            else:
+                (split_frames, split_pats) = self.split_to_tiles(frame, path, img_w, img_h, ai_config.big_image_tile_overlap_percent)
+                frame_data.extend(zip(split_frames, split_pats))
 
+        for chunk in self.split_list_extend(frame_data, self.engine.get_batch_size()):
+            self._process_images_inner(cmd, ai_config, chunk)
+
+
+    cpdef split_to_tiles(self, frame, path, img_w, img_h, overlap_percent):
+        stride_w = self.model_width * (1 - overlap_percent / 100)
+        stride_h = self.model_height * (1 - overlap_percent / 100)
+        n_tiles_x = int(np.ceil((img_w - self.model_width) / stride_w)) + 1
+        n_tiles_y = int(np.ceil((img_h - self.model_height) / stride_h)) + 1
+
+        results = []
+        for y_idx in range(n_tiles_y):
+            for x_idx in range(n_tiles_x):
+                y_start = y_idx * stride_w
+                x_start = x_idx * stride_h
+
+                # Ensure the tile doesn't go out of bounds
+                y_end = min(y_start + self.model_width, img_h)
+                x_end = min(x_start + self.model_height, img_w)
+
+                # We need to re-calculate start if we are at the edge to get a full 1280x1280 tile
+                if y_end == img_h:
+                    y_start = img_h - self.model_height
+                if x_end == img_w:
+                    x_start = img_w - self.model_width
+
+                tile = frame[y_start:y_end, x_start:x_end]
+                name = path.stem + f'.tile_{x_start}_{y_start}' + path.suffix
+                results.append((tile, name))
+        return results
+
+    cpdef _process_images_inner(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list frame_data):
+        frames = [frame for frame, _ in frame_data]
         input_blob = self.preprocess(frames)
 
         outputs = self.engine.run(input_blob)
@@ -275,7 +310,7 @@ cdef class Inference:
         list_detections = self.postprocess(outputs, ai_config)
         for i in range(len(list_detections)):
             detections = list_detections[i]
-            annotation = Annotation(image_paths[i], timestamps[i], detections)
+            annotation = Annotation(frame_data[i][1], 0, detections)
             _, image = cv2.imencode('.jpg', frames[i])
             annotation.image = image.tobytes()
             self.on_annotation(cmd, annotation)
@@ -322,7 +357,9 @@ cdef class Inference:
                     closest_det = prev_det
 
             # Check if beyond tracking distance
-            if min_distance_sq > ai_config.tracking_distance_confidence:
+            dist_px = ai_config.tracking_distance_confidence * self.model_width
+            dist_px_sq = dist_px * dist_px
+            if min_distance_sq > dist_px_sq:
                 return True
 
             # Check probability increase

Azaion.Inference/requirements.txt

@@ -7,11 +7,12 @@ cryptography==44.0.2
 psutil
 msgpack
 pyjwt
-zmq
+pyzmq
 requests
 pyyaml
 pycuda
-tensorrt
+tensorrt==10.11.0.33
 pynvml
 boto3
 loguru
+pytest

Azaion.Inference/setup.py

@@ -2,19 +2,30 @@ from setuptools import setup, Extension
 from Cython.Build import cythonize
 import numpy as np
 
+# debug_args = {}
+# trace_line = False
+
+debug_args = {
+    'extra_compile_args': ['-O0', '-g'],
+    'extra_link_args': ['-g'],
+    'define_macros': [('CYTHON_TRACE_NOGIL', '1')]
+}
+trace_line = True
+
 extensions = [
-    Extension('constants_inf', ['constants_inf.pyx']),
+    Extension('constants_inf', ['constants_inf.pyx'], **debug_args),
-    Extension('file_data', ['file_data.pyx']),
+    Extension('file_data', ['file_data.pyx'], **debug_args),
-    Extension('remote_command_inf', ['remote_command_inf.pyx']),
+    Extension('remote_command_inf', ['remote_command_inf.pyx'], **debug_args),
-    Extension('remote_command_handler_inf', ['remote_command_handler_inf.pyx']),
+    Extension('remote_command_handler_inf', ['remote_command_handler_inf.pyx'], **debug_args),
-    Extension('annotation', ['annotation.pyx']),
+    Extension('annotation', ['annotation.pyx'], **debug_args),
-    Extension('loader_client', ['loader_client.pyx']),
+    Extension('loader_client', ['loader_client.pyx'], **debug_args),
-    Extension('ai_config', ['ai_config.pyx']),
+    Extension('ai_config', ['ai_config.pyx'], **debug_args),
-    Extension('tensorrt_engine', ['tensorrt_engine.pyx'], include_dirs=[np.get_include()]),
+    Extension('tensorrt_engine', ['tensorrt_engine.pyx'], include_dirs=[np.get_include()], **debug_args),
-    Extension('onnx_engine', ['onnx_engine.pyx'], include_dirs=[np.get_include()]),
+    Extension('onnx_engine', ['onnx_engine.pyx'], include_dirs=[np.get_include()], **debug_args),
-    Extension('inference_engine', ['inference_engine.pyx'], include_dirs=[np.get_include()]),
+    Extension('inference_engine', ['inference_engine.pyx'], include_dirs=[np.get_include()], **debug_args),
-    Extension('inference', ['inference.pyx'], include_dirs=[np.get_include()]),
+    Extension('inference', ['inference.pyx'], include_dirs=[np.get_include()], **debug_args),
-    Extension('main_inference', ['main_inference.pyx']),
+    Extension('main_inference', ['main_inference.pyx'], **debug_args),
+
 ]
 
 setup(
@@ -23,10 +34,11 @@ setup(
         extensions,
         compiler_directives={
             "language_level": 3,
-            "emit_code_comments" : False,
+            "emit_code_comments": False,
             "binding": True,
             'boundscheck': False,
-            'wraparound': False
+            'wraparound': False,
+            'linetrace': trace_line
         }
     ),
     install_requires=[

Azaion.Inference/setup_old.py

@@ -0,0 +1,37 @@
+from setuptools import setup, Extension
+from Cython.Build import cythonize
+import numpy as np
+
+extensions = [
+    Extension('constants_inf', ['constants_inf.pyx']),
+    Extension('file_data', ['file_data.pyx']),
+    Extension('remote_command_inf', ['remote_command_inf.pyx']),
+    Extension('remote_command_handler_inf', ['remote_command_handler_inf.pyx']),
+    Extension('annotation', ['annotation.pyx']),
+    Extension('loader_client', ['loader_client.pyx']),
+    Extension('ai_config', ['ai_config.pyx']),
+    Extension('tensorrt_engine', ['tensorrt_engine.pyx'], include_dirs=[np.get_include()]),
+    Extension('onnx_engine', ['onnx_engine.pyx'], include_dirs=[np.get_include()]),
+    Extension('inference_engine', ['inference_engine.pyx'], include_dirs=[np.get_include()]),
+    Extension('inference', ['inference.pyx'], include_dirs=[np.get_include()]),
+    Extension('main_inference', ['main_inference.pyx'])
+]
+
+setup(
+    name="azaion.ai",
+    ext_modules=cythonize(
+        extensions,
+        compiler_directives={
+            "language_level": 3,
+            "emit_code_comments" : False,
+            "binding": True,
+            'boundscheck': False,
+            'wraparound': False
+        }
+    ),
+    install_requires=[
+        'ultralytics>=8.0.0',
+        'pywin32; platform_system=="Windows"'
+    ],
+    zip_safe=False
+)

Azaion.Inference/test/test_inference.py

@@ -0,0 +1,8 @@
+import inference
+from ai_config import AIRecognitionConfig
+from remote_command_inf import RemoteCommand
+
+
+def test_process_images():
+    inf = inference.Inference(None, None)
+    inf._process_images(RemoteCommand(30), AIRecognitionConfig(4, 2, 15,  0.15, 15, 0.8, 20, b'test', [], 4), ['test_img01.JPG', 'test_img02.jpg'])

Azaion.Loader/requirements.txt

@@ -3,7 +3,7 @@ Cython
 psutil
 msgpack
 pyjwt
-zmq
+pyzmq
 requests
 pyyaml
 boto3

Azaion.LoaderUI/App.xaml.cs

@@ -1,4 +1,5 @@
 using System.Windows;
+using Azaion.Common;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
@@ -28,7 +29,7 @@ public partial class App
         var host = Host.CreateDefaultBuilder()
             .ConfigureAppConfiguration((_, config) => config
                 .AddCommandLine(Environment.GetCommandLineArgs())
-                .AddJsonFile(Constants.CONFIG_JSON_FILE, optional: true))
+                .AddJsonFile(Constants.LOADER_CONFIG_PATH, optional: true))
             .UseSerilog()
             .ConfigureServices((context, services) =>
             {
@@ -36,7 +37,7 @@ public partial class App
                 services.Configure<DirectoriesConfig>(context.Configuration.GetSection(nameof(DirectoriesConfig)));
                 services.AddHttpClient<IAzaionApi, AzaionApi>((sp, client) =>
                 {
-                    client.BaseAddress = new Uri(Constants.API_URL);
+                    client.BaseAddress = new Uri(Constants.DEFAULT_API_URL);
                     client.DefaultRequestHeaders.Add("Accept", "application/json");
                     client.DefaultRequestHeaders.Add("User-Agent", "Azaion.LoaderUI");
                 });

Azaion.LoaderUI/Constants.cs

@@ -1,13 +0,0 @@
-namespace Azaion.LoaderUI;
-
-public static class Constants
-{
-    public const string CONFIG_JSON_FILE = "loaderconfig.json";
-    public const string API_URL = "https://api.azaion.com";
-    public const string AZAION_SUITE_EXE = "Azaion.Suite.exe";
-    public const string SUITE_FOLDER = "suite";
-    public const string INFERENCE_EXE = "azaion-inference";
-    public const string EXTERNAL_LOADER_PATH = "azaion-loader.exe";
-    public const int    EXTERNAL_LOADER_PORT = 5020;
-    public const string EXTERNAL_LOADER_HOST = "127.0.0.1";
-}

Azaion.LoaderUI/ConstantsLoader.cs

@@ -0,0 +1,7 @@
+namespace Azaion.LoaderUI;
+
+public static class ConstantsLoader
+{
+    public const string SUITE_FOLDER = "suite";
+    public const int    EXTERNAL_LOADER_PORT = 5020;
+}

Azaion.LoaderUI/Login.xaml.cs

@@ -57,7 +57,7 @@ public partial class Login
 
             TbStatus.Foreground = Brushes.Black;
             var installerVersion = await GetInstallerVer();
-            var localVersion = GetLocalVer();
+            var localVersion = Constants.GetLocalVersion();
             var credsEncrypted = Security.Encrypt(creds);
 
             if (installerVersion > localVersion)
@@ -81,7 +81,7 @@ public partial class Login
                 Process.Start(Constants.AZAION_SUITE_EXE, $"-c {credsEncrypted}");
                 await Task.Delay(800);
                 TbStatus.Text = "Loading...";
-                while (!Process.GetProcessesByName(Constants.INFERENCE_EXE).Any())
+                while (!Process.GetProcessesByName(Path.GetFileNameWithoutExtension(Constants.EXTERNAL_INFERENCE_PATH)).Any())
                     await Task.Delay(500);
                 await Task.Delay(1500);
             }
@@ -106,12 +106,12 @@ public partial class Login
             process.StartInfo = new ProcessStartInfo
             {
                 FileName = Constants.EXTERNAL_LOADER_PATH,
-                Arguments = $"--port {Constants.EXTERNAL_LOADER_PORT} --api {Constants.API_URL}",
+                Arguments = $"--port {ConstantsLoader.EXTERNAL_LOADER_PORT} --api {Constants.DEFAULT_API_URL}",
                 CreateNoWindow = true
             };
             process.Start();
             dealer.Options.Identity = Encoding.UTF8.GetBytes(Guid.NewGuid().ToString("N"));
-            dealer.Connect($"tcp://{Constants.EXTERNAL_LOADER_HOST}:{Constants.EXTERNAL_LOADER_PORT}");
+            dealer.Connect($"tcp://{Constants.DEFAULT_ZMQ_INFERENCE_HOST}:{ConstantsLoader.EXTERNAL_LOADER_PORT}");
 
             var result = SendCommand(dealer, RemoteCommand.Create(CommandType.Login, creds));
             if (result.CommandType != CommandType.Ok)
@@ -164,7 +164,7 @@ public partial class Login
     {
         TbStatus.Text = "Checking for the newer version...";
         var installerDir = string.IsNullOrWhiteSpace(_dirConfig?.SuiteInstallerDirectory)
-            ? Constants.SUITE_FOLDER
+            ? ConstantsLoader.SUITE_FOLDER
             : _dirConfig.SuiteInstallerDirectory;
         var installerName = await _azaionApi.GetLastInstallerName(installerDir);
         var match = Regex.Match(installerName, @"\d+(\.\d+)+");
@@ -173,14 +173,6 @@ public partial class Login
         return new Version(match.Value);
     }
     
-    private Version GetLocalVer()
-    {
-        var localFileInfo = FileVersionInfo.GetVersionInfo(Constants.AZAION_SUITE_EXE);
-        if (string.IsNullOrWhiteSpace(localFileInfo.ProductVersion))
-            throw new Exception($"Can't find {Constants.AZAION_SUITE_EXE} and its version");
-        return new Version(localFileInfo.FileVersion!);
-    }
-
     private void CloseClick(object sender, RoutedEventArgs e) => Close();
 
     private void MainMouseMove(object sender, MouseEventArgs e)

Azaion.Suite/App.xaml.cs

@@ -153,12 +153,12 @@ public partial class App
                         typeof(Annotator.Annotator).Assembly,
                         typeof(DatasetExplorer).Assembly,
                         typeof(AnnotationService).Assembly));
-                    services.AddSingleton<LibVLC>(_ => new LibVLC());
+                    services.AddSingleton<LibVLC>(_ => new LibVLC("--no-osd", "--no-video-title-show", "--no-snapshot-preview"));
                     services.AddSingleton<FormState>();
                     services.AddSingleton<MediaPlayer>(sp =>
                     {
-                        var libVLC = sp.GetRequiredService<LibVLC>();
+                        var libVlc = sp.GetRequiredService<LibVLC>();
-                        return new MediaPlayer(libVLC);
+                        return new MediaPlayer(libVlc);
                     });
                     services.AddSingleton<AnnotatorEventHandler>();
                     services.AddSingleton<IDbFactory, DbFactory>();
@@ -177,8 +177,6 @@ public partial class App
 
             Annotation.InitializeDirs(_host.Services.GetRequiredService<IOptions<DirectoriesConfig>>().Value);
             _host.Services.GetRequiredService<DatasetExplorer>();
-            // datasetExplorer.Show();
-            // datasetExplorer.Hide();
 
             _mediator = _host.Services.GetRequiredService<IMediator>();
 

Azaion.Suite/config.system.json

@@ -30,7 +30,8 @@
 
     "TrackingDistanceConfidence": 0.15,
     "TrackingProbabilityIncrease": 15.0,
-    "TrackingIntersectionThreshold": 0.8,
+    "TrackingIntersectionThreshold": 0.6,
+    "BigImageTileOverlapPercent": 20,
 
     "ModelBatchSize": 4
     },