add nth frame to ai recognition to config

Azaion.Annotator/Annotator.xaml.cs

@@ -7,6 +7,7 @@ using System.Windows.Controls;
 using System.Windows.Controls.Primitives;
 using System.Windows.Input;
 using System.Windows.Media;
+using System.Windows.Media.Imaging;
 using System.Windows.Threading;
 using Azaion.Annotator.DTO;
 using Azaion.Annotator.Extensions;
@@ -187,10 +188,25 @@ public partial class Annotator
                     OpenAnnotationResult((AnnotationResult)DgAnnotations.SelectedItem);
                     break;
                 case Key.Delete:
-                    var result = MessageBox.Show("Чи дійсно видалити аннотації?","Підтвердження видалення", MessageBoxButton.OKCancel, MessageBoxImage.Question);
+                    var result = MessageBox.Show(Application.Current.MainWindow, "Чи дійсно видалити аннотації?","Підтвердження видалення", MessageBoxButton.OKCancel, MessageBoxImage.Question);
                     if (result != MessageBoxResult.OK)
                         return;
 
+                    // var allWindows = Application.Current.Windows.Cast<Window>();
+                    // try
+                    // {
+                    //     foreach (var window in allWindows)
+                    //         window.IsEnabled = false;
+                    //
+                    // }
+                    // finally
+                    // {
+                    //     foreach (var window in allWindows)
+                    //     {
+                    //         window.IsEnabled = true;
+                    //     }
+                    // }
+
                     var res = DgAnnotations.SelectedItems.Cast<AnnotationResult>().ToList();
                     foreach (var annotationResult in res)
                     {
@@ -506,12 +522,14 @@ public partial class Annotator
 
     private (TimeSpan Time, List<Detection> Detections)? _previousDetection;
 
-    public void AutoDetect(object sender, RoutedEventArgs e)
+    public async void AutoDetect(object sender, RoutedEventArgs e)
     {
-        if (LvFiles.SelectedItem == null)
+        if (LvFiles.Items.IsEmpty)
             return;
+        if (LvFiles.SelectedIndex == -1)
+            LvFiles.SelectedIndex = 0;
 
-        _mediator.Publish(new PlaybackControlEvent(PlaybackControlEnum.Play));
+        await _mediator.Publish(new PlaybackControlEvent(PlaybackControlEnum.Play));
         _mediaPlayer.SetPause(true);
 
         var manualCancellationSource = new CancellationTokenSource();
@@ -528,6 +546,7 @@ public partial class Annotator
             _mediaPlayer.SeekTo(TimeSpan.Zero);
             Editor.RemoveAllAnns();
         };
+
         _autoDetectDialog.Top = Height - _autoDetectDialog.Height - 80;
         _autoDetectDialog.Left = 5;
 
@@ -535,33 +554,38 @@ public partial class Annotator
 
         _ = Task.Run(async () =>
         {
-            MediaFileInfo mediaInfo = null!;
+            var mediaInfo = Dispatcher.Invoke(() => (MediaFileInfo)LvFiles.SelectedItem);
-            Dispatcher.Invoke(() =>
-            {
-                mediaInfo = (MediaFileInfo)LvFiles.SelectedItem;
-            });
-
             while (mediaInfo != null)
             {
                 _formState.CurrentMedia = mediaInfo;
+                await Dispatcher.Invoke(async () => await ReloadAnnotations(token));
+
                 if (mediaInfo.MediaType == MediaTypes.Image)
                 {
                     await DetectImage(mediaInfo, manualCancellationSource, token);
+                    await Task.Delay(70, token);
                 }
                 else
                     await DetectVideo(mediaInfo, manualCancellationSource, token);
 
                 mediaInfo = Dispatcher.Invoke(() =>
                 {
+                    if (LvFiles.SelectedIndex == LvFiles.Items.Count - 1)
+                        return null;
                     LvFiles.SelectedIndex += 1;
                     return (MediaFileInfo)LvFiles.SelectedItem;
                 });
             }
-            Dispatcher.Invoke(() => _autoDetectDialog.Close());
+            Dispatcher.Invoke(() =>
+            {
+                _autoDetectDialog.Close();
+                _mediaPlayer.Stop();
+                LvFiles.Items.Refresh();
+            });
         }, token);
 
         _autoDetectDialog.ShowDialog();
-        Dispatcher.Invoke(() => Editor.Background = new SolidColorBrush(Color.FromArgb(1, 0, 0, 0)));
+        Dispatcher.Invoke(() => Editor.ResetBackground());
     }
 
     private async Task DetectImage(MediaFileInfo mediaInfo, CancellationTokenSource manualCancellationSource, CancellationToken token)
@@ -571,6 +595,8 @@ public partial class Annotator
             var stream = new FileStream(mediaInfo.Path, FileMode.Open);
             var detections = await _aiDetector.Detect(stream, token);
             await ProcessDetection((TimeSpan.FromMilliseconds(0), stream), detections, token);
+            if (detections.Count != 0)
+                mediaInfo.HasAnnotations = true;
         }
         catch (Exception e)
         {
@@ -584,18 +610,38 @@ public partial class Annotator
         var prevSeekTime = 0.0;
         await foreach (var timeframe in _vlcFrameExtractor.ExtractFrames(mediaInfo.Path, token))
         {
+            Console.WriteLine($"Detect time: {timeframe.Time}");
             try
             {
                 var detections = await _aiDetector.Detect(timeframe.Stream, token);
+                var isValid = IsValidDetection(timeframe.Time, detections);
+
                 if (timeframe.Time.TotalSeconds > prevSeekTime + 1)
                 {
                     Dispatcher.Invoke(() => SeekTo(timeframe.Time));
                     prevSeekTime = timeframe.Time.TotalSeconds;
+                    if (!isValid) //Show frame anyway
+                    {
+                        var bitmap = new BitmapImage();
+                        bitmap.BeginInit();
+                        timeframe.Stream.Seek(0, SeekOrigin.Begin);
+                        bitmap.StreamSource = timeframe.Stream;
+                        bitmap.CacheOption = BitmapCacheOption.OnLoad;
+                        bitmap.EndInit();
+                        bitmap.Freeze();
+
+                        Dispatcher.Invoke(() =>
+                        {
+                            Editor.RemoveAllAnns();
+                            Editor.Background = new ImageBrush { ImageSource = bitmap };
+                        });
+                    }
                 }
 
-                if (!IsValidDetection(timeframe.Time, detections))
+                if (!isValid)
                     continue;
 
+                mediaInfo.HasAnnotations = true;
                 await ProcessDetection(timeframe, detections, token);
             }
             catch (Exception ex)

Azaion.Annotator/AnnotatorEventHandler.cs

@@ -132,7 +132,7 @@ public class AnnotatorEventHandler(
                         mainWindow.BlinkHelp(HelpTexts.HelpTextsDict[HelpTextEnum.AnnotationHelp]);
                     if (formState.BackgroundTime.HasValue)
                     {
-                        mainWindow.Editor.Background = new SolidColorBrush(Color.FromArgb(1, 0, 0, 0));
+                        mainWindow.Editor.ResetBackground();
                         formState.BackgroundTime = null;
                     }
                     break;
@@ -213,6 +213,7 @@ public class AnnotatorEventHandler(
         if (mainWindow.LvFiles.SelectedItem == null)
             return;
         var mediaInfo = (MediaFileInfo)mainWindow.LvFiles.SelectedItem;
+        mainWindow.Editor.ResetBackground();
 
         formState.CurrentMedia = mediaInfo;
         mediaPlayer.Stop();
@@ -250,7 +251,7 @@ public class AnnotatorEventHandler(
             if (formState.BackgroundTime.HasValue)
             {
                 //no need to save image, it's already there, just remove background
-                mainWindow.Editor.Background = new SolidColorBrush(Color.FromArgb(1, 0, 0, 0));
+                mainWindow.Editor.ResetBackground();
                 formState.BackgroundTime = null;
 
                 //next item

Azaion.Annotator/Extensions/VLCFrameExtractor.cs

@@ -2,12 +2,13 @@
 using System.IO;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
+using Azaion.Common.DTO.Config;
 using LibVLCSharp.Shared;
 using SkiaSharp;
 
 namespace Azaion.Annotator.Extensions;
 
-public class VLCFrameExtractor(LibVLC libVLC)
+public class VLCFrameExtractor(LibVLC libVLC, AIRecognitionConfig config)
 {
     private const uint RGBA_BYTES = 4;
     private const int PLAYBACK_RATE = 4;
@@ -104,10 +105,11 @@ public class VLCFrameExtractor(LibVLC libVLC)
             _lastFrameTimestamp = playerTime;
         }
 
-        if (_frameCounter > 20 &&  _frameCounter % 10 == 0)
+        if (_frameCounter > 20 && _frameCounter % config.FramePeriodRecognition == 0)
         {
             var msToAdd = (_frameCounter - _lastFrame) * (_lastFrame == 0 ? 0 : _lastFrameTimestamp.TotalMilliseconds / _lastFrame);
             var time = _lastFrameTimestamp.Add(TimeSpan.FromMilliseconds(msToAdd));
+
             FramesQueue.Enqueue(new FrameInfo(time, _currentBitmap));
         }
         else

Azaion.Common/Constants.cs

@@ -50,9 +50,10 @@ public class Constants
     # region AIRecognitionConfig
 
     public const double DEFAULT_FRAME_RECOGNITION_SECONDS = 2;
-    public const double TRACKING_DISTANCE_CONFIDENCE = 0.15;
+    public const double TRACKING_DISTANCE_CONFIDENCE      = 0.15;
-    public const double TRACKING_PROBABILITY_INCREASE = 15;
+    public const double TRACKING_PROBABILITY_INCREASE     = 15;
-    public const double TRACKING_INTERSECTION_THRESHOLD = 0.8;
+    public const double TRACKING_INTERSECTION_THRESHOLD   = 0.8;
+    public const int    DEFAULT_FRAME_PERIOD_RECOGNITION  = 4;
 
     # endregion AIRecognitionConfig
 

Azaion.Common/Controls/CanvasEditor.cs

@@ -360,4 +360,6 @@ public class CanvasEditor : Canvas
             .ToList();
         RemoveAnnotations(expiredAnns);
     }
+
+    public void ResetBackground() => Background = new SolidColorBrush(Color.FromArgb(1, 0, 0, 0));
 }

Azaion.Common/DTO/Config/AIRecognitionConfig.cs

@@ -6,4 +6,5 @@ public class AIRecognitionConfig
     public double TrackingDistanceConfidence { get; set; }
     public double TrackingProbabilityIncrease { get; set; }
     public double TrackingIntersectionThreshold { get; set; }
+    public int FramePeriodRecognition { get; set; }
 }

Azaion.Common/DTO/Config/AppConfig.cs

@@ -77,7 +77,8 @@ public class ConfigUpdater : IConfigUpdater
                 FrameRecognitionSeconds = Constants.DEFAULT_FRAME_RECOGNITION_SECONDS,
                 TrackingDistanceConfidence = Constants.TRACKING_DISTANCE_CONFIDENCE,
                 TrackingProbabilityIncrease = Constants.TRACKING_PROBABILITY_INCREASE,
-                TrackingIntersectionThreshold = Constants.TRACKING_INTERSECTION_THRESHOLD
+                TrackingIntersectionThreshold = Constants.TRACKING_INTERSECTION_THRESHOLD,
+                FramePeriodRecognition = Constants.DEFAULT_FRAME_PERIOD_RECOGNITION
             }
         };
         Save(appConfig);

Azaion.Common/Extensions/DirectoryInfoExtensions.cs

@@ -1,10 +1,10 @@
 using System.IO;
 
-namespace Azaion.Annotator.Extensions;
+namespace Azaion.Common.Extensions;
 
 public static class DirectoryInfoExtensions
 {
     public static IEnumerable<FileInfo> GetFiles(this DirectoryInfo dir, params string[] searchExtensions) =>
         dir.GetFiles("*.*", SearchOption.AllDirectories)
-            .Where(f => searchExtensions.Any(s => f.Name.Contains(s, StringComparison.CurrentCultureIgnoreCase))).ToList();
+            .Where(f => searchExtensions.Any(s => f.Name.ToLower().Contains(s))).ToList();
 }

Azaion.Suite/App.xaml.cs

@@ -185,7 +185,7 @@ public partial class App
             if (windowEnum != WindowEnum.None)
                 return windowEnum;
 
-            element = element.Parent as FrameworkElement;
+            element = (element.Parent ?? element.TemplatedParent) as FrameworkElement;
         }
 
         return WindowEnum.None;

Azaion.Suite/MainSuite.xaml

@@ -8,8 +8,7 @@
         WindowStyle="None"
         ResizeMode="NoResize"
         Top="0"
-        Topmost="True"
+        Topmost="True">
-        >
     <Grid Background="Black">
         <Grid.ColumnDefinitions>
             <ColumnDefinition Width="48"></ColumnDefinition>
@@ -29,7 +28,6 @@
                 </ItemsPanelTemplate>
             </ListView.ItemsPanel>
         </ListView>
-
         <Border Grid.Column="2">
             <Button
                 VerticalAlignment="Top"

Azaion.Suite/appsettings_manual.json

@@ -47,7 +47,8 @@
     "FrameRecognitionSeconds" : 2,
     "TrackingDistanceConfidence" : 0.15,
     "TrackingProbabilityIncrease" : 15,
-    "TrackingIntersectionThreshold" : 0.8
+    "TrackingIntersectionThreshold" : 0.8,
+    "FramePeriodRecognition": 4
   },
 
   "ThumbnailConfig": {

Azaion.Suite/upload.cmd

@@ -38,5 +38,4 @@ curl --location %API_URL%/resources ^
     -H "Content-Type: multipart/form-data" ^
     --form "data=@%FILE2_TO_UPLOAD%"
 
-
 echo Done!