From 3944df8efe87cfa45dd4720bfd13879ff839b6bb Mon Sep 17 00:00:00 2001 From: Alex Bezdieniezhnykh Date: Tue, 3 Dec 2024 11:36:20 +0200 Subject: [PATCH] add nth frame to ai recognition to config --- Azaion.Annotator/Annotator.xaml.cs | 72 +++++++++++++++---- Azaion.Annotator/AnnotatorEventHandler.cs | 5 +- .../Extensions/VLCFrameExtractor.cs | 6 +- Azaion.Common/Constants.cs | 7 +- Azaion.Common/Controls/CanvasEditor.cs | 2 + .../DTO/Config/AIRecognitionConfig.cs | 1 + Azaion.Common/DTO/Config/AppConfig.cs | 3 +- .../Extensions/DirectoryInfoExtensions.cs | 4 +- Azaion.Suite/App.xaml.cs | 2 +- Azaion.Suite/MainSuite.xaml | 4 +- Azaion.Suite/appsettings_manual.json | 3 +- Azaion.Suite/upload.cmd | 1 - 12 files changed, 81 insertions(+), 29 deletions(-) diff --git a/Azaion.Annotator/Annotator.xaml.cs b/Azaion.Annotator/Annotator.xaml.cs index 829cabe..e3d3afe 100644 --- a/Azaion.Annotator/Annotator.xaml.cs +++ b/Azaion.Annotator/Annotator.xaml.cs @@ -7,6 +7,7 @@ using System.Windows.Controls; using System.Windows.Controls.Primitives; using System.Windows.Input; using System.Windows.Media; +using System.Windows.Media.Imaging; using System.Windows.Threading; using Azaion.Annotator.DTO; using Azaion.Annotator.Extensions; @@ -187,10 +188,25 @@ public partial class Annotator OpenAnnotationResult((AnnotationResult)DgAnnotations.SelectedItem); break; case Key.Delete: - var result = MessageBox.Show("Чи дійсно видалити аннотації?","Підтвердження видалення", MessageBoxButton.OKCancel, MessageBoxImage.Question); + var result = MessageBox.Show(Application.Current.MainWindow, "Чи дійсно видалити аннотації?","Підтвердження видалення", MessageBoxButton.OKCancel, MessageBoxImage.Question); if (result != MessageBoxResult.OK) return; + // var allWindows = Application.Current.Windows.Cast(); + // try + // { + // foreach (var window in allWindows) + // window.IsEnabled = false; + // + // } + // finally + // { + // foreach (var window in allWindows) + // { + // window.IsEnabled = true; + // } + // } + var res = DgAnnotations.SelectedItems.Cast().ToList(); foreach (var annotationResult in res) { @@ -506,12 +522,14 @@ public partial class Annotator private (TimeSpan Time, List Detections)? _previousDetection; - public void AutoDetect(object sender, RoutedEventArgs e) + public async void AutoDetect(object sender, RoutedEventArgs e) { - if (LvFiles.SelectedItem == null) + if (LvFiles.Items.IsEmpty) return; + if (LvFiles.SelectedIndex == -1) + LvFiles.SelectedIndex = 0; - _mediator.Publish(new PlaybackControlEvent(PlaybackControlEnum.Play)); + await _mediator.Publish(new PlaybackControlEvent(PlaybackControlEnum.Play)); _mediaPlayer.SetPause(true); var manualCancellationSource = new CancellationTokenSource(); @@ -528,6 +546,7 @@ public partial class Annotator _mediaPlayer.SeekTo(TimeSpan.Zero); Editor.RemoveAllAnns(); }; + _autoDetectDialog.Top = Height - _autoDetectDialog.Height - 80; _autoDetectDialog.Left = 5; @@ -535,33 +554,38 @@ public partial class Annotator _ = Task.Run(async () => { - MediaFileInfo mediaInfo = null!; - Dispatcher.Invoke(() => - { - mediaInfo = (MediaFileInfo)LvFiles.SelectedItem; - }); - + var mediaInfo = Dispatcher.Invoke(() => (MediaFileInfo)LvFiles.SelectedItem); while (mediaInfo != null) { _formState.CurrentMedia = mediaInfo; + await Dispatcher.Invoke(async () => await ReloadAnnotations(token)); + if (mediaInfo.MediaType == MediaTypes.Image) { await DetectImage(mediaInfo, manualCancellationSource, token); + await Task.Delay(70, token); } else await DetectVideo(mediaInfo, manualCancellationSource, token); mediaInfo = Dispatcher.Invoke(() => { + if (LvFiles.SelectedIndex == LvFiles.Items.Count - 1) + return null; LvFiles.SelectedIndex += 1; return (MediaFileInfo)LvFiles.SelectedItem; }); } - Dispatcher.Invoke(() => _autoDetectDialog.Close()); + Dispatcher.Invoke(() => + { + _autoDetectDialog.Close(); + _mediaPlayer.Stop(); + LvFiles.Items.Refresh(); + }); }, token); _autoDetectDialog.ShowDialog(); - Dispatcher.Invoke(() => Editor.Background = new SolidColorBrush(Color.FromArgb(1, 0, 0, 0))); + Dispatcher.Invoke(() => Editor.ResetBackground()); } private async Task DetectImage(MediaFileInfo mediaInfo, CancellationTokenSource manualCancellationSource, CancellationToken token) @@ -571,6 +595,8 @@ public partial class Annotator var stream = new FileStream(mediaInfo.Path, FileMode.Open); var detections = await _aiDetector.Detect(stream, token); await ProcessDetection((TimeSpan.FromMilliseconds(0), stream), detections, token); + if (detections.Count != 0) + mediaInfo.HasAnnotations = true; } catch (Exception e) { @@ -584,18 +610,38 @@ public partial class Annotator var prevSeekTime = 0.0; await foreach (var timeframe in _vlcFrameExtractor.ExtractFrames(mediaInfo.Path, token)) { + Console.WriteLine($"Detect time: {timeframe.Time}"); try { var detections = await _aiDetector.Detect(timeframe.Stream, token); + var isValid = IsValidDetection(timeframe.Time, detections); + if (timeframe.Time.TotalSeconds > prevSeekTime + 1) { Dispatcher.Invoke(() => SeekTo(timeframe.Time)); prevSeekTime = timeframe.Time.TotalSeconds; + if (!isValid) //Show frame anyway + { + var bitmap = new BitmapImage(); + bitmap.BeginInit(); + timeframe.Stream.Seek(0, SeekOrigin.Begin); + bitmap.StreamSource = timeframe.Stream; + bitmap.CacheOption = BitmapCacheOption.OnLoad; + bitmap.EndInit(); + bitmap.Freeze(); + + Dispatcher.Invoke(() => + { + Editor.RemoveAllAnns(); + Editor.Background = new ImageBrush { ImageSource = bitmap }; + }); + } } - if (!IsValidDetection(timeframe.Time, detections)) + if (!isValid) continue; + mediaInfo.HasAnnotations = true; await ProcessDetection(timeframe, detections, token); } catch (Exception ex) diff --git a/Azaion.Annotator/AnnotatorEventHandler.cs b/Azaion.Annotator/AnnotatorEventHandler.cs index b5e2412..07c26fb 100644 --- a/Azaion.Annotator/AnnotatorEventHandler.cs +++ b/Azaion.Annotator/AnnotatorEventHandler.cs @@ -132,7 +132,7 @@ public class AnnotatorEventHandler( mainWindow.BlinkHelp(HelpTexts.HelpTextsDict[HelpTextEnum.AnnotationHelp]); if (formState.BackgroundTime.HasValue) { - mainWindow.Editor.Background = new SolidColorBrush(Color.FromArgb(1, 0, 0, 0)); + mainWindow.Editor.ResetBackground(); formState.BackgroundTime = null; } break; @@ -213,6 +213,7 @@ public class AnnotatorEventHandler( if (mainWindow.LvFiles.SelectedItem == null) return; var mediaInfo = (MediaFileInfo)mainWindow.LvFiles.SelectedItem; + mainWindow.Editor.ResetBackground(); formState.CurrentMedia = mediaInfo; mediaPlayer.Stop(); @@ -250,7 +251,7 @@ public class AnnotatorEventHandler( if (formState.BackgroundTime.HasValue) { //no need to save image, it's already there, just remove background - mainWindow.Editor.Background = new SolidColorBrush(Color.FromArgb(1, 0, 0, 0)); + mainWindow.Editor.ResetBackground(); formState.BackgroundTime = null; //next item diff --git a/Azaion.Annotator/Extensions/VLCFrameExtractor.cs b/Azaion.Annotator/Extensions/VLCFrameExtractor.cs index d0890ad..c415a5e 100644 --- a/Azaion.Annotator/Extensions/VLCFrameExtractor.cs +++ b/Azaion.Annotator/Extensions/VLCFrameExtractor.cs @@ -2,12 +2,13 @@ using System.IO; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using Azaion.Common.DTO.Config; using LibVLCSharp.Shared; using SkiaSharp; namespace Azaion.Annotator.Extensions; -public class VLCFrameExtractor(LibVLC libVLC) +public class VLCFrameExtractor(LibVLC libVLC, AIRecognitionConfig config) { private const uint RGBA_BYTES = 4; private const int PLAYBACK_RATE = 4; @@ -104,10 +105,11 @@ public class VLCFrameExtractor(LibVLC libVLC) _lastFrameTimestamp = playerTime; } - if (_frameCounter > 20 && _frameCounter % 10 == 0) + if (_frameCounter > 20 && _frameCounter % config.FramePeriodRecognition == 0) { var msToAdd = (_frameCounter - _lastFrame) * (_lastFrame == 0 ? 0 : _lastFrameTimestamp.TotalMilliseconds / _lastFrame); var time = _lastFrameTimestamp.Add(TimeSpan.FromMilliseconds(msToAdd)); + FramesQueue.Enqueue(new FrameInfo(time, _currentBitmap)); } else diff --git a/Azaion.Common/Constants.cs b/Azaion.Common/Constants.cs index 8464329..0d50595 100644 --- a/Azaion.Common/Constants.cs +++ b/Azaion.Common/Constants.cs @@ -50,9 +50,10 @@ public class Constants # region AIRecognitionConfig public const double DEFAULT_FRAME_RECOGNITION_SECONDS = 2; - public const double TRACKING_DISTANCE_CONFIDENCE = 0.15; - public const double TRACKING_PROBABILITY_INCREASE = 15; - public const double TRACKING_INTERSECTION_THRESHOLD = 0.8; + public const double TRACKING_DISTANCE_CONFIDENCE = 0.15; + public const double TRACKING_PROBABILITY_INCREASE = 15; + public const double TRACKING_INTERSECTION_THRESHOLD = 0.8; + public const int DEFAULT_FRAME_PERIOD_RECOGNITION = 4; # endregion AIRecognitionConfig diff --git a/Azaion.Common/Controls/CanvasEditor.cs b/Azaion.Common/Controls/CanvasEditor.cs index 4893122..1ef090f 100644 --- a/Azaion.Common/Controls/CanvasEditor.cs +++ b/Azaion.Common/Controls/CanvasEditor.cs @@ -360,4 +360,6 @@ public class CanvasEditor : Canvas .ToList(); RemoveAnnotations(expiredAnns); } + + public void ResetBackground() => Background = new SolidColorBrush(Color.FromArgb(1, 0, 0, 0)); } \ No newline at end of file diff --git a/Azaion.Common/DTO/Config/AIRecognitionConfig.cs b/Azaion.Common/DTO/Config/AIRecognitionConfig.cs index fdcfe7d..1be807c 100644 --- a/Azaion.Common/DTO/Config/AIRecognitionConfig.cs +++ b/Azaion.Common/DTO/Config/AIRecognitionConfig.cs @@ -6,4 +6,5 @@ public class AIRecognitionConfig public double TrackingDistanceConfidence { get; set; } public double TrackingProbabilityIncrease { get; set; } public double TrackingIntersectionThreshold { get; set; } + public int FramePeriodRecognition { get; set; } } \ No newline at end of file diff --git a/Azaion.Common/DTO/Config/AppConfig.cs b/Azaion.Common/DTO/Config/AppConfig.cs index 60cdb37..da0d3be 100644 --- a/Azaion.Common/DTO/Config/AppConfig.cs +++ b/Azaion.Common/DTO/Config/AppConfig.cs @@ -77,7 +77,8 @@ public class ConfigUpdater : IConfigUpdater FrameRecognitionSeconds = Constants.DEFAULT_FRAME_RECOGNITION_SECONDS, TrackingDistanceConfidence = Constants.TRACKING_DISTANCE_CONFIDENCE, TrackingProbabilityIncrease = Constants.TRACKING_PROBABILITY_INCREASE, - TrackingIntersectionThreshold = Constants.TRACKING_INTERSECTION_THRESHOLD + TrackingIntersectionThreshold = Constants.TRACKING_INTERSECTION_THRESHOLD, + FramePeriodRecognition = Constants.DEFAULT_FRAME_PERIOD_RECOGNITION } }; Save(appConfig); diff --git a/Azaion.Common/Extensions/DirectoryInfoExtensions.cs b/Azaion.Common/Extensions/DirectoryInfoExtensions.cs index f0787cc..2a5f7fc 100644 --- a/Azaion.Common/Extensions/DirectoryInfoExtensions.cs +++ b/Azaion.Common/Extensions/DirectoryInfoExtensions.cs @@ -1,10 +1,10 @@ using System.IO; -namespace Azaion.Annotator.Extensions; +namespace Azaion.Common.Extensions; public static class DirectoryInfoExtensions { public static IEnumerable GetFiles(this DirectoryInfo dir, params string[] searchExtensions) => dir.GetFiles("*.*", SearchOption.AllDirectories) - .Where(f => searchExtensions.Any(s => f.Name.Contains(s, StringComparison.CurrentCultureIgnoreCase))).ToList(); + .Where(f => searchExtensions.Any(s => f.Name.ToLower().Contains(s))).ToList(); } \ No newline at end of file diff --git a/Azaion.Suite/App.xaml.cs b/Azaion.Suite/App.xaml.cs index c5bce5c..ce9a429 100644 --- a/Azaion.Suite/App.xaml.cs +++ b/Azaion.Suite/App.xaml.cs @@ -185,7 +185,7 @@ public partial class App if (windowEnum != WindowEnum.None) return windowEnum; - element = element.Parent as FrameworkElement; + element = (element.Parent ?? element.TemplatedParent) as FrameworkElement; } return WindowEnum.None; diff --git a/Azaion.Suite/MainSuite.xaml b/Azaion.Suite/MainSuite.xaml index 1589da7..1d910a8 100644 --- a/Azaion.Suite/MainSuite.xaml +++ b/Azaion.Suite/MainSuite.xaml @@ -8,8 +8,7 @@ WindowStyle="None" ResizeMode="NoResize" Top="0" - Topmost="True" - > + Topmost="True"> @@ -29,7 +28,6 @@ -