using System.Collections.ObjectModel; using System.Diagnostics; using System.IO; using System.Windows; using System.Windows.Controls; using System.Windows.Controls.Primitives; using System.Windows.Input; using System.Windows.Media; using System.Windows.Media.Imaging; using Azaion.Annotator.DTO; using Azaion.Annotator.Extensions; using Azaion.Common.Database; using Azaion.Common.DTO; using Azaion.Common.DTO.Config; using Azaion.Common.DTO.Queue; using Azaion.Common.Events; using Azaion.Common.Extensions; using Azaion.Common.Services; using LibVLCSharp.Shared; using MediatR; using Microsoft.WindowsAPICodePack.Dialogs; using Size = System.Windows.Size; using IntervalTree; using LinqToDB; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MediaPlayer = LibVLCSharp.Shared.MediaPlayer; namespace Azaion.Annotator; public partial class Annotator { private readonly AppConfig _appConfig; private readonly LibVLC _libVLC; private readonly MediaPlayer _mediaPlayer; private readonly IMediator _mediator; private readonly FormState _formState; private readonly IConfigUpdater _configUpdater; private readonly HelpWindow _helpWindow; private readonly ILogger _logger; private readonly VLCFrameExtractor _vlcFrameExtractor; private readonly IAIDetector _aiDetector; private readonly AnnotationService _annotationService; private readonly IDbFactory _dbFactory; private readonly CancellationTokenSource _ctSource = new(); private ObservableCollection AnnotationClasses { get; set; } = new(); private bool _suspendLayout; private readonly TimeSpan _thresholdBefore = TimeSpan.FromMilliseconds(100); private readonly TimeSpan _thresholdAfter = TimeSpan.FromMilliseconds(300); public ObservableCollection AllMediaFiles { get; set; } = new(); public ObservableCollection FilteredMediaFiles { get; set; } = new(); public IntervalTree TimedAnnotations { get; set; } = new(); private AutodetectDialog _autoDetectDialog = new() { Topmost = true }; public Annotator( IConfigUpdater configUpdater, IOptions appConfig, LibVLC libVLC, MediaPlayer mediaPlayer, IMediator mediator, FormState formState, HelpWindow helpWindow, ILogger logger, VLCFrameExtractor vlcFrameExtractor, IAIDetector aiDetector, AnnotationService annotationService, IDbFactory dbFactory) { InitializeComponent(); _appConfig = appConfig.Value; _configUpdater = configUpdater; _libVLC = libVLC; _mediaPlayer = mediaPlayer; _mediator = mediator; _formState = formState; _helpWindow = helpWindow; _logger = logger; _vlcFrameExtractor = vlcFrameExtractor; _aiDetector = aiDetector; _annotationService = annotationService; _dbFactory = dbFactory; Loaded += OnLoaded; Closed += OnFormClosed; Activated += (_, _) => _formState.ActiveWindow = WindowEnum.Annotator; TbFolder.TextChanged += async (sender, args) => { if (!Path.Exists(TbFolder.Text)) return; try { _appConfig.DirectoriesConfig.VideosDirectory = TbFolder.Text; await ReloadFiles(); await SaveUserSettings(); } catch (Exception e) { _logger.LogError(e, e.Message); } }; Editor.GetTimeFunc = () => TimeSpan.FromMilliseconds(_mediaPlayer.Time); } private void OnLoaded(object sender, RoutedEventArgs e) { Core.Initialize(); InitControls(); _suspendLayout = true; MainGrid.ColumnDefinitions.FirstOrDefault()!.Width = new GridLength(_appConfig.AnnotationConfig.LeftPanelWidth); MainGrid.ColumnDefinitions.LastOrDefault()!.Width = new GridLength(_appConfig.AnnotationConfig.RightPanelWidth); _suspendLayout = false; TbFolder.Text = _appConfig.DirectoriesConfig.VideosDirectory; AnnotationClasses = new ObservableCollection(_appConfig.AnnotationConfig.AnnotationClasses); LvClasses.ItemsSource = AnnotationClasses; LvClasses.SelectedIndex = 0; if (LvFiles.Items.IsEmpty) BlinkHelp(HelpTexts.HelpTextsDict[HelpTextEnum.Initial]); } public void BlinkHelp(string helpText, int times = 2) { _ = Task.Run(async () => { for (int i = 0; i < times; i++) { Dispatcher.Invoke(() => StatusHelp.Text = helpText); await Task.Delay(200); Dispatcher.Invoke(() => StatusHelp.Text = ""); await Task.Delay(200); } Dispatcher.Invoke(() => StatusHelp.Text = helpText); }); } private void InitControls() { VideoView.MediaPlayer = _mediaPlayer; //On start playing media _mediaPlayer.Playing += async (sender, args) => { if (_formState.CurrentMrl == _mediaPlayer.Media?.Mrl) return; //already loaded all the info _formState.CurrentMrl = _mediaPlayer.Media?.Mrl ?? ""; uint vw = 0, vh = 0; _mediaPlayer.Size(0, ref vw, ref vh); _formState.CurrentVideoSize = new Size(vw, vh); _formState.CurrentVideoLength = TimeSpan.FromMilliseconds(_mediaPlayer.Length); await Dispatcher.Invoke(async () => await ReloadAnnotations()); if (_formState.CurrentMedia?.MediaType == MediaTypes.Image) { await Task.Delay(100); //wait to load the frame and set on pause ShowTimeAnnotations(TimeSpan.FromMilliseconds(_mediaPlayer.Time)); _mediaPlayer.SetPause(true); } }; LvFiles.MouseDoubleClick += async (_, _) => await _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.Play)); LvClasses.SelectionChanged += (_, _) => { var selectedClass = (DetectionClass)LvClasses.SelectedItem; Editor.CurrentAnnClass = selectedClass; _mediator.Publish(new AnnClassSelectedEvent(selectedClass)); }; _mediaPlayer.PositionChanged += (o, args) => ShowTimeAnnotations(TimeSpan.FromMilliseconds(_mediaPlayer.Time)); VideoSlider.ValueChanged += (value, newValue) => _mediaPlayer.Position = (float)(newValue / VideoSlider.Maximum); VideoSlider.KeyDown += (sender, args) => _mediator.Publish(new KeyEvent(sender, args, WindowEnum.Annotator)); Volume.ValueChanged += (_, newValue) => _mediator.Publish(new VolumeChangedEvent((int)newValue)); SizeChanged += async (_, _) => await SaveUserSettings(); LocationChanged += async (_, _) => await SaveUserSettings(); StateChanged += async (_, _) => await SaveUserSettings(); DgAnnotations.MouseDoubleClick += (sender, args) => { var dgRow = ItemsControl.ContainerFromElement((DataGrid)sender, (args.OriginalSource as DependencyObject)!) as DataGridRow; OpenAnnotationResult((AnnotationResult)dgRow!.Item); }; DgAnnotations.KeyUp += async (sender, args) => { switch (args.Key) { case Key.Up: case Key.Down: //cursor is already moved by system behaviour OpenAnnotationResult((AnnotationResult)DgAnnotations.SelectedItem); break; case Key.Delete: var result = MessageBox.Show("Чи дійсно видалити аннотації?","Підтвердження видалення", MessageBoxButton.OKCancel, MessageBoxImage.Question); if (result != MessageBoxResult.OK) return; var res = DgAnnotations.SelectedItems.Cast().ToList(); var annotations = res.Select(x => x.Annotation).ToList(); await _mediator.Publish(new AnnotationsDeletedEvent(annotations)); break; } }; Editor.Mediator = _mediator; DgAnnotations.ItemsSource = _formState.AnnotationResults; } public void OpenAnnotationResult(AnnotationResult res) { _mediaPlayer.SetPause(true); Editor.RemoveAllAnns(); _mediaPlayer.Time = (long)res.Annotation.Time.TotalMilliseconds; Dispatcher.Invoke(() => { VideoSlider.Value = _mediaPlayer.Position * VideoSlider.Maximum; StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}"; Editor.ClearExpiredAnnotations(res.Annotation.Time); }); ShowAnnotations(res.Annotation, showImage: true); } private async Task SaveUserSettings() { if (_suspendLayout) return; _appConfig.AnnotationConfig.LeftPanelWidth = MainGrid.ColumnDefinitions.FirstOrDefault()!.Width.Value; _appConfig.AnnotationConfig.RightPanelWidth = MainGrid.ColumnDefinitions.LastOrDefault()!.Width.Value; await ThrottleExt.ThrottleRunFirst(() => { _configUpdater.Save(_appConfig); return Task.CompletedTask; }, TimeSpan.FromSeconds(5)); } private void ShowTimeAnnotations(TimeSpan time) { Dispatcher.Invoke(() => { VideoSlider.Value = _mediaPlayer.Position * VideoSlider.Maximum; StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}"; Editor.ClearExpiredAnnotations(time); }); ShowAnnotations(TimedAnnotations.Query(time).FirstOrDefault()); } private void ShowAnnotations(Annotation? annotation, bool showImage = false) { if (annotation == null) return; Dispatcher.Invoke(async () => { var canvasSize = Editor.RenderSize; var videoSize = _formState.CurrentVideoSize; if (showImage) { if (File.Exists(annotation.ImagePath)) { Editor.Background = new ImageBrush { ImageSource = await annotation.ImagePath.OpenImage() }; _formState.BackgroundTime = annotation.Time; videoSize = Editor.RenderSize; } } foreach (var detection in annotation.Detections) { var annClass = _appConfig.AnnotationConfig.AnnotationClasses[detection.ClassNumber]; var canvasLabel = new CanvasLabel(detection, canvasSize, videoSize, detection.Probability); Editor.CreateDetectionControl(annClass, annotation.Time, canvasLabel); } }); } private async Task ReloadAnnotations() { _formState.AnnotationResults.Clear(); TimedAnnotations.Clear(); Editor.RemoveAllAnns(); var annotations = await _dbFactory.Run(async db => await db.Annotations.LoadWith(x => x.Detections) .Where(x => x.Name.Contains(_formState.VideoName)) .ToListAsync(token: _ctSource.Token)); foreach (var ann in annotations) AddAnnotation(ann); } //Add manually public void AddAnnotation(Annotation annotation) { var time = annotation.Time; var previousAnnotations = TimedAnnotations.Query(time); TimedAnnotations.Remove(previousAnnotations); TimedAnnotations.Add(time.Subtract(_thresholdBefore), time.Add(_thresholdAfter), annotation); var existingResult = _formState.AnnotationResults.FirstOrDefault(x => x.Annotation.Time == time); if (existingResult != null) _formState.AnnotationResults.Remove(existingResult); var dict = _formState.AnnotationResults .Select((x, i) => new { x.Annotation.Time, Index = i }) .ToDictionary(x => x.Time, x => x.Index); var index = dict.Where(x => x.Key < time) .OrderBy(x => time - x.Key) .Select(x => x.Value + 1) .FirstOrDefault(); _formState.AnnotationResults.Insert(index, new AnnotationResult(_appConfig.AnnotationConfig.DetectionClassesDict, annotation)); } private async Task ReloadFiles() { var dir = new DirectoryInfo(_appConfig.DirectoriesConfig.VideosDirectory); if (!dir.Exists) return; var videoFiles = dir.GetFiles(_appConfig.AnnotationConfig.VideoFormats.ToArray()).Select(x => { using var media = new Media(_libVLC, x.FullName); media.Parse(); var fInfo = new MediaFileInfo { Name = x.Name, Path = x.FullName, MediaType = MediaTypes.Video }; media.ParsedChanged += (_, _) => fInfo.Duration = TimeSpan.FromMilliseconds(media.Duration); return fInfo; }).ToList(); var imageFiles = dir.GetFiles(_appConfig.AnnotationConfig.ImageFormats.ToArray()) .Select(x => new MediaFileInfo { Name = x.Name, Path = x.FullName, MediaType = MediaTypes.Image }); var allFiles = videoFiles.Concat(imageFiles).ToList(); var allFileNames = allFiles.Select(x => x.FName).ToList(); var labelsDict = await _dbFactory.Run(async db => await db.Annotations .GroupBy(x => x.Name.Substring(0, x.Name.Length - 7)) .Where(x => allFileNames.Contains(x.Key)) .ToDictionaryAsync(x => x.Key, x => x.Key)); foreach (var mediaFile in allFiles) mediaFile.HasAnnotations = labelsDict.ContainsKey(mediaFile.FName); AllMediaFiles = new ObservableCollection(allFiles); LvFiles.ItemsSource = AllMediaFiles; BlinkHelp(AllMediaFiles.Count == 0 ? HelpTexts.HelpTextsDict[HelpTextEnum.Initial] : HelpTexts.HelpTextsDict[HelpTextEnum.PlayVideo]); DataContext = this; } private void OnFormClosed(object? sender, EventArgs e) { _mediaPlayer.Stop(); _mediaPlayer.Dispose(); _libVLC.Dispose(); } private void OpenContainingFolder(object sender, RoutedEventArgs e) { var mediaFileInfo = (sender as MenuItem)?.DataContext as MediaFileInfo; if (mediaFileInfo == null) return; Process.Start("explorer.exe", "/select,\"" + mediaFileInfo.Path +"\""); } public void SeekTo(long timeMilliseconds) { _mediaPlayer.SetPause(true); _mediaPlayer.Time = timeMilliseconds; VideoSlider.Value = _mediaPlayer.Position * 100; StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}"; } private void SeekTo(TimeSpan time) => SeekTo((long)time.TotalMilliseconds); // private void AddClassBtnClick(object sender, RoutedEventArgs e) // { // LvClasses.IsReadOnly = false; // DetectionClasses.Add(new DetectionClass(DetectionClasses.Count)); // LvClasses.SelectedIndex = DetectionClasses.Count - 1; // } private async void OpenFolderItemClick(object sender, RoutedEventArgs e) => await OpenFolder(); private async void OpenFolderButtonClick(object sender, RoutedEventArgs e) => await OpenFolder(); private async Task OpenFolder() { var dlg = new CommonOpenFileDialog { Title = "Open Video folder", IsFolderPicker = true, InitialDirectory = Path.GetDirectoryName(AppDomain.CurrentDomain.BaseDirectory) }; var dialogResult = dlg.ShowDialog(); if (dialogResult != CommonFileDialogResult.Ok || string.IsNullOrEmpty(dlg.FileName)) return; _appConfig.DirectoriesConfig.VideosDirectory = dlg.FileName; TbFolder.Text = dlg.FileName; await ReloadFiles(); await SaveUserSettings(); } private void TbFilter_OnTextChanged(object sender, TextChangedEventArgs e) { FilteredMediaFiles = new ObservableCollection(AllMediaFiles.Where(x => x.Name.ToLower().Contains(TbFilter.Text.ToLower())).ToList()); LvFiles.ItemsSource = FilteredMediaFiles; LvFiles.ItemsSource = FilteredMediaFiles; } private void PlayClick(object sender, RoutedEventArgs e) { _mediator.Publish(new AnnotatorControlEvent(_mediaPlayer.CanPause ? PlaybackControlEnum.Pause : PlaybackControlEnum.Play)); } private void PauseClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.Pause)); private void StopClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.Stop)); private void PreviousFrameClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.PreviousFrame)); private void NextFrameClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.NextFrame)); private void SaveAnnotationsClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.SaveAnnotations)); private void RemoveSelectedClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.RemoveSelectedAnns)); private void RemoveAllClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.RemoveAllAnns)); private void TurnOffVolume(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.TurnOffVolume)); private void TurnOnVolume(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.TurnOnVolume)); private void OpenHelpWindowClick(object sender, RoutedEventArgs e) { _helpWindow.Show(); _helpWindow.Activate(); } private void Thumb_OnDragCompleted(object sender, DragCompletedEventArgs e) => _ = SaveUserSettings(); private void LvFilesContextOpening(object sender, ContextMenuEventArgs e) { var listItem = sender as ListViewItem; LvFilesContextMenu.DataContext = listItem!.DataContext; } private (TimeSpan Time, List Detections)? _previousDetection; public async void AutoDetect(object sender, RoutedEventArgs e) { if (LvFiles.Items.IsEmpty) return; if (LvFiles.SelectedIndex == -1) LvFiles.SelectedIndex = 0; await _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.Play)); _mediaPlayer.Stop(); var manualCancellationSource = new CancellationTokenSource(); var token = manualCancellationSource.Token; _autoDetectDialog = new AutodetectDialog { Topmost = true, Owner = this }; _autoDetectDialog.Closing += (_, _) => { manualCancellationSource.Cancel(); _mediaPlayer.SeekTo(TimeSpan.Zero); Editor.RemoveAllAnns(); }; _autoDetectDialog.Top = Height - _autoDetectDialog.Height - 80; _autoDetectDialog.Left = 5; _autoDetectDialog.Log("Ініціалізація AI..."); _ = Task.Run(async () => { var mediaInfo = Dispatcher.Invoke(() => (MediaFileInfo)LvFiles.SelectedItem); while (mediaInfo != null) { _formState.CurrentMedia = mediaInfo; await Dispatcher.Invoke(async () => await ReloadAnnotations()); if (mediaInfo.MediaType == MediaTypes.Image) { await DetectImage(mediaInfo, manualCancellationSource, token); await Task.Delay(70, token); } else await DetectVideo(mediaInfo, manualCancellationSource, token); mediaInfo = Dispatcher.Invoke(() => { if (LvFiles.SelectedIndex == LvFiles.Items.Count - 1) return null; LvFiles.SelectedIndex += 1; return (MediaFileInfo)LvFiles.SelectedItem; }); } Dispatcher.Invoke(() => { _autoDetectDialog.Close(); _mediaPlayer.Stop(); LvFiles.Items.Refresh(); }); }, token); _autoDetectDialog.ShowDialog(); Dispatcher.Invoke(() => Editor.ResetBackground()); } private async Task DetectImage(MediaFileInfo mediaInfo, CancellationTokenSource manualCancellationSource, CancellationToken token) { try { var fName = Path.GetFileNameWithoutExtension(mediaInfo.Path); var stream = new FileStream(mediaInfo.Path, FileMode.Open); var detections = await _aiDetector.Detect(fName, stream, token); await ProcessDetection((TimeSpan.FromMilliseconds(0), stream), Path.GetExtension(mediaInfo.Path), detections, token); if (detections.Count != 0) mediaInfo.HasAnnotations = true; } catch (Exception e) { _logger.LogError(e, e.Message); await manualCancellationSource.CancelAsync(); } } private async Task DetectVideo(MediaFileInfo mediaInfo, CancellationTokenSource manualCancellationSource, CancellationToken token) { var prevSeekTime = 0.0; await foreach (var timeframe in _vlcFrameExtractor.ExtractFrames(mediaInfo.Path, token)) { Console.WriteLine($"Detect time: {timeframe.Time}"); try { var fName = _formState.GetTimeName(timeframe.Time); var detections = await _aiDetector.Detect(fName, timeframe.Stream, token); var isValid = IsValidDetection(timeframe.Time, detections); Console.WriteLine($"Detection time: {timeframe.Time}"); var log = string.Join(Environment.NewLine, detections.Select(det => $"{_appConfig.AnnotationConfig.DetectionClassesDict[det.ClassNumber].Name}: " + $"xy=({det.CenterX:F2},{det.CenterY:F2}), " + $"size=({det.Width:F2}, {det.Height:F2}), " + $"prob: {det.Probability:F1}%")); log = $"Detection time: {timeframe.Time}, Valid: {isValid}. {Environment.NewLine} {log}"; Dispatcher.Invoke(() => _autoDetectDialog.Log(log)); if (timeframe.Time.TotalMilliseconds > prevSeekTime + 250) { Dispatcher.Invoke(() => SeekTo(timeframe.Time)); prevSeekTime = timeframe.Time.TotalMilliseconds; if (!isValid) //Show frame anyway { Dispatcher.Invoke(() => { Editor.RemoveAllAnns(); Editor.Background = new ImageBrush { ImageSource = timeframe.Stream.OpenImage() }; }); } } if (!isValid) continue; mediaInfo.HasAnnotations = true; await ProcessDetection(timeframe, ".jpg", detections, token); await timeframe.Stream.DisposeAsync(); } catch (Exception ex) { _logger.LogError(ex, ex.Message); await manualCancellationSource.CancelAsync(); } } } private bool IsValidDetection(TimeSpan time, List detections) { // No AI detection, forbid if (detections.Count == 0) return false; // Very first detection, allow if (!_previousDetection.HasValue) return true; var prev = _previousDetection.Value; // Time between detections is >= than Frame Recognition Seconds, allow if (time >= prev.Time.Add(TimeSpan.FromSeconds(_appConfig.AIRecognitionConfig.FrameRecognitionSeconds))) return true; // Detection is earlier than previous + FrameRecognitionSeconds. // Look to the detections more in detail // More detected objects, allow if (detections.Count > prev.Detections.Count) return true; foreach (var det in detections) { var point = new Point(det.CenterX, det.CenterY); var closestObject = prev.Detections .Select(p => new { Point = p, Distance = point.SqrDistance(new Point(p.CenterX, p.CenterY)) }) .OrderBy(x => x.Distance) .First(); // Closest object is farther than Tracking distance confidence, hence it's a different object, allow if (closestObject.Distance > _appConfig.AIRecognitionConfig.TrackingDistanceConfidence) return true; // Since closest object within distance confidence, then it is tracking of the same object. Then if recognition probability for the object > increase from previous if (det.Probability >= closestObject.Point.Probability + _appConfig.AIRecognitionConfig.TrackingProbabilityIncrease) return true; } return false; } private async Task ProcessDetection((TimeSpan Time, Stream Stream) timeframe, string imageExtension, List detections, CancellationToken token = default) { _previousDetection = (timeframe.Time, detections); await Dispatcher.Invoke(async () => { try { var fName = _formState.GetTimeName(timeframe.Time); var annotation = await _annotationService.SaveAnnotation(fName, imageExtension, detections, SourceEnum.AI, timeframe.Stream, token); Editor.Background = new ImageBrush { ImageSource = await annotation.ImagePath.OpenImage() }; Editor.RemoveAllAnns(); ShowAnnotations(annotation, true); AddAnnotation(annotation); var log = string.Join(Environment.NewLine, detections.Select(det => $"{_appConfig.AnnotationConfig.DetectionClassesDict[det.ClassNumber].Name}: " + $"xy=({det.CenterX:F2},{det.CenterY:F2}), " + $"size=({det.Width:F2}, {det.Height:F2}), " + $"prob: {det.Probability:F1}%")); Dispatcher.Invoke(() => _autoDetectDialog.Log(log)); } catch (Exception e) { _logger.LogError(e, e.Message); } }); } }