using System.Collections.ObjectModel; using System.Diagnostics; using System.IO; using System.Windows; using System.Windows.Controls; using System.Windows.Controls.Primitives; using System.Windows.Input; using System.Windows.Media; using Azaion.Annotator.DTO; using Azaion.Common; using Azaion.Common.Database; using Azaion.Common.DTO; using Azaion.Common.DTO.Config; using Azaion.Common.Events; using Azaion.Common.Extensions; using Azaion.Common.Services; using Azaion.Common.Services.Inference; using LibVLCSharp.Shared; using MediatR; using Microsoft.WindowsAPICodePack.Dialogs; using Size = System.Windows.Size; using IntervalTree; using LinqToDB; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MediaPlayer = LibVLCSharp.Shared.MediaPlayer; namespace Azaion.Annotator; public partial class Annotator { private readonly AppConfig _appConfig; private readonly LibVLC _libVlc; private readonly MediaPlayer _mediaPlayer; private readonly IMediator _mediator; private readonly FormState _formState; private readonly IConfigUpdater _configUpdater; private readonly HelpWindow _helpWindow; private readonly ILogger _logger; private readonly IDbFactory _dbFactory; private readonly IInferenceService _inferenceService; private readonly IInferenceClient _inferenceClient; private bool _suspendLayout; private bool _gpsPanelVisible; private readonly CancellationTokenSource _mainCancellationSource = new(); public CancellationTokenSource DetectionCancellationSource = new(); private bool _isInferenceNow; private readonly TimeSpan _thresholdBefore = TimeSpan.FromMilliseconds(50); private readonly TimeSpan _thresholdAfter = TimeSpan.FromMilliseconds(150); public ObservableCollection AllMediaFiles { get; set; } = new(); private ObservableCollection FilteredMediaFiles { get; set; } = new(); public Dictionary MediaFilesDict = new(); public IntervalTree TimedAnnotations { get; set; } = new(); public string MainTitle { get; set; } public Annotator( IConfigUpdater configUpdater, IOptions appConfig, LibVLC libVlc, MediaPlayer mediaPlayer, IMediator mediator, FormState formState, HelpWindow helpWindow, ILogger logger, IDbFactory dbFactory, IInferenceService inferenceService, IInferenceClient inferenceClient, IGpsMatcherService gpsMatcherService) { InitializeComponent(); MainTitle = $"Azaion Annotator {Constants.GetLocalVersion()}"; Title = MainTitle; _appConfig = appConfig.Value; _configUpdater = configUpdater; _libVlc = libVlc; _mediaPlayer = mediaPlayer; _mediator = mediator; _formState = formState; _helpWindow = helpWindow; _logger = logger; _dbFactory = dbFactory; _inferenceService = inferenceService; _inferenceClient = inferenceClient; Loaded += OnLoaded; Closed += OnFormClosed; Activated += (_, _) => _formState.ActiveWindow = WindowEnum.Annotator; TbFolder.TextChanged += async (_, _) => { if (!Path.Exists(TbFolder.Text)) return; try { _appConfig.DirectoriesConfig.VideosDirectory = TbFolder.Text; await ReloadFiles(); SaveUserSettings(); } catch (Exception e) { _logger.LogError(e, e.Message); } }; Editor.GetTimeFunc = () => TimeSpan.FromMilliseconds(_mediaPlayer.Time); MapMatcherComponent.Init(_appConfig, gpsMatcherService); } private void OnLoaded(object sender, RoutedEventArgs e) { Core.Initialize(); InitControls(); _suspendLayout = true; MainGrid.ColumnDefinitions.FirstOrDefault()!.Width = new GridLength(_appConfig.UIConfig.LeftPanelWidth); MainGrid.ColumnDefinitions.LastOrDefault()!.Width = new GridLength(_appConfig.UIConfig.RightPanelWidth); _suspendLayout = false; TbFolder.Text = _appConfig.DirectoriesConfig.VideosDirectory; LvClasses.Init(_appConfig.AnnotationConfig.DetectionClasses); } public void BlinkHelp(string helpText, int times = 2) { _ = Task.Run(async () => { for (int i = 0; i < times; i++) { Dispatcher.Invoke(() => StatusHelp.Text = helpText); await Task.Delay(200); Dispatcher.Invoke(() => StatusHelp.Text = ""); await Task.Delay(200); } Dispatcher.Invoke(() => StatusHelp.Text = helpText); }); } private void InitControls() { VideoView.MediaPlayer = _mediaPlayer; //On start playing media _mediaPlayer.Playing += (_, _) => { uint vw = 0, vh = 0; _mediaPlayer.Size(0, ref vw, ref vh); _formState.CurrentMediaSize = new Size(vw, vh); _formState.CurrentVideoLength = TimeSpan.FromMilliseconds(_mediaPlayer.Length); }; LvFiles.MouseDoubleClick += async (_, _) => { await _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.Play)); }; LvClasses.DetectionClassChanged += (_, args) => { var selectedClass = args.DetectionClass; Editor.CurrentAnnClass = selectedClass; _mediator.Publish(new AnnClassSelectedEvent(selectedClass)); }; _mediaPlayer.PositionChanged += (_, _) => ShowTimeAnnotations(TimeSpan.FromMilliseconds(_mediaPlayer.Time)); VideoSlider.ValueChanged += (_, newValue) => _mediaPlayer.Position = (float)(newValue / VideoSlider.Maximum); VideoSlider.KeyDown += (sender, args) => _mediator.Publish(new KeyEvent(sender, args, WindowEnum.Annotator)); Volume.ValueChanged += (_, newValue) => _mediator.Publish(new VolumeChangedEvent((int)newValue)); DgAnnotations.MouseDoubleClick += (sender, args) => { if (ItemsControl.ContainerFromElement((DataGrid)sender, (args.OriginalSource as DependencyObject)!) is DataGridRow dgRow) OpenAnnotationResult((Annotation)dgRow.Item); }; DgAnnotations.KeyUp += async (_, args) => { switch (args.Key) { case Key.Down: //cursor is already moved by system behaviour OpenAnnotationResult((Annotation)DgAnnotations.SelectedItem); break; case Key.Delete: var result = MessageBox.Show("Чи дійсно видалити аннотації?","Підтвердження видалення", MessageBoxButton.OKCancel, MessageBoxImage.Question); if (result != MessageBoxResult.OK) return; var res = DgAnnotations.SelectedItems.Cast().ToList(); var annotationNames = res.Select(x => x.Name).ToList(); await _mediator.Publish(new AnnotationsDeletedEvent(annotationNames)); break; } }; DgAnnotations.ItemsSource = _formState.AnnotationResults; } private void OpenAnnotationResult(Annotation ann) { _mediaPlayer.SetPause(true); if (!ann.IsSplit) Editor.RemoveAllAnns(); _mediaPlayer.Time = (long)ann.Time.TotalMilliseconds; Dispatcher.Invoke(() => { VideoSlider.Value = _mediaPlayer.Position * VideoSlider.Maximum; StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}"; Editor.ClearExpiredAnnotations(ann.Time); }); ShowAnnotation(ann, showImage: true, openResult: true); } private void SaveUserSettings() { if (_suspendLayout) return; _appConfig.UIConfig.LeftPanelWidth = MainGrid.ColumnDefinitions.FirstOrDefault()!.Width.Value; _appConfig.UIConfig.RightPanelWidth = MainGrid.ColumnDefinitions.LastOrDefault()!.Width.Value; _configUpdater.Save(_appConfig); } public void ShowTimeAnnotations(TimeSpan time, bool showImage = false) { Dispatcher.Invoke(() => { VideoSlider.Value = _mediaPlayer.Position * VideoSlider.Maximum; StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}"; Editor.ClearExpiredAnnotations(time); }); var annotations = TimedAnnotations.Query(time).ToList(); if (!annotations.Any()) return; foreach (var ann in annotations) ShowAnnotation(ann, showImage); } private void ShowAnnotation(Annotation annotation, bool showImage = false, bool openResult = false) { Dispatcher.Invoke(async () => { if (showImage && !annotation.IsSplit && File.Exists(annotation.ImagePath)) { Editor.SetBackground(await annotation.ImagePath.OpenImage()); _formState.BackgroundTime = annotation.Time; } if (annotation.SplitTile != null && openResult) { var canvasTileLocation = new CanvasLabel(new YoloLabel(annotation.SplitTile, _formState.CurrentMediaSize), RenderSize); Editor.ZoomTo(new Point(canvasTileLocation.CenterX, canvasTileLocation.CenterY)); } else Editor.CreateDetections(annotation, _appConfig.AnnotationConfig.DetectionClasses, _formState.CurrentMediaSize); }); } public async Task ReloadAnnotations() { await Dispatcher.InvokeAsync(async () => { _formState.AnnotationResults.Clear(); TimedAnnotations.Clear(); Editor.RemoveAllAnns(); var annotations = await _dbFactory.Run(async db => await db.Annotations.LoadWith(x => x.Detections) .Where(x => x.OriginalMediaName == _formState.MediaName) .OrderBy(x => x.Time) .ToListAsync(token: _mainCancellationSource.Token)); TimedAnnotations.Clear(); _formState.AnnotationResults.Clear(); foreach (var ann in annotations) { // Duplicate for speed TimedAnnotations.Add(ann.Time.Subtract(_thresholdBefore), ann.Time.Add(_thresholdAfter), ann); _formState.AnnotationResults.Add(ann); } }); } //Add manually public void AddAnnotation(Annotation annotation) { var time = annotation.Time; var previousAnnotations = TimedAnnotations.Query(time); TimedAnnotations.Remove(previousAnnotations); TimedAnnotations.Add(time.Subtract(_thresholdBefore), time.Add(_thresholdAfter), annotation); var existingResult = _formState.AnnotationResults.FirstOrDefault(x => x.Time == time); if (existingResult != null) { try { _formState.AnnotationResults.Remove(existingResult); } catch (Exception e) { _logger.LogError(e, e.Message); throw; } } var dict = _formState.AnnotationResults .Select((x, i) => new { x.Time, Index = i }) .ToDictionary(x => x.Time, x => x.Index); var index = dict.Where(x => x.Key < time) .OrderBy(x => time - x.Key) .Select(x => x.Value + 1) .FirstOrDefault(); _formState.AnnotationResults.Insert(index, annotation); } private async Task ReloadFiles() { var dir = new DirectoryInfo(_appConfig.DirectoriesConfig.VideosDirectory); if (!dir.Exists) return; var videoFiles = dir.GetFiles(_appConfig.AnnotationConfig.VideoFormats.ToArray()).Select(x => { var media = new Media(_libVlc, x.FullName); media.Parse(); var fInfo = new MediaFileInfo { Name = x.Name, Path = x.FullName, MediaType = MediaTypes.Video }; media.ParsedChanged += (_, _) => fInfo.Duration = TimeSpan.FromMilliseconds(media.Duration); return fInfo; }).ToList(); var imageFiles = dir.GetFiles(_appConfig.AnnotationConfig.ImageFormats.ToArray()) .Select(x => new MediaFileInfo { Name = x.Name, Path = x.FullName, MediaType = MediaTypes.Image }); var allFiles = videoFiles.Concat(imageFiles).ToList(); var allFileNames = allFiles.Select(x => x.FName).ToList(); var labelsDict = await _dbFactory.Run(async db => await db.Annotations .GroupBy(x => x.OriginalMediaName) .Where(x => allFileNames.Contains(x.Key)) .Select(x => x.Key) .ToDictionaryAsync(x => x, x => x)); foreach (var mediaFile in allFiles) mediaFile.HasAnnotations = labelsDict.ContainsKey(mediaFile.FName); AllMediaFiles = new ObservableCollection(allFiles); MediaFilesDict = AllMediaFiles.GroupBy(x => x.Name) .ToDictionary(gr => gr.Key, gr => gr.First()); LvFiles.ItemsSource = AllMediaFiles; DataContext = this; } private void OnFormClosed(object? sender, EventArgs e) { _mainCancellationSource.Cancel(); _inferenceService.StopInference(); DetectionCancellationSource.Cancel(); _mediaPlayer.Stop(); _mediaPlayer.Dispose(); _libVlc.Dispose(); } private void OpenContainingFolder(object sender, RoutedEventArgs e) { var mediaFileInfo = (sender as MenuItem)?.DataContext as MediaFileInfo; if (mediaFileInfo == null) return; Process.Start("explorer.exe", "/select,\"" + mediaFileInfo.Path +"\""); } public void SeekTo(long timeMilliseconds, bool setPause = true) { _mediaPlayer.SetPause(setPause); _mediaPlayer.Time = timeMilliseconds; VideoSlider.Value = _mediaPlayer.Position * 100; StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}"; } private void OpenFolderItemClick(object sender, RoutedEventArgs e) => OpenFolder(); private void OpenFolderButtonClick(object sender, RoutedEventArgs e) => OpenFolder(); private void OpenFolder() { var dlg = new CommonOpenFileDialog { Title = "Open Video folder", IsFolderPicker = true, InitialDirectory = Path.GetDirectoryName(_appConfig.DirectoriesConfig.VideosDirectory) }; var dialogResult = dlg.ShowDialog(); if (dialogResult != CommonFileDialogResult.Ok || string.IsNullOrEmpty(dlg.FileName)) return; _appConfig.DirectoriesConfig.VideosDirectory = dlg.FileName; TbFolder.Text = dlg.FileName; } private void TbFilter_OnTextChanged(object sender, TextChangedEventArgs e) { FilteredMediaFiles = new ObservableCollection(AllMediaFiles.Where(x => x.Name.ToLower().Contains(TbFilter.Text.ToLower())).ToList()); MediaFilesDict = FilteredMediaFiles.ToDictionary(x => x.FName); LvFiles.ItemsSource = FilteredMediaFiles; LvFiles.ItemsSource = FilteredMediaFiles; } private void PlayClick(object sender, RoutedEventArgs e) { _mediator.Publish(new AnnotatorControlEvent(_mediaPlayer.CanPause ? PlaybackControlEnum.Pause : PlaybackControlEnum.Play)); } private void PauseClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.Pause)); private void StopClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.Stop)); private void PreviousFrameClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.PreviousFrame)); private void NextFrameClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.NextFrame)); private void SaveAnnotationsClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.SaveAnnotations)); private void RemoveSelectedClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.RemoveSelectedAnns)); private void RemoveAllClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.RemoveAllAnns)); private void TurnOffVolume(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.TurnOffVolume)); private void TurnOnVolume(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.TurnOnVolume)); private void OpenHelpWindowClick(object sender, RoutedEventArgs e) { _helpWindow.Show(); _helpWindow.Activate(); } private void Thumb_OnDragCompleted(object sender, DragCompletedEventArgs e) => SaveUserSettings(); private void LvFilesContextOpening(object sender, ContextMenuEventArgs e) { var listItem = sender as ListViewItem; LvFilesContextMenu.DataContext = listItem!.DataContext; } private async void AIDetectBtn_OnClick(object sender, RoutedEventArgs e) { try { await AutoDetect(); } catch (Exception ex) { _logger.LogError(ex, ex.Message); } } public async Task AutoDetect() { if (_isInferenceNow) return; if (LvFiles.Items.IsEmpty) return; if (LvFiles.SelectedIndex == -1) LvFiles.SelectedIndex = 0; Dispatcher.Invoke(() => Editor.SetBackground(null)); _isInferenceNow = true; AIDetectBtn.IsEnabled = false; DetectionCancellationSource = new CancellationTokenSource(); var files = (FilteredMediaFiles.Count == 0 ? AllMediaFiles : FilteredMediaFiles) .Skip(LvFiles.SelectedIndex) .Select(x => x.Path) .ToList(); if (files.Count == 0) return; //TODO: Get Tile Size from UI based on height setup var tileSize = 550; await _inferenceService.RunInference(files, tileSize, DetectionCancellationSource.Token); LvFiles.Items.Refresh(); _isInferenceNow = false; StatusHelp.Text = "Розпізнавання завершено"; AIDetectBtn.IsEnabled = true; } private void SwitchGpsPanel(object sender, RoutedEventArgs e) { _gpsPanelVisible = !_gpsPanelVisible; if (_gpsPanelVisible) { GpsSplitterRow.Height = new GridLength(4); GpsSplitter.Visibility = Visibility.Visible; GpsSectionRow.Height = new GridLength(1, GridUnitType.Star); MapMatcherComponent.Visibility = Visibility.Visible; } else { GpsSplitterRow.Height = new GridLength(0); GpsSplitter.Visibility = Visibility.Collapsed; GpsSectionRow.Height = new GridLength(0); MapMatcherComponent.Visibility = Visibility.Collapsed; } } #region Denys Wishes private void SoundDetections(object sender, RoutedEventArgs e) { MessageBox.Show("Функція Аудіоаналіз знаходиться в стадії розробки","Система", MessageBoxButton.OK, MessageBoxImage.Information); } private void RunDroneMaintenance(object sender, RoutedEventArgs e) { MessageBox.Show("Функція Аналіз стану БПЛА знаходиться в стадії розробки","Система", MessageBoxButton.OK, MessageBoxImage.Information); } #endregion } public class GradientStyleSelector : StyleSelector { public override Style? SelectStyle(object item, DependencyObject container) { if (container is not DataGridRow row || row.DataContext is not Annotation result) return null; var style = new Style(typeof(DataGridRow)); var brush = new LinearGradientBrush { StartPoint = new Point(0, 0), EndPoint = new Point(1, 0) }; var gradients = new List(); if (result.Colors.Count == 0) { var color = (Color)ColorConverter.ConvertFromString("#40DDDDDD"); gradients = [new GradientStop(color, 0.99)]; } else { var increment = 1.0 / result.Colors.Count; var currentStop = increment; foreach (var c in result.Colors) { var resultColor = c.Color.ToConfidenceColor(c.Confidence); brush.GradientStops.Add(new GradientStop(resultColor, currentStop)); currentStop += increment; } } foreach (var gradientStop in gradients) brush.GradientStops.Add(gradientStop); style.Setters.Add(new Setter(DataGridRow.BackgroundProperty, brush)); return style; } }