annotations/Azaion.Annotator/Annotator.xaml.cs

using System.Collections.ObjectModel;
using System.Diagnostics;
using System.IO;
using System.Windows;
using System.Windows.Controls;
using System.Windows.Controls.Primitives;
using System.Windows.Input;
using System.Windows.Media;
using Azaion.Annotator.DTO;
using Azaion.Common;
using Azaion.Common.Database;
using Azaion.Common.DTO;
using Azaion.Common.DTO.Config;
using Azaion.Common.Events;
using Azaion.Common.Extensions;
using Azaion.Common.Services;
using LibVLCSharp.Shared;
using MediatR;
using Microsoft.WindowsAPICodePack.Dialogs;
using Size = System.Windows.Size;
using IntervalTree;
using LinqToDB;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MediaPlayer = LibVLCSharp.Shared.MediaPlayer;

namespace Azaion.Annotator;

public partial class Annotator
{
    private readonly AppConfig _appConfig;
    private readonly LibVLC _libVLC;
    private readonly MediaPlayer _mediaPlayer;
    private readonly IMediator _mediator;
    private readonly FormState _formState;

    private readonly IConfigUpdater _configUpdater;
    private readonly HelpWindow _helpWindow;
    private readonly ILogger<Annotator> _logger;
    private readonly IDbFactory _dbFactory;
    private readonly IInferenceService _inferenceService;
    private readonly IInferenceClient _inferenceClient;
    
    private bool _suspendLayout;
    private bool _gpsPanelVisible = false;

    public readonly CancellationTokenSource MainCancellationSource = new();
    public CancellationTokenSource DetectionCancellationSource = new();
    public bool IsInferenceNow = false;

    private readonly TimeSpan _thresholdBefore = TimeSpan.FromMilliseconds(50);
    private readonly TimeSpan _thresholdAfter = TimeSpan.FromMilliseconds(150);

    public ObservableCollection<MediaFileInfo> AllMediaFiles { get; set; } = new();
    public ObservableCollection<MediaFileInfo> FilteredMediaFiles { get; set; } = new();
    public Dictionary<string, MediaFileInfo> MediaFilesDict = new();

    public IntervalTree<TimeSpan, Annotation> TimedAnnotations { get; set; } = new();

    public Annotator(
        IConfigUpdater configUpdater,
        IOptions<AppConfig> appConfig,
        LibVLC libVLC,
        MediaPlayer mediaPlayer,
        IMediator mediator,
        FormState formState,
        HelpWindow helpWindow,
        ILogger<Annotator> logger,
        IDbFactory dbFactory,
        IInferenceService inferenceService,
        IInferenceClient inferenceClient,
        IGpsMatcherService gpsMatcherService)
    {
        InitializeComponent();

        _appConfig = appConfig.Value;
        _configUpdater = configUpdater;
        _libVLC = libVLC;
        _mediaPlayer = mediaPlayer;
        _mediator = mediator;
        _formState = formState;
        _helpWindow = helpWindow;
        _logger = logger;
        _dbFactory = dbFactory;
        _inferenceService = inferenceService;
        _inferenceClient = inferenceClient;

        Loaded += OnLoaded;
        Closed += OnFormClosed;
        Activated += (_, _) => _formState.ActiveWindow = WindowEnum.Annotator;
        TbFolder.TextChanged += async (sender, args) =>
        {
            if (!Path.Exists(TbFolder.Text))
                return;
            try
            {
                _appConfig.DirectoriesConfig.VideosDirectory = TbFolder.Text;
                await ReloadFiles();
                SaveUserSettings();
            }
            catch (Exception e)
            {
                _logger.LogError(e, e.Message);
            }
        };
        _inferenceClient.AIAvailabilityReceived += (_, command) =>
        {
            Dispatcher.Invoke(() =>
            {
                _logger.LogInformation(command.Message);
                var aiEnabled = command.Message == "enabled";
                AIDetectBtn.IsEnabled = aiEnabled;
                var aiDisabledText = "Будь ласка, зачекайте, наразі розпізнавання AI недоступне";
                var messagesDict = new Dictionary<string, string>
                {
                    { "disabled", aiDisabledText },
                    { "downloading", "Будь ласка зачекайте, йде завантаження AI для Вашої відеокарти" },
                    { "converting", "Будь ласка зачекайте, йде налаштування AI під Ваше залізо. (5-12 хвилин в залежності від моделі відеокарти, до 50 хв на старих GTX1650)" },
                    { "uploading", "Будь ласка зачекайте, йде зберігання" },
                    { "enabled", "AI готовий для розпізнавання" }
                };

                if (command.Message?.StartsWith("Error") ?? false)
                {
                    _logger.LogError(command.Message);
                    StatusHelp.Text = command.Message;
                }

                else
                    StatusHelp.Text = messagesDict!.GetValueOrDefault(command.Message, aiDisabledText);

                if (aiEnabled)
                    StatusHelp.Foreground = aiEnabled ? Brushes.White : Brushes.Red;
            });
        };
        _inferenceClient.Send(RemoteCommand.Create(CommandType.AIAvailabilityCheck));

        Editor.GetTimeFunc = () => TimeSpan.FromMilliseconds(_mediaPlayer.Time);
        MapMatcherComponent.Init(_appConfig, gpsMatcherService);
    }

    private void OnLoaded(object sender, RoutedEventArgs e)
    {
        Core.Initialize();
        InitControls();

        _suspendLayout = true;

        MainGrid.ColumnDefinitions.FirstOrDefault()!.Width = new GridLength(_appConfig.UIConfig.LeftPanelWidth);
        MainGrid.ColumnDefinitions.LastOrDefault()!.Width = new GridLength(_appConfig.UIConfig.RightPanelWidth);

        _suspendLayout = false;
        TbFolder.Text = _appConfig.DirectoriesConfig.VideosDirectory;

        LvClasses.Init(_appConfig.AnnotationConfig.DetectionClasses);
    }

    public void BlinkHelp(string helpText, int times = 2)
    {
        _ = Task.Run(async () =>
        {
            for (int i = 0; i < times; i++)
            {
                Dispatcher.Invoke(() => StatusHelp.Text = helpText);
                await Task.Delay(200);
                Dispatcher.Invoke(() => StatusHelp.Text = "");
                await Task.Delay(200);
            }

            Dispatcher.Invoke(() => StatusHelp.Text = helpText);
        });
    }

    private void InitControls()
    {
        VideoView.MediaPlayer = _mediaPlayer;

        //On start playing media
        _mediaPlayer.Playing += async (sender, args) =>
        {
            if (_formState.CurrentMrl == _mediaPlayer.Media?.Mrl)
                return; //already loaded all the info
            
            await Dispatcher.Invoke(async () => await ReloadAnnotations());
            
            //show image
            if (_formState.CurrentMedia?.MediaType == MediaTypes.Image)
            {
                await Task.Delay(100); //wait to load the frame and set on pause
                ShowTimeAnnotations(TimeSpan.FromMilliseconds(_mediaPlayer.Time), showImage: true);
                return;
            }
            
            _formState.CurrentMrl = _mediaPlayer.Media?.Mrl ?? "";
            uint vw = 0, vh = 0;
            _mediaPlayer.Size(0, ref vw, ref vh);
            _formState.CurrentVideoSize = new Size(vw, vh);
            _formState.CurrentVideoLength = TimeSpan.FromMilliseconds(_mediaPlayer.Length);
        };

        LvFiles.MouseDoubleClick += async (_, _) =>
        {
            await _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.Play));
        };

        LvClasses.DetectionClassChanged += (_, args) =>
        {
            var selectedClass = args.DetectionClass;
            Editor.CurrentAnnClass = selectedClass;
            _mediator.Publish(new AnnClassSelectedEvent(selectedClass));
        };

        _mediaPlayer.PositionChanged += (o, args) =>
            ShowTimeAnnotations(TimeSpan.FromMilliseconds(_mediaPlayer.Time));

        VideoSlider.ValueChanged += (value, newValue) =>
            _mediaPlayer.Position = (float)(newValue / VideoSlider.Maximum);

        VideoSlider.KeyDown += (sender, args) =>
            _mediator.Publish(new KeyEvent(sender, args, WindowEnum.Annotator));

        Volume.ValueChanged += (_, newValue) =>
            _mediator.Publish(new VolumeChangedEvent((int)newValue));

        DgAnnotations.MouseDoubleClick += (sender, args) =>
        {
            var dgRow = ItemsControl.ContainerFromElement((DataGrid)sender, (args.OriginalSource as DependencyObject)!) as DataGridRow;
            if (dgRow != null)
                OpenAnnotationResult((AnnotationResult)dgRow!.Item);

        };

        DgAnnotations.KeyUp += async (sender, args) =>
        {
            switch (args.Key)
            {
                case Key.Up:
                case Key.Down: //cursor is already moved by system behaviour
                    OpenAnnotationResult((AnnotationResult)DgAnnotations.SelectedItem);
                    break;
                case Key.Delete:
                    var result = MessageBox.Show("Чи дійсно видалити аннотації?","Підтвердження видалення", MessageBoxButton.OKCancel, MessageBoxImage.Question);
                    if (result != MessageBoxResult.OK)
                        return;

                    var res = DgAnnotations.SelectedItems.Cast<AnnotationResult>().ToList();
                    var annotationNames = res.Select(x => x.Annotation.Name).ToList();

                    await _mediator.Publish(new AnnotationsDeletedEvent(annotationNames));
                    break;
            }
        };

        Editor.Mediator = _mediator;
        DgAnnotations.ItemsSource = _formState.AnnotationResults;
    }

    public void OpenAnnotationResult(AnnotationResult res)
    {
        _mediaPlayer.SetPause(true);
        Editor.RemoveAllAnns();
        _mediaPlayer.Time = (long)res.Annotation.Time.TotalMilliseconds;

        Dispatcher.Invoke(() =>
        {
            VideoSlider.Value = _mediaPlayer.Position * VideoSlider.Maximum;
            StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}";
            Editor.ClearExpiredAnnotations(res.Annotation.Time);
        });

        ShowAnnotation(res.Annotation, showImage: true);
    }
    private void SaveUserSettings()
    {
        if (_suspendLayout)
            return;

        _appConfig.UIConfig.LeftPanelWidth = MainGrid.ColumnDefinitions.FirstOrDefault()!.Width.Value;
        _appConfig.UIConfig.RightPanelWidth = MainGrid.ColumnDefinitions.LastOrDefault()!.Width.Value;

        _configUpdater.Save(_appConfig);
    }

    private void ShowTimeAnnotations(TimeSpan time, bool showImage = false)
    {
        Dispatcher.Invoke(() =>
        {
            VideoSlider.Value = _mediaPlayer.Position * VideoSlider.Maximum;
            StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}";
            Editor.ClearExpiredAnnotations(time);
        });

        ShowAnnotation(TimedAnnotations.Query(time).FirstOrDefault(), showImage);
    }

    private void ShowAnnotation(Annotation? annotation, bool showImage = false)
    {
        if (annotation == null)
            return;
        Dispatcher.Invoke(async () =>
        {
            var videoSize = _formState.CurrentVideoSize;
            if (showImage)
            {
                if (File.Exists(annotation.ImagePath))
                {
                    Editor.SetImageSource(await annotation.ImagePath.OpenImage());
                    _formState.BackgroundTime = annotation.Time;
                    videoSize = Editor.RenderSize;
                }
            }
            Editor.CreateDetections(annotation.Time, annotation.Detections, _appConfig.AnnotationConfig.DetectionClasses, videoSize);
        });
    }

    private async Task ReloadAnnotations()
    {
        _formState.AnnotationResults.Clear();
        TimedAnnotations.Clear();
        Editor.RemoveAllAnns();

        var annotations = await _dbFactory.Run(async db =>
            await db.Annotations.LoadWith(x => x.Detections)
                .Where(x => x.OriginalMediaName == _formState.VideoName)
                .OrderBy(x => x.Time)
                .ToListAsync(token: MainCancellationSource.Token));

        TimedAnnotations.Clear();
        _formState.AnnotationResults.Clear();
        foreach (var ann in annotations)
        {
            TimedAnnotations.Add(ann.Time.Subtract(_thresholdBefore), ann.Time.Add(_thresholdAfter), ann);
            _formState.AnnotationResults.Add(new AnnotationResult(_appConfig.AnnotationConfig.DetectionClassesDict, ann));
        }
    }

    //Add manually
    public void AddAnnotation(Annotation annotation)
    {
        var mediaInfo = (MediaFileInfo)LvFiles.SelectedItem;
        if ((mediaInfo?.FName ?? "") != annotation.OriginalMediaName)
            return;

        var time = annotation.Time;
        var previousAnnotations = TimedAnnotations.Query(time);
        TimedAnnotations.Remove(previousAnnotations);
        TimedAnnotations.Add(time.Subtract(_thresholdBefore), time.Add(_thresholdAfter), annotation);

        var existingResult = _formState.AnnotationResults.FirstOrDefault(x => x.Annotation.Time == time);
        if (existingResult != null)
        {
            try
            {
                _formState.AnnotationResults.Remove(existingResult);
            }
            catch (Exception e)
            {
                _logger.LogError(e, e.Message);
                throw;
            }
        }

        var dict = _formState.AnnotationResults
            .Select((x, i) => new { x.Annotation.Time, Index = i })
            .ToDictionary(x => x.Time, x => x.Index);

        var index = dict.Where(x => x.Key < time)
            .OrderBy(x => time - x.Key)
            .Select(x => x.Value + 1)
            .FirstOrDefault();

        var annRes = new AnnotationResult(_appConfig.AnnotationConfig.DetectionClassesDict, annotation);
        _formState.AnnotationResults.Insert(index, annRes);
    }

    private async Task ReloadFiles()
    {
        var dir = new DirectoryInfo(_appConfig.DirectoriesConfig.VideosDirectory);
        if (!dir.Exists)
            return;

        var videoFiles = dir.GetFiles(_appConfig.AnnotationConfig.VideoFormats.ToArray()).Select(x =>
        {
            using var media = new Media(_libVLC, x.FullName);
            media.Parse();
            var fInfo = new MediaFileInfo
            {
                Name = x.Name,
                Path = x.FullName,
                MediaType = MediaTypes.Video
            };
            media.ParsedChanged += (_, _) => fInfo.Duration = TimeSpan.FromMilliseconds(media.Duration);
            return fInfo;
        }).ToList();

        var imageFiles = dir.GetFiles(_appConfig.AnnotationConfig.ImageFormats.ToArray())
            .Select(x => new MediaFileInfo
            {
                Name = x.Name,
                Path = x.FullName,
                MediaType = MediaTypes.Image
            });
        var allFiles = videoFiles.Concat(imageFiles).ToList();

        var allFileNames = allFiles.Select(x => x.FName).ToList();

        var labelsDict = await _dbFactory.Run(async db => await db.Annotations
            .GroupBy(x => x.Name.Substring(0, x.Name.Length - 7))
            .Where(x => allFileNames.Contains(x.Key))
            .ToDictionaryAsync(x => x.Key, x => x.Key));

        foreach (var mediaFile in allFiles)
            mediaFile.HasAnnotations = labelsDict.ContainsKey(mediaFile.FName);

        AllMediaFiles = new ObservableCollection<MediaFileInfo>(allFiles);
        MediaFilesDict = AllMediaFiles.GroupBy(x => x.Name)
            .ToDictionary(gr => gr.Key, gr => gr.First());
        LvFiles.ItemsSource = AllMediaFiles;
        DataContext = this;
    }

    private void OnFormClosed(object? sender, EventArgs e)
    {
        MainCancellationSource.Cancel();
        _inferenceService.StopInference();
        DetectionCancellationSource.Cancel();

        _mediaPlayer.Stop();
        _mediaPlayer.Dispose();
        _libVLC.Dispose();
    }

    private void OpenContainingFolder(object sender, RoutedEventArgs e)
    {
        var mediaFileInfo = (sender as MenuItem)?.DataContext as MediaFileInfo;
        if (mediaFileInfo == null)
            return;

        Process.Start("explorer.exe", "/select,\"" + mediaFileInfo.Path +"\"");
    }

    public void SeekTo(long timeMilliseconds, bool setPause = true)
    {
        _mediaPlayer.SetPause(setPause);
        _mediaPlayer.Time = timeMilliseconds;
        VideoSlider.Value = _mediaPlayer.Position * 100;

        StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}";
    }

    private void SeekTo(TimeSpan time) =>
        SeekTo((long)time.TotalMilliseconds);

    private async void OpenFolderItemClick(object sender, RoutedEventArgs e) => await OpenFolder();
    private async void OpenFolderButtonClick(object sender, RoutedEventArgs e) => await OpenFolder();

    private async Task OpenFolder()
    {
        var dlg = new CommonOpenFileDialog
        {
            Title = "Open Video folder",
            IsFolderPicker = true,
            InitialDirectory = Path.GetDirectoryName(_appConfig.DirectoriesConfig.VideosDirectory)
        };
        var dialogResult = dlg.ShowDialog();

        if (dialogResult != CommonFileDialogResult.Ok || string.IsNullOrEmpty(dlg.FileName))
            return;

        _appConfig.DirectoriesConfig.VideosDirectory = dlg.FileName;
        TbFolder.Text = dlg.FileName;
        await Task.CompletedTask;
    }

    private void TbFilter_OnTextChanged(object sender, TextChangedEventArgs e)
    {
        FilteredMediaFiles = new ObservableCollection<MediaFileInfo>(AllMediaFiles.Where(x => x.Name.ToLower().Contains(TbFilter.Text.ToLower())).ToList());
        MediaFilesDict = FilteredMediaFiles.ToDictionary(x => x.FName);
        LvFiles.ItemsSource = FilteredMediaFiles;
        LvFiles.ItemsSource = FilteredMediaFiles;
    }

    private void PlayClick(object sender, RoutedEventArgs e)
    {
        _mediator.Publish(new AnnotatorControlEvent(_mediaPlayer.CanPause ? PlaybackControlEnum.Pause : PlaybackControlEnum.Play));
    }

    private void PauseClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.Pause));
    private void StopClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.Stop));

    private void PreviousFrameClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.PreviousFrame));
    private void NextFrameClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.NextFrame));

    private void SaveAnnotationsClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.SaveAnnotations));

    private void RemoveSelectedClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.RemoveSelectedAnns));
    private void RemoveAllClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.RemoveAllAnns));
    private void TurnOffVolume(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.TurnOffVolume));
    private void TurnOnVolume(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.TurnOnVolume));

    private void OpenHelpWindowClick(object sender, RoutedEventArgs e)
    {
        _helpWindow.Show();
        _helpWindow.Activate();
    }

    private void Thumb_OnDragCompleted(object sender, DragCompletedEventArgs e) => SaveUserSettings();

    private void LvFilesContextOpening(object sender, ContextMenuEventArgs e)
    {
        var listItem = sender as ListViewItem;
        LvFilesContextMenu.DataContext = listItem!.DataContext;
    }

    private async void AIDetectBtn_OnClick(object sender, RoutedEventArgs e)
    {
        try
        {
            await AutoDetect();
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, ex.Message);
        }
    }

    public async Task AutoDetect()
    {
        if (IsInferenceNow)
            return;

        if (LvFiles.Items.IsEmpty)
            return;
        if (LvFiles.SelectedIndex == -1)
            LvFiles.SelectedIndex = 0;

        Dispatcher.Invoke(() => Editor.ResetBackground());

        IsInferenceNow = true;
        AIDetectBtn.IsEnabled = false;

        DetectionCancellationSource = new CancellationTokenSource();

        var files = (FilteredMediaFiles.Count == 0 ? AllMediaFiles : FilteredMediaFiles)
            .Skip(LvFiles.SelectedIndex)
            .Select(x => x.Path)
            .ToList();
        if (files.Count == 0)
            return;

        await _inferenceService.RunInference(files, DetectionCancellationSource.Token);

        LvFiles.Items.Refresh();
        IsInferenceNow = false;
        StatusHelp.Text = "Розпізнавання зваершено";
        AIDetectBtn.IsEnabled = true;
    }

    private void SwitchGpsPanel(object sender, RoutedEventArgs e)
    {
        _gpsPanelVisible = !_gpsPanelVisible;

        if (_gpsPanelVisible)
        {
            GpsSplitterRow.Height = new GridLength(4);
            GpsSplitter.Visibility = Visibility.Visible;

            GpsSectionRow.Height = new GridLength(1, GridUnitType.Star);
            MapMatcherComponent.Visibility = Visibility.Visible;
        }
        else
        {
            GpsSplitterRow.Height = new GridLength(0);
            GpsSplitter.Visibility = Visibility.Collapsed;

            GpsSectionRow.Height = new GridLength(0);
            MapMatcherComponent.Visibility = Visibility.Collapsed;
        }
    }

    #region Denys Wishes

    private void SoundDetections(object sender, RoutedEventArgs e)
    {
        MessageBox.Show("Функція Аудіоаналіз знаходиться в стадії розробки","Система", MessageBoxButton.OK, MessageBoxImage.Information);
        _logger.LogInformation("Denys wishes #1. To be implemented");
    }

    private void RunDroneMaintenance(object sender, RoutedEventArgs e)
    {
        MessageBox.Show("Функція Аналіз стану БПЛА знаходиться в стадії розробки","Система", MessageBoxButton.OK, MessageBoxImage.Information);
        _logger.LogInformation("Denys wishes #2. To be implemented");
    }

    #endregion
}

public class GradientStyleSelector : StyleSelector
{
    public override Style? SelectStyle(object item, DependencyObject container)
    {
        if (container is not DataGridRow row || row.DataContext is not AnnotationResult result)
            return null;

        var style = new Style(typeof(DataGridRow));
        var brush = new LinearGradientBrush
        {
            StartPoint = new Point(0, 0),
            EndPoint = new Point(1, 0)
        };

        var gradients = new List<GradientStop>();
        if (result.Colors.Count == 0)
        {
            var color = (Color)ColorConverter.ConvertFromString("#40DDDDDD");
            gradients = [new GradientStop(color, 0.99)];
        }
        else
        {
            var increment = 1.0 / result.Colors.Count;
            var currentStop = increment;
            foreach (var c in result.Colors)
            {
                var resultColor = c.Color.ToConfidenceColor(c.Confidence);
                brush.GradientStops.Add(new GradientStop(resultColor, currentStop));
                currentStop += increment;
            }
        }
        foreach (var gradientStop in gradients)
            brush.GradientStops.Add(gradientStop);

        style.Setters.Add(new Setter(DataGridRow.BackgroundProperty, brush));
        return style;
    }
}