annotations/Azaion.Annotator/Annotator.xaml.cs

using System.Collections.ObjectModel;
using System.Diagnostics;
using System.IO;
using System.Windows;
using System.Windows.Controls;
using System.Windows.Controls.Primitives;
using System.Windows.Input;
using System.Windows.Media;
using Azaion.Annotator.DTO;
using Azaion.Common;
using Azaion.Common.Database;
using Azaion.Common.DTO;
using Azaion.Common.DTO.Config;
using Azaion.Common.Events;
using Azaion.Common.Extensions;
using Azaion.Common.Services;
using Azaion.Common.Services.Inference;
using LibVLCSharp.Shared;
using MediatR;
using Microsoft.WindowsAPICodePack.Dialogs;
using Size = System.Windows.Size;
using IntervalTree;
using LinqToDB;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MediaPlayer = LibVLCSharp.Shared.MediaPlayer;

namespace Azaion.Annotator;

public partial class Annotator
{
    private readonly AppConfig? _appConfig;
    private readonly LibVLC _libVlc;
    private readonly MediaPlayer _mediaPlayer;
    private readonly IMediator _mediator;
    private readonly FormState _formState;

    private readonly IConfigUpdater _configUpdater;
    private readonly HelpWindow _helpWindow;
    private readonly ILogger<Annotator> _logger;
    private readonly IDbFactory _dbFactory;
    private readonly IInferenceService _inferenceService;
    private readonly IInferenceClient _inferenceClient;

    private bool _suspendLayout;
    private bool _gpsPanelVisible;

    private readonly CancellationTokenSource _mainCancellationSource = new();
    public CancellationTokenSource DetCancelSource = new();
    private bool _isInferenceNow;

    private readonly TimeSpan _thresholdBefore = TimeSpan.FromMilliseconds(50);
    private readonly TimeSpan _thresholdAfter = TimeSpan.FromMilliseconds(150);

    public ObservableCollection<MediaFileInfo> AllMediaFiles { get; set; } = new();
    private ObservableCollection<MediaFileInfo> FilteredMediaFiles { get; set; } = new();
    public Dictionary<string, MediaFileInfo> MediaFilesDict = new();

    public IntervalTree<TimeSpan, Annotation> TimedAnnotations { get; set; } = new();
    public string MainTitle { get; set; }

    public CameraConfig Camera => _appConfig?.CameraConfig ?? new CameraConfig();

    public Annotator(
        IConfigUpdater configUpdater,
        IOptions<AppConfig> appConfig,
        LibVLC libVlc,
        MediaPlayer mediaPlayer,
        IMediator mediator,
        FormState formState,
        HelpWindow helpWindow,
        ILogger<Annotator> logger,
        IDbFactory dbFactory,
        IInferenceService inferenceService,
        IInferenceClient inferenceClient,
        IGpsMatcherService gpsMatcherService)
    {
        // Initialize configuration and services BEFORE InitializeComponent so bindings can see real values
        _appConfig = appConfig.Value;
        _configUpdater = configUpdater;
        _libVlc = libVlc;
        _mediaPlayer = mediaPlayer;
        _mediator = mediator;
        _formState = formState;
        _helpWindow = helpWindow;
        _logger = logger;
        _dbFactory = dbFactory;
        _inferenceService = inferenceService;
        _inferenceClient = inferenceClient;

        // Ensure bindings (e.g., Camera) resolve immediately
        DataContext = this;

        InitializeComponent();

        MainTitle = $"Azaion Annotator {Constants.GetLocalVersion()}";
        Title = MainTitle;

        Loaded += OnLoaded;
        Closed += OnFormClosed;
        Activated += (_, _) => _formState.ActiveWindow = WindowEnum.Annotator;
        TbFolder.TextChanged += async (_, _) =>
        {
            if (!Path.Exists(TbFolder.Text))
                return;
            try
            {
                _appConfig.DirectoriesConfig.VideosDirectory = TbFolder.Text;
                await ReloadFiles();
            }
            catch (Exception e)
            {
                _logger.LogError(e, e.Message);
            }
        };
        Editor.GetTimeFunc = () => TimeSpan.FromMilliseconds(_mediaPlayer.Time);
        MapMatcherComponent.Init(_appConfig, gpsMatcherService);

        // When camera settings change, persist config
        CameraConfigControl.CameraChanged += (_, _) =>
        {
            if (_appConfig != null)
                _configUpdater.Save(_appConfig);
        };
    }

    private void OnLoaded(object sender, RoutedEventArgs e)
    {
        Core.Initialize();
        InitControls();

        _suspendLayout = true;

        MainGrid.ColumnDefinitions.FirstOrDefault()!.Width = new GridLength(_appConfig?.UIConfig.LeftPanelWidth ?? Constants.DEFAULT_LEFT_PANEL_WIDTH);
        MainGrid.ColumnDefinitions.LastOrDefault()!.Width = new GridLength(_appConfig?.UIConfig.RightPanelWidth ?? Constants.DEFAULT_RIGHT_PANEL_WIDTH);

        _suspendLayout = false;
        TbFolder.Text = _appConfig?.DirectoriesConfig.VideosDirectory ?? Constants.DEFAULT_VIDEO_DIR;

        LvClasses.Init(_appConfig?.AnnotationConfig.DetectionClasses ?? Constants.DefaultAnnotationClasses);
    }

    public void BlinkHelp(string helpText, int times = 2)
    {
        _ = Task.Run(async () =>
        {
            for (int i = 0; i < times; i++)
            {
                Dispatcher.Invoke(() => StatusHelp.Text = helpText);
                await Task.Delay(200);
                Dispatcher.Invoke(() => StatusHelp.Text = "");
                await Task.Delay(200);
            }

            Dispatcher.Invoke(() => StatusHelp.Text = helpText);
        });
    }

    private void InitControls()
    {
        VideoView.MediaPlayer = _mediaPlayer;

        //On start playing media
        _mediaPlayer.Playing += (_, _) =>
        {
            uint vw = 0, vh = 0;
            _mediaPlayer.Size(0, ref vw, ref vh);
            _formState.CurrentMediaSize = new Size(vw, vh);
            _formState.CurrentVideoLength = TimeSpan.FromMilliseconds(_mediaPlayer.Length);
        };

        LvFiles.MouseDoubleClick += async (_, _) =>
        {
            await _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.Play));
        };

        LvClasses.DetectionClassChanged += (_, args) =>
        {
            var selectedClass = args.DetectionClass;
            Editor.CurrentAnnClass = selectedClass;
            _mediator.Publish(new AnnClassSelectedEvent(selectedClass));
       };

        _mediaPlayer.PositionChanged += (_, _) =>
            ShowTimeAnnotations(TimeSpan.FromMilliseconds(_mediaPlayer.Time));

        VideoSlider.ValueChanged += (_, newValue) =>
            _mediaPlayer.Position = (float)(newValue / VideoSlider.Maximum);

        VideoSlider.KeyDown += (sender, args) =>
            _mediator.Publish(new KeyEvent(sender, args, WindowEnum.Annotator));

        Volume.ValueChanged += (_, newValue) =>
            _mediator.Publish(new VolumeChangedEvent((int)newValue));

        DgAnnotations.MouseDoubleClick += (sender, args) =>
        {
            if (ItemsControl.ContainerFromElement((DataGrid)sender, (args.OriginalSource as DependencyObject)!) is DataGridRow dgRow)
                OpenAnnotationResult((Annotation)dgRow.Item);

        };

        DgAnnotations.KeyUp += async (_, args) =>
        {
            switch (args.Key)
            {
                case Key.Down: //cursor is already moved by system behaviour
                    OpenAnnotationResult((Annotation)DgAnnotations.SelectedItem);
                    break;
                case Key.Delete:
                    var result = MessageBox.Show("Чи дійсно видалити аннотації?","Підтвердження видалення", MessageBoxButton.OKCancel, MessageBoxImage.Question);
                    if (result != MessageBoxResult.OK)
                        return;

                    var res = DgAnnotations.SelectedItems.Cast<Annotation>().ToList();
                    var annotationNames = res.Select(x => x.Name).ToList();

                    await _mediator.Publish(new AnnotationsDeletedEvent(annotationNames));
                    break;
            }
        };
        DgAnnotations.ItemsSource = _formState.AnnotationResults;
    }

    private void OpenAnnotationResult(Annotation ann)
    {
        _mediaPlayer.SetPause(true);
        if (!ann.IsSplit)
            Editor.RemoveAllAnns();

        _mediaPlayer.Time = (long)ann.Time.TotalMilliseconds;

        Dispatcher.Invoke(() =>
        {
            VideoSlider.Value = _mediaPlayer.Position * VideoSlider.Maximum;
            StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}";
            Editor.ClearExpiredAnnotations(ann.Time);
        });

        ShowAnnotation(ann, showImage: true, openResult: true);
    }
    private void SaveUserSettings()
    {
        if (_suspendLayout || _appConfig is null)
            return;

        _appConfig.UIConfig.LeftPanelWidth = MainGrid.ColumnDefinitions.FirstOrDefault()!.Width.Value;
        _appConfig.UIConfig.RightPanelWidth = MainGrid.ColumnDefinitions.LastOrDefault()!.Width.Value;

        _configUpdater.Save(_appConfig);
    }

    public void ShowTimeAnnotations(TimeSpan time, bool showImage = false)
    {
        Dispatcher.Invoke(() =>
        {
            VideoSlider.Value = _mediaPlayer.Position * VideoSlider.Maximum;
            StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}";
            Editor.ClearExpiredAnnotations(time);
        });
        var annotations = TimedAnnotations.Query(time).ToList();
        if (!annotations.Any())
            return;
        foreach (var ann in annotations)
            ShowAnnotation(ann, showImage);
    }

    private void ShowAnnotation(Annotation annotation, bool showImage = false, bool openResult = false)
    {
        Dispatcher.Invoke(async () =>
        {
            if (showImage && !annotation.IsSplit && File.Exists(annotation.ImagePath))
            {
                Editor.SetBackground(await annotation.ImagePath.OpenImage());
                _formState.BackgroundTime = annotation.Time;
            }

            if (annotation.SplitTile != null && openResult)
            {
                var canvasTileLocation = new CanvasLabel(new YoloLabel(annotation.SplitTile, _formState.CurrentMediaSize),
                        RenderSize);
                Editor.ZoomTo(new Point(canvasTileLocation.CenterX, canvasTileLocation.CenterY));
            }
            else
                Editor.CreateDetections(annotation, _appConfig?.AnnotationConfig.DetectionClasses ?? [], _formState.CurrentMediaSize);
        });
    }

    public async Task ReloadAnnotations()
    {
        await Dispatcher.InvokeAsync(async () =>
        {
            _formState.AnnotationResults.Clear();
            TimedAnnotations.Clear();
            Editor.RemoveAllAnns();

            var annotations = await _dbFactory.Run(async db =>
                await db.Annotations.LoadWith(x => x.Detections)
                    .Where(x => x.OriginalMediaName == _formState.MediaName)
                    .OrderBy(x => x.Time)
                    .ToListAsync(token: _mainCancellationSource.Token));
            
            TimedAnnotations.Clear();
            _formState.AnnotationResults.Clear();
            foreach (var ann in annotations)
            {
                // Duplicate for speed
                TimedAnnotations.Add(ann.Time.Subtract(_thresholdBefore), ann.Time.Add(_thresholdAfter), ann);
                _formState.AnnotationResults.Add(ann);
            }
        });
    }

    //Add manually
    public void AddAnnotation(Annotation annotation)
    {
        var time = annotation.Time;
        var previousAnnotations = TimedAnnotations.Query(time);
        TimedAnnotations.Remove(previousAnnotations);
        TimedAnnotations.Add(time.Subtract(_thresholdBefore), time.Add(_thresholdAfter), annotation);

        var existingResult = _formState.AnnotationResults.FirstOrDefault(x => x.Time == time);
        if (existingResult != null)
        {
            try
            {
                _formState.AnnotationResults.Remove(existingResult);
            }
            catch (Exception e)
            {
                _logger.LogError(e, e.Message);
                throw;
            }
        }

        var dict = _formState.AnnotationResults
            .Select((x, i) => new { x.Time, Index = i })
            .ToDictionary(x => x.Time, x => x.Index);

        var index = dict.Where(x => x.Key < time)
            .OrderBy(x => time - x.Key)
            .Select(x => x.Value + 1)
            .FirstOrDefault();
        _formState.AnnotationResults.Insert(index, annotation);
    }

    private async Task ReloadFiles()
    {
        var dir = new DirectoryInfo(_appConfig?.DirectoriesConfig.VideosDirectory ?? Constants.DEFAULT_VIDEO_DIR);
        if (!dir.Exists)
            return;

        var videoFiles = dir.GetFiles((_appConfig?.AnnotationConfig.VideoFormats ?? Constants.DefaultVideoFormats)
                .ToArray()).Select(x =>
        {
            var media = new Media(_libVlc, x.FullName);
            media.Parse();
            var fInfo = new MediaFileInfo
            {
                Name = x.Name,
                Path = x.FullName,
                MediaType = MediaTypes.Video
            };
            media.ParsedChanged += (_, _) => fInfo.Duration = TimeSpan.FromMilliseconds(media.Duration);
            return fInfo;
        }).ToList();

        var imageFiles = dir.GetFiles((_appConfig?.AnnotationConfig.ImageFormats ?? Constants.DefaultImageFormats).ToArray())
            .Select(x => new MediaFileInfo
            {
                Name = x.Name,
                Path = x.FullName,
                MediaType = MediaTypes.Image
            });
        var allFiles = videoFiles.Concat(imageFiles).ToList();

        var allFileNames = allFiles.Select(x => x.FName).ToList();

        var labelsDict = await _dbFactory.Run(async db => 
            await db.Annotations
            .GroupBy(x => x.OriginalMediaName)
            .Where(x => allFileNames.Contains(x.Key))
            .Select(x => x.Key)
            .ToDictionaryAsync(x => x, x => x));
        
        foreach (var mediaFile in allFiles)
            mediaFile.HasAnnotations = labelsDict.ContainsKey(mediaFile.FName);
            
        AllMediaFiles = new ObservableCollection<MediaFileInfo>(allFiles);
        MediaFilesDict = AllMediaFiles.GroupBy(x => x.Name)
            .ToDictionary(gr => gr.Key, gr => gr.First());
        LvFiles.ItemsSource = AllMediaFiles;
        DataContext = this;
    }

    private void OnFormClosed(object? sender, EventArgs e)
    {
        _mainCancellationSource.Cancel();
        _inferenceService.StopInference();
        DetCancelSource.Cancel();

        _mediaPlayer.Stop();
        _mediaPlayer.Dispose();
        _libVlc.Dispose();
    }

    private void OpenContainingFolder(object sender, RoutedEventArgs e)
    {
        var mediaFileInfo = (sender as MenuItem)?.DataContext as MediaFileInfo;
        if (mediaFileInfo == null)
            return;

        Process.Start("explorer.exe", "/select,\"" + mediaFileInfo.Path +"\"");
    }

    public void SeekTo(long timeMilliseconds, bool setPause = true)
    {
        _mediaPlayer.SetPause(setPause);
        _mediaPlayer.Time = timeMilliseconds;
        VideoSlider.Value = _mediaPlayer.Position * 100;

        StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}";
    }

    private void OpenFolderItemClick(object sender, RoutedEventArgs e) => OpenFolder();
    private void OpenFolderButtonClick(object sender, RoutedEventArgs e) => OpenFolder();

    private void OpenFolder()
    {
        var dlg = new CommonOpenFileDialog
        {
            Title = "Open Video folder",
            IsFolderPicker = true,
            InitialDirectory = Path.GetDirectoryName(_appConfig?.DirectoriesConfig.VideosDirectory ?? Constants.DEFAULT_VIDEO_DIR)
        };
        var dialogResult = dlg.ShowDialog();

        if (dialogResult != CommonFileDialogResult.Ok || string.IsNullOrEmpty(dlg.FileName))
            return;

        if (_appConfig is not null)
            _appConfig.DirectoriesConfig.VideosDirectory = dlg.FileName;

        TbFolder.Text = dlg.FileName;
    }

    private void TbFilter_OnTextChanged(object sender, TextChangedEventArgs e)
    {
        FilteredMediaFiles = new ObservableCollection<MediaFileInfo>(AllMediaFiles.Where(x => x.Name.ToLower().Contains(TbFilter.Text.ToLower())).ToList());
        MediaFilesDict = FilteredMediaFiles.ToDictionary(x => x.FName);
        LvFiles.ItemsSource = FilteredMediaFiles;
        LvFiles.ItemsSource = FilteredMediaFiles;
    }

    private void PlayClick(object sender, RoutedEventArgs e)
    {
        _mediator.Publish(new AnnotatorControlEvent(_mediaPlayer.CanPause ? PlaybackControlEnum.Pause : PlaybackControlEnum.Play));
    }

    private void PauseClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.Pause));
    private void StopClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.Stop));

    private void PreviousFrameClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.PreviousFrame));
    private void NextFrameClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.NextFrame));

    private void SaveAnnotationsClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.SaveAnnotations));

    private void RemoveSelectedClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.RemoveSelectedAnns));
    private void RemoveAllClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.RemoveAllAnns));
    private void TurnOffVolume(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.TurnOffVolume));
    private void TurnOnVolume(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.TurnOnVolume));

    private void OpenHelpWindowClick(object sender, RoutedEventArgs e)
    {
        _helpWindow.Show();
        _helpWindow.Activate();
    }

    private void Thumb_OnDragCompleted(object sender, DragCompletedEventArgs e) => SaveUserSettings();

    private void LvFilesContextOpening(object sender, ContextMenuEventArgs e)
    {
        var listItem = sender as ListViewItem;
        LvFilesContextMenu.DataContext = listItem!.DataContext;
    }

    private async void AIDetectBtn_OnClick(object sender, RoutedEventArgs e)
    {
        try
        {
            await AutoDetect();
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, ex.Message);
        }
    }

    public async Task AutoDetect()
    {
        if (_isInferenceNow)
            return;

        if (LvFiles.Items.IsEmpty)
            return;
        if (LvFiles.SelectedIndex == -1)
            LvFiles.SelectedIndex = 0;

        Dispatcher.Invoke(() => Editor.SetBackground(null));

        _isInferenceNow = true;
        AIDetectBtn.IsEnabled = false;

        DetCancelSource = new CancellationTokenSource();

        var files = (FilteredMediaFiles.Count == 0 ? AllMediaFiles : FilteredMediaFiles)
            .Skip(LvFiles.SelectedIndex)
            .Select(x => x.Path)
            .ToList();
        if (files.Count == 0)
            return;

        await _inferenceService.RunInference(files, _appConfig?.CameraConfig ?? Constants.DefaultCameraConfig, DetCancelSource.Token);

        LvFiles.Items.Refresh();
        _isInferenceNow = false;
        StatusHelp.Text = "Розпізнавання завершено";
        AIDetectBtn.IsEnabled = true;
    }

    private void SwitchGpsPanel(object sender, RoutedEventArgs e)
    {
        _gpsPanelVisible = !_gpsPanelVisible;

        if (_gpsPanelVisible)
        {
            GpsSplitterRow.Height = new GridLength(4);
            GpsSplitter.Visibility = Visibility.Visible;

            GpsSectionRow.Height = new GridLength(1, GridUnitType.Star);
            MapMatcherComponent.Visibility = Visibility.Visible;
        }
        else
        {
            GpsSplitterRow.Height = new GridLength(0);
            GpsSplitter.Visibility = Visibility.Collapsed;

            GpsSectionRow.Height = new GridLength(0);
            MapMatcherComponent.Visibility = Visibility.Collapsed;
        }
    }

    #region Denys Wishes

    private void SoundDetections(object sender, RoutedEventArgs e)
    {
        MessageBox.Show("Функція Аудіоаналіз знаходиться в стадії розробки","Система", MessageBoxButton.OK, MessageBoxImage.Information);
    }

    private void RunDroneMaintenance(object sender, RoutedEventArgs e)
    {
        MessageBox.Show("Функція Аналіз стану БПЛА знаходиться в стадії розробки","Система", MessageBoxButton.OK, MessageBoxImage.Information);
    }

    #endregion

    private void DeleteMedia(object sender, RoutedEventArgs e)
    {
        var mediaFileInfo = (sender as MenuItem)?.DataContext as MediaFileInfo;
        if (mediaFileInfo == null)
            return;
        DeleteMedia(mediaFileInfo);
    }

    public void DeleteMedia(MediaFileInfo mediaFileInfo)
    {
        var obj = mediaFileInfo.MediaType == MediaTypes.Image
            ? "цю картинку "
            : "це відео ";
        var result = MessageBox.Show($"Видалити {obj}?",
            "Підтвердження видалення", MessageBoxButton.YesNo, MessageBoxImage.Warning);
        if (result != MessageBoxResult.Yes)
            return;

        File.Delete(mediaFileInfo.Path);
        AllMediaFiles.Remove(mediaFileInfo);
    }
}

public class GradientStyleSelector : StyleSelector
{
    public override Style? SelectStyle(object item, DependencyObject container)
    {
        if (container is not DataGridRow row || row.DataContext is not Annotation result)
            return null;

        var style = new Style(typeof(DataGridRow));
        var brush = new LinearGradientBrush
        {
            StartPoint = new Point(0, 0),
            EndPoint = new Point(1, 0)
        };

        var gradients = new List<GradientStop>();
        if (result.Colors.Count == 0)
        {
            var color = (Color)ColorConverter.ConvertFromString("#40DDDDDD");
            gradients = [new GradientStop(color, 0.99)];
        }
        else
        {
            var increment = 1.0 / result.Colors.Count;
            var currentStop = increment;
            foreach (var c in result.Colors)
            {
                var resultColor = c.Color.ToConfidenceColor(c.Confidence);
                brush.GradientStops.Add(new GradientStop(resultColor, currentStop));
                currentStop += increment;
            }
        }
        foreach (var gradientStop in gradients)
            brush.GradientStops.Add(gradientStop);

        style.Setters.Add(new Setter(Control.BackgroundProperty, brush));
        return style;
    }
}