using System.IO; using System.Windows; using System.Windows.Controls; using System.Windows.Input; using System.Windows.Media; using System.Windows.Media.Imaging; using Azaion.Annotator.Controls; using Azaion.Annotator.DTO; using Azaion.Common; using Azaion.Common.Database; using Azaion.Common.DTO; using Azaion.Common.DTO.Config; using Azaion.Common.Events; using Azaion.Common.Extensions; using Azaion.Common.Services; using Azaion.Common.Services.Inference; using GMap.NET; using GMap.NET.WindowsPresentation; using LibVLCSharp.Shared; using MediatR; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MediaPlayer = LibVLCSharp.Shared.MediaPlayer; namespace Azaion.Annotator; public class AnnotatorEventHandler( LibVLC libVlc, MediaPlayer mediaPlayer, Annotator mainWindow, FormState formState, IAnnotationService annotationService, ILogger logger, IOptions dirConfig, IOptions annotationConfig, IInferenceService inferenceService, IDbFactory dbFactory, IAzaionApi api, FailsafeAnnotationsProducer producer, IAnnotationPathResolver pathResolver, IFileSystem fileSystem, IUICommandDispatcher uiDispatcher) : INotificationHandler, INotificationHandler, INotificationHandler, INotificationHandler, INotificationHandler, INotificationHandler, INotificationHandler, INotificationHandler, INotificationHandler { private const int STEP = 20; private const int LARGE_STEP = 5000; private readonly string _tempImgPath = Path.Combine(dirConfig.Value.ImagesDirectory, "___temp___.jpg"); private readonly Dictionary _keysControlEnumDict = new() { { Key.Space, PlaybackControlEnum.Pause }, { Key.Left, PlaybackControlEnum.PreviousFrame }, { Key.Right, PlaybackControlEnum.NextFrame }, { Key.Enter, PlaybackControlEnum.SaveAnnotations }, { Key.Delete, PlaybackControlEnum.RemoveSelectedAnns }, { Key.X, PlaybackControlEnum.RemoveAllAnns }, { Key.PageUp, PlaybackControlEnum.Previous }, { Key.PageDown, PlaybackControlEnum.Next }, }; public async Task Handle(AnnClassSelectedEvent notification, CancellationToken ct) { SelectClass(notification.DetectionClass); await Task.CompletedTask; } private void SelectClass(DetectionClass detClass) { mainWindow.Editor.CurrentAnnClass = detClass; foreach (var ann in mainWindow.Editor.CurrentDetections.Where(x => x.IsSelected)) ann.DetectionClass = detClass; mainWindow.LvClasses.SelectNum(detClass.Id); } public async Task Handle(KeyEvent keyEvent, CancellationToken ct = default) { if (keyEvent.WindowEnum != WindowEnum.Annotator) return; var key = keyEvent.Args.Key; var keyNumber = (int?)null; if ((int)key >= (int)Key.D1 && (int)key <= (int)Key.D9) keyNumber = key - Key.D1; if ((int)key >= (int)Key.NumPad1 && (int)key <= (int)Key.NumPad9) keyNumber = key - Key.NumPad1; if (keyNumber.HasValue) SelectClass((DetectionClass)mainWindow.LvClasses.DetectionDataGrid.Items[keyNumber.Value]!); if (_keysControlEnumDict.TryGetValue(key, out var value)) await ControlPlayback(value, ct); if (key == Key.R) await mainWindow.AutoDetect(); #region Volume switch (key) { case Key.VolumeMute when mediaPlayer.Volume == 0: await ControlPlayback(PlaybackControlEnum.TurnOnVolume, ct); break; case Key.VolumeMute: await ControlPlayback(PlaybackControlEnum.TurnOffVolume, ct); break; case Key.Up: case Key.VolumeUp: var vUp = Math.Min(100, mediaPlayer.Volume + 5); ChangeVolume(vUp); mainWindow.Volume.Value = vUp; break; case Key.Down: case Key.VolumeDown: var vDown = Math.Max(0, mediaPlayer.Volume - 5); ChangeVolume(vDown); mainWindow.Volume.Value = vDown; break; } #endregion } public async Task Handle(AnnotatorControlEvent notification, CancellationToken ct = default) { await ControlPlayback(notification.PlaybackControl, ct); mainWindow.VideoView.Focus(); } private async Task ControlPlayback(PlaybackControlEnum controlEnum, CancellationToken cancellationToken = default) { try { var isCtrlPressed = Keyboard.IsKeyDown(Key.LeftCtrl) || Keyboard.IsKeyDown(Key.RightCtrl); var step = isCtrlPressed ? LARGE_STEP : STEP; switch (controlEnum) { case PlaybackControlEnum.Play: await Play(cancellationToken); break; case PlaybackControlEnum.Pause: if (mediaPlayer.IsPlaying) { mediaPlayer.Pause(); mediaPlayer.TakeSnapshot(0, _tempImgPath, 0, 0); mainWindow.Editor.SetBackground(await _tempImgPath.OpenImage()); formState.BackgroundTime = TimeSpan.FromMilliseconds(mediaPlayer.Time); } else { mediaPlayer.Play(); if (formState.BackgroundTime.HasValue) { mainWindow.Editor.SetBackground(null); formState.BackgroundTime = null; } } break; case PlaybackControlEnum.Stop: inferenceService.StopInference(); await mainWindow.DetCancelSource.CancelAsync(); mediaPlayer.Stop(); break; case PlaybackControlEnum.PreviousFrame: mainWindow.SeekTo(mediaPlayer.Time - step); break; case PlaybackControlEnum.NextFrame: mainWindow.SeekTo(mediaPlayer.Time + step); break; case PlaybackControlEnum.SaveAnnotations: await SaveAnnotation(cancellationToken); break; case PlaybackControlEnum.RemoveSelectedAnns: var focusedElement = FocusManager.GetFocusedElement(mainWindow); if (focusedElement is ListViewItem item) { if (item.DataContext is not MediaFile mediaFileInfo) return; mainWindow.DeleteMedia(mediaFileInfo); } else mainWindow.Editor.RemoveSelectedAnns(); break; case PlaybackControlEnum.RemoveAllAnns: mainWindow.Editor.RemoveAllAnns(); break; case PlaybackControlEnum.TurnOnVolume: mainWindow.TurnOnVolumeBtn.Visibility = Visibility.Collapsed; mainWindow.TurnOffVolumeBtn.Visibility = Visibility.Visible; mediaPlayer.Volume = formState.CurrentVolume; break; case PlaybackControlEnum.TurnOffVolume: mainWindow.TurnOffVolumeBtn.Visibility = Visibility.Collapsed; mainWindow.TurnOnVolumeBtn.Visibility = Visibility.Visible; formState.CurrentVolume = mediaPlayer.Volume; mediaPlayer.Volume = 0; break; case PlaybackControlEnum.Previous: await NextMedia(isPrevious: true, ct: cancellationToken); break; case PlaybackControlEnum.Next: await NextMedia(ct: cancellationToken); break; case PlaybackControlEnum.None: break; default: throw new ArgumentOutOfRangeException(nameof(controlEnum), controlEnum, null); } } catch (Exception e) { logger.LogError(e, e.Message); throw; } } private async Task NextMedia(bool isPrevious = false, CancellationToken ct = default) { var increment = isPrevious ? -1 : 1; var check = isPrevious ? -1 : mainWindow.LvFiles.Items.Count; if (mainWindow.LvFiles.SelectedIndex + increment == check) return; mainWindow.LvFiles.SelectedIndex += increment; await Play(ct); } public async Task Handle(VolumeChangedEvent notification, CancellationToken ct) { ChangeVolume(notification.Volume); await Task.CompletedTask; } private void ChangeVolume(int volume) { formState.CurrentVolume = volume; mediaPlayer.Volume = volume; } private async Task Play(CancellationToken ct = default) { if (mainWindow.LvFiles.SelectedItem == null) return; var mediaInfo = (MediaFile)mainWindow.LvFiles.SelectedItem; if (formState.CurrentMedia == mediaInfo) return; //already loaded formState.CurrentMedia = mediaInfo; mainWindow.Title = $"{mainWindow.MainTitle} - {mediaInfo.Name}"; await mainWindow.ReloadAnnotations(); if (mediaInfo.MediaType == MediaTypes.Video) { mainWindow.Editor.SetBackground(null); //need to wait a bit for correct VLC playback event handling await Task.Delay(100, ct); mediaPlayer.Stop(); mediaPlayer.Play(new Media(libVlc, mediaInfo.MediaUrl)); } else { formState.BackgroundTime = TimeSpan.Zero; var image = await mediaInfo.MediaUrl.OpenImage(); formState.CurrentMediaSize = new Size(image.PixelWidth, image.PixelHeight); mainWindow.Editor.SetBackground(image); mediaPlayer.Stop(); mainWindow.ShowTimeAnnotations(TimeSpan.Zero, showImage: true); } } //SAVE: MANUAL private async Task SaveAnnotation(CancellationToken cancellationToken = default) { if (formState.CurrentMedia == null) return; var time = formState.BackgroundTime ?? TimeSpan.FromMilliseconds(mediaPlayer.Time); var timeName = formState.CurrentMedia.Name.ToTimeName(time); var isVideo = formState.CurrentMedia.MediaType == MediaTypes.Video; var imgPath = Path.Combine(dirConfig.Value.ImagesDirectory, $"{timeName}{Constants.JPG_EXT}"); var annotations = await SaveAnnotationInner(imgPath, cancellationToken); if (isVideo) { foreach (var annotation in annotations) mainWindow.AddAnnotation(annotation); mediaPlayer.Play(); // next item. Probably not needed // var annGrid = mainWindow.DgAnnotations; // annGrid.SelectedIndex = Math.Min(annGrid.Items.Count, annGrid.SelectedIndex + 1); // mainWindow.OpenAnnotationResult((AnnotationResult)annGrid.SelectedItem); mainWindow.Editor.SetBackground(null); formState.BackgroundTime = null; } else { await NextMedia(ct: cancellationToken); } mainWindow.LvFiles.Items.Refresh(); mainWindow.Editor.RemoveAllAnns(); } private async Task> SaveAnnotationInner(string imgPath, CancellationToken ct = default) { var canvasDetections = mainWindow.Editor.CurrentDetections.Select(x => x.ToCanvasLabel()).ToList(); var source = (mainWindow.Editor.BackgroundImage.Source as BitmapSource)!; var mediaSize = new Size(source.PixelWidth, source.PixelHeight); var annotationsResult = new List(); var time = formState.BackgroundTime ?? TimeSpan.FromMilliseconds(mediaPlayer.Time); if (!fileSystem.FileExists(imgPath)) { if (mediaSize.FitSizeForAI()) await source.SaveImage(imgPath, ct); else { //Tiling //1. Convert from RenderSize to CurrentMediaSize var detectionCoords = canvasDetections.Select(x => new CanvasLabel( new YoloLabel(x, mainWindow.Editor.RenderSize, formState.CurrentMediaSize), formState.CurrentMediaSize, null, x.Confidence)) .ToList(); //2. Split to frames var results = TileProcessor.Split(formState.CurrentMediaSize, detectionCoords, ct); //3. Save each frame as a separate annotation foreach (var res in results) { var annotationName = $"{formState.CurrentMedia?.Name ?? ""}{Constants.SPLIT_SUFFIX}{res.Tile.Width}_{res.Tile.Left:0000}_{res.Tile.Top:0000}!".ToTimeName(time); var tempAnnotation = new Annotation { Name = annotationName, ImageExtension = Constants.JPG_EXT }; var tileImgPath = pathResolver.GetImagePath(tempAnnotation); var bitmap = new CroppedBitmap(source, new Int32Rect((int)res.Tile.Left, (int)res.Tile.Top, (int)res.Tile.Width, (int)res.Tile.Height)); await bitmap.SaveImage(tileImgPath, ct); var frameSize = new Size(res.Tile.Width, res.Tile.Height); var detections = res.Detections .Select(det => det.ReframeToSmall(res.Tile)) .Select(x => new Detection(annotationName, new YoloLabel(x, frameSize))) .ToList(); annotationsResult.Add(await annotationService.SaveAnnotation(formState.CurrentMediaHash, formState.CurrentMediaName, annotationName, time, detections, token: ct)); } return annotationsResult; } } var annName = (formState.CurrentMedia?.Name ?? "").ToTimeName(time); var currentDetections = canvasDetections.Select(x => new Detection(annName, new YoloLabel(x, mainWindow.Editor.RenderSize, mediaSize))) .ToList(); var annotation = await annotationService.SaveAnnotation(formState.CurrentMediaHash, formState.CurrentMediaName, annName, time, currentDetections, token: ct); return [annotation]; } public async Task Handle(AnnotationsDeletedEvent notification, CancellationToken ct) { try { uiDispatcher.Execute(() => { var namesSet = notification.AnnotationNames.ToHashSet(); var remainAnnotations = formState.AnnotationResults .Where(x => !namesSet.Contains(x.Name)).ToList(); formState.AnnotationResults.Clear(); foreach (var ann in remainAnnotations) formState.AnnotationResults.Add(ann); var timedAnnotationsToRemove = mainWindow.TimedAnnotations .Where(x => namesSet.Contains(x.Value.Name)) .Select(x => x.Value).ToList(); mainWindow.TimedAnnotations.Remove(timedAnnotationsToRemove); mainWindow.ReloadFilesThrottled(); }); await dbFactory.DeleteAnnotations(notification.AnnotationNames, ct); foreach (var name in notification.AnnotationNames) { try { var tempAnnotation = new Annotation { Name = name, ImageExtension = Constants.JPG_EXT }; fileSystem.DeleteFile(pathResolver.GetImagePath(tempAnnotation)); fileSystem.DeleteFile(pathResolver.GetLabelPath(tempAnnotation)); fileSystem.DeleteFile(pathResolver.GetThumbPath(tempAnnotation)); fileSystem.DeleteFile(Path.Combine(dirConfig.Value.ResultsDirectory, $"{name}{Constants.RESULT_PREFIX}{Constants.JPG_EXT}")); } catch (Exception e) { logger.LogError(e, e.Message); } } //Only validators can send Delete to the queue var currentUser = await api.GetCurrentUserAsync(); if (!notification.FromQueue && currentUser.Role.IsValidator()) await producer.SendToInnerQueue(notification.AnnotationNames, AnnotationStatus.Deleted, ct); } catch (Exception e) { logger.LogError(e, e.Message); throw; } } public Task Handle(AnnotationAddedEvent e, CancellationToken ct) { uiDispatcher.Execute(() => { var mediaInfo = (MediaFile)mainWindow.LvFiles.SelectedItem; if ((mediaInfo?.Name ?? "") == e.Annotation.OriginalMediaName) mainWindow.AddAnnotation(e.Annotation); var log = string.Join(Environment.NewLine, e.Annotation.Detections.Select(det => $"Розпізнавання {e.Annotation.OriginalMediaName}: {annotationConfig.Value.DetectionClassesDict[det.ClassNumber].ShortName}: " + $"xy=({det.CenterX:F2},{det.CenterY:F2}), " + $"розмір=({det.Width:F2}, {det.Height:F2}), " + $"conf: {det.Confidence*100:F0}%")); mainWindow.LvFiles.Items.Refresh(); var media = mainWindow.MediaFilesDict.GetValueOrDefault(e.Annotation.OriginalMediaName); if (media != null) mainWindow.ReloadFilesThrottled(); mainWindow.LvFiles.Items.Refresh(); mainWindow.StatusHelp.Text = log; }); return Task.CompletedTask; } public Task Handle(SetStatusTextEvent e, CancellationToken cancellationToken) { uiDispatcher.Execute(() => { mainWindow.StatusHelp.Text = e.Text; mainWindow.StatusHelp.Foreground = e.IsError ? Brushes.Red : Brushes.White; }); return Task.CompletedTask; } public Task Handle(GPSMatcherResultProcessedEvent e, CancellationToken cancellationToken) { uiDispatcher.Execute(() => { var ann = mainWindow.MapMatcherComponent.Annotations[e.Index]; AddMarker(e.GeoPoint, e.Image, Brushes.Blue); if (e.ProcessedGeoPoint != e.GeoPoint) AddMarker(e.ProcessedGeoPoint, $"{e.Image}: corrected", Brushes.DarkViolet); ann.Lat = e.GeoPoint.Lat; ann.Lon = e.GeoPoint.Lon; }); return Task.CompletedTask; } private void AddMarker(GeoPoint point, string text, SolidColorBrush color) { var map = mainWindow.MapMatcherComponent; var pointLatLon = new PointLatLng(point.Lat, point.Lon); var marker = new GMapMarker(pointLatLon); marker.Shape = new CircleVisual(marker, size: 14, text: text, background: color); map.SatelliteMap.Markers.Add(marker); map.SatelliteMap.Position = pointLatLon; map.SatelliteMap.ZoomAndCenterMarkers(null); } public async Task Handle(AIAvailabilityStatusEvent e, CancellationToken cancellationToken) { uiDispatcher.Execute(() => { logger.LogInformation(e.ToString()); mainWindow.AIDetectBtn.IsEnabled = e.Status == AIAvailabilityEnum.Enabled; mainWindow.StatusHelp.Text = e.ToString(); }); if (e.Status is AIAvailabilityEnum.Enabled or AIAvailabilityEnum.Error) await inferenceService.CheckAIAvailabilityTokenSource.CancelAsync(); } }