Files
annotations/Azaion.Annotator/Annotator.xaml.cs
T
Alex Bezdieniezhnykh babcbc0fc7 clean postbuild script
clean warnings
2025-04-28 10:20:06 +03:00

678 lines
25 KiB
C#

using System.Collections.ObjectModel;
using System.Diagnostics;
using System.IO;
using System.Windows;
using System.Windows.Controls;
using System.Windows.Controls.Primitives;
using System.Windows.Input;
using System.Windows.Media;
using Azaion.Annotator.DTO;
using Azaion.Common;
using Azaion.Common.Database;
using Azaion.Common.DTO;
using Azaion.Common.DTO.Config;
using Azaion.Common.Events;
using Azaion.Common.Extensions;
using Azaion.Common.Services;
using LibVLCSharp.Shared;
using MediatR;
using Microsoft.WindowsAPICodePack.Dialogs;
using Size = System.Windows.Size;
using IntervalTree;
using LinqToDB;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MediaPlayer = LibVLCSharp.Shared.MediaPlayer;
namespace Azaion.Annotator;
public partial class Annotator
{
private readonly AppConfig _appConfig;
private readonly LibVLC _libVLC;
private readonly MediaPlayer _mediaPlayer;
private readonly IMediator _mediator;
private readonly FormState _formState;
private readonly IConfigUpdater _configUpdater;
private readonly HelpWindow _helpWindow;
private readonly ILogger<Annotator> _logger;
private readonly AnnotationService _annotationService;
private readonly IDbFactory _dbFactory;
private readonly IInferenceService _inferenceService;
private ObservableCollection<DetectionClass> AnnotationClasses { get; set; } = new();
private bool _suspendLayout;
private bool _gpsPanelVisible = false;
public readonly CancellationTokenSource MainCancellationSource = new();
public CancellationTokenSource DetectionCancellationSource = new();
public bool FollowAI = false;
public bool IsInferenceNow = false;
private readonly TimeSpan _thresholdBefore = TimeSpan.FromMilliseconds(50);
private readonly TimeSpan _thresholdAfter = TimeSpan.FromMilliseconds(150);
private readonly IGpsMatcherService _gpsMatcherService;
private static readonly Guid SaveConfigTaskId = Guid.NewGuid();
public ObservableCollection<MediaFileInfo> AllMediaFiles { get; set; } = new();
public ObservableCollection<MediaFileInfo> FilteredMediaFiles { get; set; } = new();
public IntervalTree<TimeSpan, Annotation> TimedAnnotations { get; set; } = new();
public Annotator(
IConfigUpdater configUpdater,
IOptions<AppConfig> appConfig,
LibVLC libVLC,
MediaPlayer mediaPlayer,
IMediator mediator,
FormState formState,
HelpWindow helpWindow,
ILogger<Annotator> logger,
AnnotationService annotationService,
IDbFactory dbFactory,
IInferenceService inferenceService,
IGpsMatcherService gpsMatcherService)
{
InitializeComponent();
_appConfig = appConfig.Value;
_configUpdater = configUpdater;
_libVLC = libVLC;
_mediaPlayer = mediaPlayer;
_mediator = mediator;
_formState = formState;
_helpWindow = helpWindow;
_logger = logger;
_annotationService = annotationService;
_dbFactory = dbFactory;
_inferenceService = inferenceService;
_gpsMatcherService = gpsMatcherService;
Loaded += OnLoaded;
Closed += OnFormClosed;
Activated += (_, _) => _formState.ActiveWindow = WindowEnum.Annotator;
TbFolder.TextChanged += async (sender, args) =>
{
if (!Path.Exists(TbFolder.Text))
return;
try
{
_appConfig.DirectoriesConfig.VideosDirectory = TbFolder.Text;
await ReloadFiles();
await SaveUserSettings();
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
}
};
Editor.GetTimeFunc = () => TimeSpan.FromMilliseconds(_mediaPlayer.Time);
MapMatcherComponent.Init(_appConfig, _gpsMatcherService);
}
private void OnLoaded(object sender, RoutedEventArgs e)
{
Core.Initialize();
InitControls();
_suspendLayout = true;
MainGrid.ColumnDefinitions.FirstOrDefault()!.Width = new GridLength(_appConfig.UIConfig.LeftPanelWidth);
MainGrid.ColumnDefinitions.LastOrDefault()!.Width = new GridLength(_appConfig.UIConfig.RightPanelWidth);
_suspendLayout = false;
TbFolder.Text = _appConfig.DirectoriesConfig.VideosDirectory;
LvClasses.Init(_appConfig.AnnotationConfig.DetectionClasses);
if (LvFiles.Items.IsEmpty)
BlinkHelp(HelpTexts.HelpTextsDict[HelpTextEnum.Initial]);
}
public void BlinkHelp(string helpText, int times = 2)
{
_ = Task.Run(async () =>
{
for (int i = 0; i < times; i++)
{
Dispatcher.Invoke(() => StatusHelp.Text = helpText);
await Task.Delay(200);
Dispatcher.Invoke(() => StatusHelp.Text = "");
await Task.Delay(200);
}
Dispatcher.Invoke(() => StatusHelp.Text = helpText);
});
}
private void InitControls()
{
VideoView.MediaPlayer = _mediaPlayer;
//On start playing media
_mediaPlayer.Playing += async (sender, args) =>
{
if (_formState.CurrentMrl == _mediaPlayer.Media?.Mrl)
return; //already loaded all the info
_formState.CurrentMrl = _mediaPlayer.Media?.Mrl ?? "";
uint vw = 0, vh = 0;
_mediaPlayer.Size(0, ref vw, ref vh);
_formState.CurrentVideoSize = new Size(vw, vh);
_formState.CurrentVideoLength = TimeSpan.FromMilliseconds(_mediaPlayer.Length);
await Dispatcher.Invoke(async () => await ReloadAnnotations());
if (_formState.CurrentMedia?.MediaType == MediaTypes.Image)
{
await Task.Delay(100); //wait to load the frame and set on pause
ShowTimeAnnotations(TimeSpan.FromMilliseconds(_mediaPlayer.Time));
_mediaPlayer.SetPause(true);
}
};
LvFiles.MouseDoubleClick += async (_, _) =>
{
if (IsInferenceNow)
FollowAI = false;
await _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.Play));
};
LvClasses.DetectionClassChanged += (_, args) =>
{
var selectedClass = args.DetectionClass;
Editor.CurrentAnnClass = selectedClass;
_mediator.Publish(new AnnClassSelectedEvent(selectedClass));
};
_mediaPlayer.PositionChanged += (o, args) =>
ShowTimeAnnotations(TimeSpan.FromMilliseconds(_mediaPlayer.Time));
VideoSlider.ValueChanged += (value, newValue) =>
_mediaPlayer.Position = (float)(newValue / VideoSlider.Maximum);
VideoSlider.KeyDown += (sender, args) =>
_mediator.Publish(new KeyEvent(sender, args, WindowEnum.Annotator));
Volume.ValueChanged += (_, newValue) =>
_mediator.Publish(new VolumeChangedEvent((int)newValue));
SizeChanged += async (_, _) => await SaveUserSettings();
LocationChanged += async (_, _) => await SaveUserSettings();
StateChanged += async (_, _) => await SaveUserSettings();
DgAnnotations.MouseDoubleClick += (sender, args) =>
{
var dgRow = ItemsControl.ContainerFromElement((DataGrid)sender, (args.OriginalSource as DependencyObject)!) as DataGridRow;
if (dgRow != null)
OpenAnnotationResult((AnnotationResult)dgRow!.Item);
};
DgAnnotations.KeyUp += async (sender, args) =>
{
switch (args.Key)
{
case Key.Up:
case Key.Down: //cursor is already moved by system behaviour
OpenAnnotationResult((AnnotationResult)DgAnnotations.SelectedItem);
break;
case Key.Delete:
var result = MessageBox.Show("Чи дійсно видалити аннотації?","Підтвердження видалення", MessageBoxButton.OKCancel, MessageBoxImage.Question);
if (result != MessageBoxResult.OK)
return;
var res = DgAnnotations.SelectedItems.Cast<AnnotationResult>().ToList();
var annotations = res.Select(x => x.Annotation).ToList();
await _mediator.Publish(new AnnotationsDeletedEvent(annotations));
break;
}
};
Editor.Mediator = _mediator;
DgAnnotations.ItemsSource = _formState.AnnotationResults;
}
public void OpenAnnotationResult(AnnotationResult res)
{
if (IsInferenceNow)
FollowAI = false;
_mediaPlayer.SetPause(true);
Editor.RemoveAllAnns();
_mediaPlayer.Time = (long)res.Annotation.Time.TotalMilliseconds;
Dispatcher.Invoke(() =>
{
VideoSlider.Value = _mediaPlayer.Position * VideoSlider.Maximum;
StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}";
Editor.ClearExpiredAnnotations(res.Annotation.Time);
});
ShowAnnotations(res.Annotation, showImage: true);
}
private Task SaveUserSettings()
{
if (_suspendLayout)
return Task.CompletedTask;
_appConfig.UIConfig.LeftPanelWidth = MainGrid.ColumnDefinitions.FirstOrDefault()!.Width.Value;
_appConfig.UIConfig.RightPanelWidth = MainGrid.ColumnDefinitions.LastOrDefault()!.Width.Value;
ThrottleExt.Throttle(() =>
{
_configUpdater.Save(_appConfig);
return Task.CompletedTask;
}, SaveConfigTaskId, TimeSpan.FromSeconds(5));
return Task.CompletedTask;
}
private void ShowTimeAnnotations(TimeSpan time)
{
Dispatcher.Invoke(() =>
{
VideoSlider.Value = _mediaPlayer.Position * VideoSlider.Maximum;
StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}";
Editor.ClearExpiredAnnotations(time);
});
ShowAnnotations(TimedAnnotations.Query(time).FirstOrDefault());
}
private void ShowAnnotations(Annotation? annotation, bool showImage = false)
{
if (annotation == null)
return;
Dispatcher.Invoke(async () =>
{
var videoSize = _formState.CurrentVideoSize;
if (showImage)
{
if (File.Exists(annotation.ImagePath))
{
Editor.Background = new ImageBrush { ImageSource = await annotation.ImagePath.OpenImage() };
_formState.BackgroundTime = annotation.Time;
videoSize = Editor.RenderSize;
}
}
Editor.CreateDetections(annotation.Time, annotation.Detections, _appConfig.AnnotationConfig.DetectionClasses, videoSize);
});
}
private async Task ReloadAnnotations()
{
_formState.AnnotationResults.Clear();
TimedAnnotations.Clear();
Editor.RemoveAllAnns();
var annotations = await _dbFactory.Run(async db =>
await db.Annotations.LoadWith(x => x.Detections)
.Where(x => x.OriginalMediaName == _formState.VideoName)
.OrderBy(x => x.Time)
.ToListAsync(token: MainCancellationSource.Token));
TimedAnnotations.Clear();
_formState.AnnotationResults.Clear();
foreach (var ann in annotations)
{
TimedAnnotations.Add(ann.Time.Subtract(_thresholdBefore), ann.Time.Add(_thresholdAfter), ann);
_formState.AnnotationResults.Add(new AnnotationResult(_appConfig.AnnotationConfig.DetectionClassesDict, ann));
}
}
//Add manually
public void AddAnnotation(Annotation annotation)
{
var time = annotation.Time;
var previousAnnotations = TimedAnnotations.Query(time);
TimedAnnotations.Remove(previousAnnotations);
TimedAnnotations.Add(time.Subtract(_thresholdBefore), time.Add(_thresholdAfter), annotation);
var existingResult = _formState.AnnotationResults.FirstOrDefault(x => x.Annotation.Time == time);
if (existingResult != null)
{
try
{
_formState.AnnotationResults.Remove(existingResult);
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
throw;
}
}
var dict = _formState.AnnotationResults
.Select((x, i) => new { x.Annotation.Time, Index = i })
.ToDictionary(x => x.Time, x => x.Index);
var index = dict.Where(x => x.Key < time)
.OrderBy(x => time - x.Key)
.Select(x => x.Value + 1)
.FirstOrDefault();
var annRes = new AnnotationResult(_appConfig.AnnotationConfig.DetectionClassesDict, annotation);
_formState.AnnotationResults.Insert(index, annRes);
}
private async Task ReloadFiles()
{
var dir = new DirectoryInfo(_appConfig.DirectoriesConfig.VideosDirectory);
if (!dir.Exists)
return;
var videoFiles = dir.GetFiles(_appConfig.AnnotationConfig.VideoFormats.ToArray()).Select(x =>
{
using var media = new Media(_libVLC, x.FullName);
media.Parse();
var fInfo = new MediaFileInfo
{
Name = x.Name,
Path = x.FullName,
MediaType = MediaTypes.Video
};
media.ParsedChanged += (_, _) => fInfo.Duration = TimeSpan.FromMilliseconds(media.Duration);
return fInfo;
}).ToList();
var imageFiles = dir.GetFiles(_appConfig.AnnotationConfig.ImageFormats.ToArray())
.Select(x => new MediaFileInfo
{
Name = x.Name,
Path = x.FullName,
MediaType = MediaTypes.Image
});
var allFiles = videoFiles.Concat(imageFiles).ToList();
var allFileNames = allFiles.Select(x => x.FName).ToList();
var labelsDict = await _dbFactory.Run(async db => await db.Annotations
.GroupBy(x => x.Name.Substring(0, x.Name.Length - 7))
.Where(x => allFileNames.Contains(x.Key))
.ToDictionaryAsync(x => x.Key, x => x.Key));
foreach (var mediaFile in allFiles)
mediaFile.HasAnnotations = labelsDict.ContainsKey(mediaFile.FName);
AllMediaFiles = new ObservableCollection<MediaFileInfo>(allFiles);
LvFiles.ItemsSource = AllMediaFiles;
BlinkHelp(AllMediaFiles.Count == 0
? HelpTexts.HelpTextsDict[HelpTextEnum.Initial]
: HelpTexts.HelpTextsDict[HelpTextEnum.PlayVideo]);
DataContext = this;
}
private void OnFormClosed(object? sender, EventArgs e)
{
MainCancellationSource.Cancel();
_inferenceService.StopInference();
DetectionCancellationSource.Cancel();
_mediaPlayer.Stop();
_mediaPlayer.Dispose();
_libVLC.Dispose();
}
private void OpenContainingFolder(object sender, RoutedEventArgs e)
{
var mediaFileInfo = (sender as MenuItem)?.DataContext as MediaFileInfo;
if (mediaFileInfo == null)
return;
Process.Start("explorer.exe", "/select,\"" + mediaFileInfo.Path +"\"");
}
public void SeekTo(long timeMilliseconds, bool setPause = true)
{
_mediaPlayer.SetPause(setPause);
_mediaPlayer.Time = timeMilliseconds;
VideoSlider.Value = _mediaPlayer.Position * 100;
StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}";
}
private void SeekTo(TimeSpan time) =>
SeekTo((long)time.TotalMilliseconds);
private async void OpenFolderItemClick(object sender, RoutedEventArgs e) => await OpenFolder();
private async void OpenFolderButtonClick(object sender, RoutedEventArgs e) => await OpenFolder();
private async Task OpenFolder()
{
var dlg = new CommonOpenFileDialog
{
Title = "Open Video folder",
IsFolderPicker = true,
InitialDirectory = Path.GetDirectoryName(AppDomain.CurrentDomain.BaseDirectory)
};
var dialogResult = dlg.ShowDialog();
if (dialogResult != CommonFileDialogResult.Ok || string.IsNullOrEmpty(dlg.FileName))
return;
_appConfig.DirectoriesConfig.VideosDirectory = dlg.FileName;
TbFolder.Text = dlg.FileName;
await Task.CompletedTask;
}
private void TbFilter_OnTextChanged(object sender, TextChangedEventArgs e)
{
FilteredMediaFiles = new ObservableCollection<MediaFileInfo>(AllMediaFiles.Where(x => x.Name.ToLower().Contains(TbFilter.Text.ToLower())).ToList());
LvFiles.ItemsSource = FilteredMediaFiles;
LvFiles.ItemsSource = FilteredMediaFiles;
}
private void PlayClick(object sender, RoutedEventArgs e)
{
if (IsInferenceNow)
FollowAI = false;
_mediator.Publish(new AnnotatorControlEvent(_mediaPlayer.CanPause ? PlaybackControlEnum.Pause : PlaybackControlEnum.Play));
}
private void PauseClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.Pause));
private void StopClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.Stop));
private void PreviousFrameClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.PreviousFrame));
private void NextFrameClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.NextFrame));
private void SaveAnnotationsClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.SaveAnnotations));
private void RemoveSelectedClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.RemoveSelectedAnns));
private void RemoveAllClick(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.RemoveAllAnns));
private void TurnOffVolume(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.TurnOffVolume));
private void TurnOnVolume(object sender, RoutedEventArgs e) => _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.TurnOnVolume));
private void OpenHelpWindowClick(object sender, RoutedEventArgs e)
{
_helpWindow.Show();
_helpWindow.Activate();
}
private void Thumb_OnDragCompleted(object sender, DragCompletedEventArgs e) => _ = SaveUserSettings();
private void LvFilesContextOpening(object sender, ContextMenuEventArgs e)
{
var listItem = sender as ListViewItem;
LvFilesContextMenu.DataContext = listItem!.DataContext;
}
public void AutoDetect(object sender, RoutedEventArgs e)
{
if (IsInferenceNow)
{
FollowAI = true;
return;
}
if (LvFiles.Items.IsEmpty)
return;
if (LvFiles.SelectedIndex == -1)
LvFiles.SelectedIndex = 0;
Dispatcher.Invoke(() => Editor.ResetBackground());
IsInferenceNow = true;
FollowAI = true;
DetectionCancellationSource = new CancellationTokenSource();
var detectToken = DetectionCancellationSource.Token;
_ = Task.Run(async () =>
{
while (!detectToken.IsCancellationRequested)
{
var files = new List<string>();
await Dispatcher.Invoke(async () =>
{
//Take all medias
files = (LvFiles.ItemsSource as IEnumerable<MediaFileInfo>)?.Skip(LvFiles.SelectedIndex)
//.Where(x => !x.HasAnnotations)
.Take(Constants.DETECTION_BATCH_SIZE)
.Select(x => x.Path)
.ToList() ?? [];
if (files.Count != 0)
{
await _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.Play), detectToken);
await ReloadAnnotations();
}
});
if (files.Count == 0)
break;
await _inferenceService.RunInference(files, async annotationImage => await ProcessDetection(annotationImage, detectToken), detectToken);
Dispatcher.Invoke(() =>
{
if (LvFiles.SelectedIndex + files.Count >= LvFiles.Items.Count)
DetectionCancellationSource.Cancel();
LvFiles.SelectedIndex += files.Count;
});
}
Dispatcher.Invoke(() =>
{
LvFiles.Items.Refresh();
IsInferenceNow = false;
FollowAI = false;
});
});
}
private async Task ProcessDetection(AnnotationImage annotationImage, CancellationToken ct = default)
{
await Dispatcher.Invoke(async () =>
{
try
{
var annotation = await _annotationService.SaveAnnotation(annotationImage, ct);
if (annotation.OriginalMediaName != _formState.CurrentMedia?.FName)
{
var nextFile = (LvFiles.ItemsSource as IEnumerable<MediaFileInfo>)?
.Select((info, i) => new
{
MediaInfo = info,
Index = i
})
.FirstOrDefault(x => x.MediaInfo.FName == annotation.OriginalMediaName);
if (nextFile != null)
{
LvFiles.SelectedIndex = nextFile.Index;
await _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.Play), ct);
}
}
AddAnnotation(annotation);
if (FollowAI)
SeekTo(annotationImage.Milliseconds, false);
var log = string.Join(Environment.NewLine, annotation.Detections.Select(det =>
$"{_appConfig.AnnotationConfig.DetectionClassesDict[det.ClassNumber].Name}: " +
$"xy=({det.CenterX:F2},{det.CenterY:F2}), " +
$"size=({det.Width:F2}, {det.Height:F2}), " +
$"conf: {det.Confidence*100:F0}%"));
Dispatcher.Invoke(() =>
{
if (_formState.CurrentMedia != null)
_formState.CurrentMedia.HasAnnotations = true;
LvFiles.Items.Refresh();
StatusHelp.Text = log;
});
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
}
});
}
private void SwitchGpsPanel(object sender, RoutedEventArgs e)
{
_gpsPanelVisible = !_gpsPanelVisible;
if (_gpsPanelVisible)
{
GpsSplitterRow.Height = new GridLength(4);
GpsSplitter.Visibility = Visibility.Visible;
GpsSectionRow.Height = new GridLength(1, GridUnitType.Star);
MapMatcherComponent.Visibility = Visibility.Visible;
}
else
{
GpsSplitterRow.Height = new GridLength(0);
GpsSplitter.Visibility = Visibility.Collapsed;
GpsSectionRow.Height = new GridLength(0);
MapMatcherComponent.Visibility = Visibility.Collapsed;
}
}
private void SoundDetections(object sender, RoutedEventArgs e)
{
throw new NotImplementedException();
}
}
public class GradientStyleSelector : StyleSelector
{
public override Style? SelectStyle(object item, DependencyObject container)
{
if (container is not DataGridRow row || row.DataContext is not AnnotationResult result)
return null;
var style = new Style(typeof(DataGridRow));
var brush = new LinearGradientBrush
{
StartPoint = new Point(0, 0),
EndPoint = new Point(1, 0)
};
var gradients = new List<GradientStop>();
if (result.Colors.Count == 0)
{
var color = (Color)ColorConverter.ConvertFromString("#40DDDDDD");
gradients = [new GradientStop(color, 0.99)];
}
else
{
var increment = 1.0 / result.Colors.Count;
var currentStop = increment;
foreach (var c in result.Colors)
{
var resultColor = c.Color.ToConfidenceColor(c.Confidence);
brush.GradientStops.Add(new GradientStop(resultColor, currentStop));
currentStop += increment;
}
}
foreach (var gradientStop in gradients)
brush.GradientStops.Add(gradientStop);
style.Setters.Add(new Setter(DataGridRow.BackgroundProperty, brush));
return style;
}
}