From a81a6f881dc0954fa04297c2b6a6f4296b9949cc Mon Sep 17 00:00:00 2001 From: Oleksandr Bezdieniezhnykh Date: Wed, 7 Aug 2024 13:22:17 +0300 Subject: [PATCH] add image editing --- Azaion.Annotator/App.xaml.cs | 8 +-- Azaion.Annotator/DTO/Config.cs | 12 +++- Azaion.Annotator/DTO/FormState.cs | 11 ++-- Azaion.Annotator/DTO/MediaFileInfo.cs | 11 ++++ Azaion.Annotator/DTO/MediaTypes.cs | 8 +++ Azaion.Annotator/DTO/PlaybackControlEnum.cs | 4 +- Azaion.Annotator/DTO/VideoFileInfo.cs | 10 --- Azaion.Annotator/MainWindow.xaml | 7 +- Azaion.Annotator/MainWindow.xaml.cs | 56 ++++++++++++---- Azaion.Annotator/PlayerControlHandler.cs | 72 ++++++++++++++++----- Azaion.Annotator/config.json | 4 +- 11 files changed, 146 insertions(+), 57 deletions(-) create mode 100644 Azaion.Annotator/DTO/MediaFileInfo.cs create mode 100644 Azaion.Annotator/DTO/MediaTypes.cs delete mode 100644 Azaion.Annotator/DTO/VideoFileInfo.cs diff --git a/Azaion.Annotator/App.xaml.cs b/Azaion.Annotator/App.xaml.cs index d59187a..f293936 100644 --- a/Azaion.Annotator/App.xaml.cs +++ b/Azaion.Annotator/App.xaml.cs @@ -1,18 +1,14 @@ -using System.IO; -using System.Reflection; +using System.Reflection; using System.Windows; using System.Windows.Input; using System.Windows.Threading; using Azaion.Annotator.DTO; -using Azaion.Annotator.Extensions; using LibVLCSharp.Shared; using MediatR; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Serilog; -using ILogger = Microsoft.Extensions.Logging.ILogger; -using LogLevel = Microsoft.Extensions.Logging.LogLevel; namespace Azaion.Annotator; @@ -26,7 +22,7 @@ public partial class App : Application { Log.Logger = new LoggerConfiguration() .Enrich.FromLogContext() - .MinimumLevel.Warning() + .MinimumLevel.Information() .WriteTo.Console() .WriteTo.File( path: "Logs/log.txt", diff --git a/Azaion.Annotator/DTO/Config.cs b/Azaion.Annotator/DTO/Config.cs index e39a794..81e30ba 100644 --- a/Azaion.Annotator/DTO/Config.cs +++ b/Azaion.Annotator/DTO/Config.cs @@ -1,5 +1,4 @@ using System.IO; -using System.Reflection; using System.Text; using Microsoft.Extensions.Logging; using Newtonsoft.Json; @@ -28,6 +27,9 @@ public class Config public double RightPanelWidth { get; set; } public bool ShowHelpOnStart { get; set; } + + public List VideoFormats { get; set; } + public List ImageFormats { get; set; } } public interface IConfigRepository @@ -49,6 +51,9 @@ public class FileConfigRepository(ILogger logger) : IConfi private static readonly Size DefaultWindowSize = new(1280, 720); private static readonly Point DefaultWindowLocation = new(100, 100); + private static readonly List DefaultVideoFormats = ["mp4", "mov", "avi"]; + private static readonly List DefaultImageFormats = ["jpg", "jpeg", "png", "bmp"]; + public Config Get() { var exePath = Path.GetDirectoryName(AppDomain.CurrentDomain.BaseDirectory)!; @@ -65,7 +70,10 @@ public class FileConfigRepository(ILogger logger) : IConfi WindowLocation = DefaultWindowLocation, WindowSize = DefaultWindowSize, - ShowHelpOnStart = true + ShowHelpOnStart = true, + + VideoFormats = DefaultVideoFormats, + ImageFormats = DefaultImageFormats }; } var str = File.ReadAllText(CONFIG_PATH); diff --git a/Azaion.Annotator/DTO/FormState.cs b/Azaion.Annotator/DTO/FormState.cs index 327b15f..8374f2c 100644 --- a/Azaion.Annotator/DTO/FormState.cs +++ b/Azaion.Annotator/DTO/FormState.cs @@ -1,5 +1,4 @@ using System.Collections.ObjectModel; -using System.Globalization; using System.IO; using System.Windows; @@ -8,14 +7,16 @@ namespace Azaion.Annotator.DTO; public class FormState { public SelectionState SelectionState { get; set; } = SelectionState.None; - - public string CurrentFile { get; set; } = null!; + + public MediaFileInfo? CurrentMedia { get; set; } public Size CurrentVideoSize { get; set; } - public string VideoName => Path.GetFileNameWithoutExtension(CurrentFile).Replace(" ", ""); + public string VideoName => string.IsNullOrEmpty(CurrentMedia?.Name) + ? "" + : Path.GetFileNameWithoutExtension(CurrentMedia.Name).Replace(" ", ""); public TimeSpan CurrentVideoLength { get; set; } public int CurrentVolume { get; set; } = 100; public ObservableCollection AnnotationResults { get; set; } = []; - + public string GetTimeName(TimeSpan ts) => $"{VideoName}_{ts:hmmssf}"; public TimeSpan? GetTime(string name) diff --git a/Azaion.Annotator/DTO/MediaFileInfo.cs b/Azaion.Annotator/DTO/MediaFileInfo.cs new file mode 100644 index 0000000..3ba625d --- /dev/null +++ b/Azaion.Annotator/DTO/MediaFileInfo.cs @@ -0,0 +1,11 @@ +namespace Azaion.Annotator.DTO; + +public class MediaFileInfo +{ + public string Name { get; set; } = null!; + public string Path { get; set; } = null!; + public TimeSpan Duration { get; set; } + public string DurationStr => $"{Duration:h\\:mm\\:ss}"; + public bool HasAnnotations { get; set; } + public MediaTypes MediaType { get; set; } +} \ No newline at end of file diff --git a/Azaion.Annotator/DTO/MediaTypes.cs b/Azaion.Annotator/DTO/MediaTypes.cs new file mode 100644 index 0000000..d5c5ede --- /dev/null +++ b/Azaion.Annotator/DTO/MediaTypes.cs @@ -0,0 +1,8 @@ +namespace Azaion.Annotator.DTO; + +public enum MediaTypes +{ + None = 0, + Video = 1, + Image = 2 +} \ No newline at end of file diff --git a/Azaion.Annotator/DTO/PlaybackControlEnum.cs b/Azaion.Annotator/DTO/PlaybackControlEnum.cs index d0e76d7..21a2c99 100644 --- a/Azaion.Annotator/DTO/PlaybackControlEnum.cs +++ b/Azaion.Annotator/DTO/PlaybackControlEnum.cs @@ -12,5 +12,7 @@ public enum PlaybackControlEnum RemoveSelectedAnns = 7, RemoveAllAnns = 8, TurnOffVolume = 9, - TurnOnVolume = 10 + TurnOnVolume = 10, + Previous = 11, + Next = 12 } \ No newline at end of file diff --git a/Azaion.Annotator/DTO/VideoFileInfo.cs b/Azaion.Annotator/DTO/VideoFileInfo.cs deleted file mode 100644 index b4b1ba6..0000000 --- a/Azaion.Annotator/DTO/VideoFileInfo.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace Azaion.Annotator.DTO; - -public class VideoFileInfo -{ - public string Name { get; set; } = null!; - public string Path { get; set; } = null!; - public TimeSpan Duration { get; set; } - public string DurationStr => $"{Duration:h\\:mm\\:ss}"; - public bool HasAnnotations { get; set; } -} \ No newline at end of file diff --git a/Azaion.Annotator/MainWindow.xaml b/Azaion.Annotator/MainWindow.xaml index 95b00b7..5abc0d8 100644 --- a/Azaion.Annotator/MainWindow.xaml +++ b/Azaion.Annotator/MainWindow.xaml @@ -187,12 +187,15 @@ HorizontalAlignment="Stretch" VerticalAlignment="Stretch" /> - - + > AnnotationsDict { get; set; } = new(); - private IntervalTree> Annotations { get; set; } = new(); + public IntervalTree> Annotations { get; set; } = new(); public MainWindow(LibVLC libVLC, MediaPlayer mediaPlayer, IMediator mediator, @@ -113,12 +112,21 @@ public partial class MainWindow { VideoView.MediaPlayer = _mediaPlayer; - _mediaPlayer.Playing += (sender, args) => + _mediaPlayer.Playing += async (sender, args) => { uint vw = 0, vh = 0; _mediaPlayer.Size(0, ref vw, ref vh); _formState.CurrentVideoSize = new Size(vw, vh); _formState.CurrentVideoLength = TimeSpan.FromMilliseconds(_mediaPlayer.Length); + + await Dispatcher.Invoke(async () => await ReloadAnnotations()); + if (_formState.CurrentMedia?.MediaType != MediaTypes.Image) + return; + + //if image show annotations, give 100ms to load the frame and set on pause + await Task.Delay(100); + ShowCurrentAnnotations(); + _mediaPlayer.SetPause(true); }; LvFiles.MouseDoubleClick += async (_, _) => await _mediator.Publish(new PlaybackControlEvent(PlaybackControlEnum.Play)); @@ -165,6 +173,8 @@ public partial class MainWindow saveConfigFn.Debounce(TimeSpan.FromSeconds(5)).Invoke(); } + public void ShowCurrentAnnotations() => ShowTimeAnnotations(TimeSpan.FromMilliseconds(_mediaPlayer.Time)); + private void ShowTimeAnnotations(TimeSpan time) { Dispatcher.Invoke(() => VideoSlider.Value = _mediaPlayer.Position * VideoSlider.Maximum); @@ -181,11 +191,11 @@ public partial class MainWindow } } - public void ReloadAnnotations() + public async Task ReloadAnnotations() { _formState.AnnotationResults.Clear(); - AnnotationsDict.Clear(); Annotations.Clear(); + Editor.RemoveAllAnns(); var labelDir = new DirectoryInfo(_config.LabelsDirectory); if (!labelDir.Exists) @@ -197,20 +207,27 @@ public partial class MainWindow var name = Path.GetFileNameWithoutExtension(file.Name); var time = _formState.GetTime(name)!.Value; - var str = File.ReadAllText(file.FullName); + var str = await File.ReadAllTextAsync(file.FullName); var annotations = str.Split(Environment.NewLine).Select(YoloLabel.Parse) .Where(ann => ann != null) .ToList(); - AddAnnotation(time, annotations!); + await AddAnnotation(time, annotations!); } } public async Task AddAnnotation(TimeSpan time, List annotations) { var fName = _formState.GetTimeName(time); - AnnotationsDict.Add(time, annotations!); + + var previousAnnotations = Annotations.Query(time); + Annotations.Remove(previousAnnotations); Annotations.Add(time.Subtract(_thresholdBefore), time.Add(_thresholdAfter), annotations); + + var existingResult = _formState.AnnotationResults.FirstOrDefault(x => x.Time == time); + if (existingResult != null) + _formState.AnnotationResults.Remove(existingResult); + _formState.AnnotationResults.Add(new AnnotationResult(time, fName, annotations, _config)); await File.WriteAllTextAsync($"{_config.ResultsDirectory}/{fName}.json", JsonConvert.SerializeObject(_formState.AnnotationResults)); } @@ -226,25 +243,35 @@ public partial class MainWindow .GroupBy(x => x) .Select(gr => gr.Key) .ToDictionary(x => x); - - var files = dir.GetFiles("mp4", "mov").Select(x => + + var videoFiles = dir.GetFiles(_config.VideoFormats.ToArray()).Select(x => { var media = new Media(_libVLC, x.FullName); media.Parse(); - var fInfo = new VideoFileInfo + var fInfo = new MediaFileInfo { Name = x.Name, Path = x.FullName, + MediaType = MediaTypes.Video, HasAnnotations = labelNames.ContainsKey(Path.GetFileNameWithoutExtension(x.Name).Replace(" ", "")) }; media.ParsedChanged += (_, _) => fInfo.Duration = TimeSpan.FromMilliseconds(media.Duration); return fInfo; }).ToList(); - LvFiles.ItemsSource = new ObservableCollection(files); + var imageFiles = dir.GetFiles(_config.ImageFormats.ToArray()).Select(x => new MediaFileInfo + { + Name = x.Name, + Path = x.FullName, + MediaType = MediaTypes.Image, + HasAnnotations = labelNames.ContainsKey(Path.GetFileNameWithoutExtension(x.Name).Replace(" ", "")) + }); + + var mediaFiles = videoFiles.Concat(imageFiles).ToList(); + LvFiles.ItemsSource = new ObservableCollection(mediaFiles); TbFolder.Text = _config.VideosDirectory; - BlinkHelp(files.Count == 0 + BlinkHelp(mediaFiles.Count == 0 ? HelpTexts.HelpTextsDict[HelpTextEnum.Initial] : HelpTexts.HelpTextsDict[HelpTextEnum.PlayVideo]); } @@ -289,7 +316,7 @@ public partial class MainWindow _mediator.Publish(new PlaybackControlEvent(_mediaPlayer.CanPause ? PlaybackControlEnum.Pause : PlaybackControlEnum.Play)); } - private void PauseClick(object sender, RoutedEventArgs e)=> _mediator.Publish(new PlaybackControlEvent(PlaybackControlEnum.Pause)); + private void PauseClick(object sender, RoutedEventArgs e) => _mediator.Publish(new PlaybackControlEvent(PlaybackControlEnum.Pause)); private void StopClick(object sender, RoutedEventArgs e) => _mediator.Publish(new PlaybackControlEvent(PlaybackControlEnum.Stop)); private void PreviousFrameClick(object sender, RoutedEventArgs e) => _mediator.Publish(new PlaybackControlEvent(PlaybackControlEnum.PreviousFrame)); @@ -321,3 +348,4 @@ public partial class MainWindow }; } } + diff --git a/Azaion.Annotator/PlayerControlHandler.cs b/Azaion.Annotator/PlayerControlHandler.cs index c09ab33..3c306cf 100644 --- a/Azaion.Annotator/PlayerControlHandler.cs +++ b/Azaion.Annotator/PlayerControlHandler.cs @@ -1,13 +1,20 @@ using System.IO; using System.Windows; using System.Windows.Input; +using System.Windows.Threading; using Azaion.Annotator.DTO; using LibVLCSharp.Shared; using MediatR; +using Microsoft.Extensions.Logging; namespace Azaion.Annotator; -public class PlayerControlHandler(LibVLC libVLC, MediaPlayer mediaPlayer, MainWindow mainWindow, FormState formState, Config config) : +public class PlayerControlHandler(LibVLC libVLC, + MediaPlayer mediaPlayer, + MainWindow mainWindow, + FormState formState, + Config config, + ILogger logger) : INotificationHandler, INotificationHandler, INotificationHandler, @@ -19,14 +26,16 @@ public class PlayerControlHandler(LibVLC libVLC, MediaPlayer mediaPlayer, MainWi private static readonly string[] CatchSenders = ["ForegroundWindow", "ScrollViewer", "VideoView", "GridSplitter"]; - private readonly Dictionary KeysControlEnumDict = new() + private readonly Dictionary _keysControlEnumDict = new() { { Key.Space, PlaybackControlEnum.Pause }, { Key.Left, PlaybackControlEnum.PreviousFrame }, { Key.Right, PlaybackControlEnum.NextFrame }, { Key.Enter, PlaybackControlEnum.SaveAnnotations }, { Key.Delete, PlaybackControlEnum.RemoveSelectedAnns }, - { Key.X, PlaybackControlEnum.RemoveAllAnns } + { Key.X, PlaybackControlEnum.RemoveAllAnns }, + { Key.PageUp, PlaybackControlEnum.Previous }, + { Key.PageDown, PlaybackControlEnum.Next }, }; public async Task Handle(AnnClassSelectedEvent notification, CancellationToken cancellationToken) @@ -59,7 +68,7 @@ public class PlayerControlHandler(LibVLC libVLC, MediaPlayer mediaPlayer, MainWi if (keyNumber.HasValue) SelectClass(mainWindow.AnnotationClasses[keyNumber.Value]); - if (KeysControlEnumDict.TryGetValue(key, out var value)) + if (_keysControlEnumDict.TryGetValue(key, out var value)) await ControlPlayback(value); await VolumeControl(key); @@ -106,7 +115,7 @@ public class PlayerControlHandler(LibVLC libVLC, MediaPlayer mediaPlayer, MainWi switch (controlEnum) { case PlaybackControlEnum.Play: - Play(); + await Play(); break; case PlaybackControlEnum.Pause: mediaPlayer.Pause(); @@ -146,6 +155,12 @@ public class PlayerControlHandler(LibVLC libVLC, MediaPlayer mediaPlayer, MainWi formState.CurrentVolume = mediaPlayer.Volume; mediaPlayer.Volume = 0; break; + case PlaybackControlEnum.Previous: + await NextMedia(isPrevious: true); + break; + case PlaybackControlEnum.Next: + await NextMedia(); + break; case PlaybackControlEnum.None: break; default: @@ -159,6 +174,17 @@ public class PlayerControlHandler(LibVLC libVLC, MediaPlayer mediaPlayer, MainWi } } + private async Task NextMedia(bool isPrevious = false) + { + var increment = isPrevious ? -1 : 1; + var check = isPrevious ? -1 : mainWindow.LvFiles.Items.Count; + if (mainWindow.LvFiles.SelectedIndex + increment == check) + return; + + mainWindow.LvFiles.SelectedIndex += increment; + await Play(); + } + public async Task Handle(VolumeChangedEvent notification, CancellationToken cancellationToken) { ChangeVolume(notification.Volume); @@ -171,28 +197,27 @@ public class PlayerControlHandler(LibVLC libVLC, MediaPlayer mediaPlayer, MainWi mediaPlayer.Volume = volume; } - private void Play() + private async Task Play() { if (mainWindow.LvFiles.SelectedItem == null) return; - var fileInfo = (VideoFileInfo)mainWindow.LvFiles.SelectedItem; - - formState.CurrentFile = fileInfo.Name; - mainWindow.ReloadAnnotations(); + var mediaInfo = (MediaFileInfo)mainWindow.LvFiles.SelectedItem; + formState.CurrentMedia = mediaInfo; mediaPlayer.Stop(); - mediaPlayer.Play(new Media(libVLC, fileInfo.Path)); - mainWindow.Title = $"Azaion Annotator - {fileInfo.Name}"; + mainWindow.Title = $"Azaion Annotator - {mediaInfo.Name}"; mainWindow.BlinkHelp(HelpTexts.HelpTextsDict[HelpTextEnum.PauseForAnnotations]); + mediaPlayer.Play(new Media(libVLC, mediaInfo.Path)); } private async Task SaveAnnotations() { - if (string.IsNullOrEmpty(formState.CurrentFile)) + if (formState.CurrentMedia == null) return; var time = TimeSpan.FromMilliseconds(mediaPlayer.Time); var fName = formState.GetTimeName(time); + var currentAnns = mainWindow.Editor.CurrentAnns .Select(x => new YoloLabel(x.Info, mainWindow.Editor.RenderSize, formState.CurrentVideoSize)) .ToList(); @@ -205,12 +230,27 @@ public class PlayerControlHandler(LibVLC libVLC, MediaPlayer mediaPlayer, MainWi if (!Directory.Exists(config.ResultsDirectory)) Directory.CreateDirectory(config.ResultsDirectory); - await File.WriteAllTextAsync($"{config.LabelsDirectory}/{fName}.txt", labels); + await File.WriteAllTextAsync(Path.Combine(config.LabelsDirectory, $"{fName}.txt"), labels); var resultHeight = (uint)Math.Round(RESULT_WIDTH / formState.CurrentVideoSize.Width * formState.CurrentVideoSize.Height); - mediaPlayer.TakeSnapshot(0, $"{config.ImagesDirectory}/{fName}.jpg", RESULT_WIDTH, resultHeight); await mainWindow.AddAnnotation(time, currentAnns); + + formState.CurrentMedia.HasAnnotations = mainWindow.Annotations.Count != 0; + mainWindow.LvFiles.Items.Refresh(); + + var isVideo = formState.CurrentMedia.MediaType == MediaTypes.Video; + var destinationPath = Path.Combine(config.ImagesDirectory, $"{fName}{(isVideo ? ".jpg" : Path.GetExtension(formState.CurrentMedia.Path))}"); + mainWindow.Editor.RemoveAllAnns(); - mediaPlayer.Play(); + if (isVideo) + { + mediaPlayer.TakeSnapshot(0, destinationPath, RESULT_WIDTH, resultHeight); + mediaPlayer.Play(); + } + else + { + File.Copy(formState.CurrentMedia.Path, destinationPath, overwrite: true); + await NextMedia(); + } } } \ No newline at end of file diff --git a/Azaion.Annotator/config.json b/Azaion.Annotator/config.json index 186ed04..81bf945 100644 --- a/Azaion.Annotator/config.json +++ b/Azaion.Annotator/config.json @@ -70,5 +70,7 @@ "FullScreen": true, "LeftPanelWidth": 30, "RightPanelWidth": 30, - "ShowHelpOnStart": false + "ShowHelpOnStart": false, + "VideoFormats": ["mov", "mp4"], + "ImageFormats": ["jpg", "jpeg", "png", "bmp", "gif"] } \ No newline at end of file