diff --git a/Azaion.Annotator.Test/Azaion.Annotator.Test.csproj b/Azaion.Annotator.Test/Azaion.Annotator.Test.csproj index ac0cca5..239175f 100644 --- a/Azaion.Annotator.Test/Azaion.Annotator.Test.csproj +++ b/Azaion.Annotator.Test/Azaion.Annotator.Test.csproj @@ -11,6 +11,7 @@ + diff --git a/Azaion.Annotator.Test/IntervalTreeTest.cs b/Azaion.Annotator.Test/IntervalTreeTest.cs new file mode 100644 index 0000000..f0de9bd --- /dev/null +++ b/Azaion.Annotator.Test/IntervalTreeTest.cs @@ -0,0 +1,32 @@ +using FluentAssertions; +using Xunit; +using IntervalTree; + +namespace Azaion.Annotator.Test; + +public class IntervalTreeTest +{ + [Theory] + [MemberData(nameof(IntervalTreeTestQueryTestData))] + public void IntervalTreeTestQuery(int second, string[] expected) + { + var mainTree = new IntervalTree + { + { TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(3), "res01" }, + { TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(4), "res02" }, + { TimeSpan.FromSeconds(4), TimeSpan.FromSeconds(7), "res04" }, + { TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(8), "res05" }, + }; + + var result = mainTree.Query(TimeSpan.FromSeconds(second)).ToArray(); + result.Should().Equal(expected); + } + + public static IEnumerable IntervalTreeTestQueryTestData() + { + yield return [1, new[] {"res01"}]; + yield return [5, new[] {"res04", "res05"}]; + yield return [9, new string[] {}]; + + } +} \ No newline at end of file diff --git a/Azaion.Annotator/Azaion.Annotator.csproj b/Azaion.Annotator/Azaion.Annotator.csproj index f0f557c..92c88fc 100644 --- a/Azaion.Annotator/Azaion.Annotator.csproj +++ b/Azaion.Annotator/Azaion.Annotator.csproj @@ -17,6 +17,7 @@ + diff --git a/Azaion.Annotator/Controls/CanvasEditor.cs b/Azaion.Annotator/Controls/CanvasEditor.cs index ffd5f21..c8e1ddc 100644 --- a/Azaion.Annotator/Controls/CanvasEditor.cs +++ b/Azaion.Annotator/Controls/CanvasEditor.cs @@ -313,7 +313,7 @@ public class CanvasEditor : Canvas #endregion - public void RemoveAnnotations(IEnumerable listToRemove) + private void RemoveAnnotations(IEnumerable listToRemove) { foreach (var ann in listToRemove) { @@ -321,8 +321,15 @@ public class CanvasEditor : Canvas CurrentAnns.Remove(ann); } } - public void RemoveAllAnns() => RemoveAnnotations(CurrentAnns); - public void RemoveSelectedAnns() => RemoveAnnotations(CurrentAnns.Where(x => x.IsSelected)); + + public void RemoveAllAnns() + { + foreach (var ann in CurrentAnns) + Children.Remove(ann); + CurrentAnns.Clear(); + } + + public void RemoveSelectedAnns() => RemoveAnnotations(CurrentAnns.Where(x => x.IsSelected).ToList()); private void ClearSelections() { @@ -331,7 +338,8 @@ public class CanvasEditor : Canvas } public void ClearExpiredAnnotations(TimeSpan time) - { - RemoveAnnotations(CurrentAnns.Where(x => time - x.Time > _viewThreshold)); + { + var expiredAnns = CurrentAnns.Where(x => Math.Abs((time - x.Time).TotalMilliseconds) > _viewThreshold.TotalMilliseconds).ToList(); + RemoveAnnotations(expiredAnns); } } \ No newline at end of file diff --git a/Azaion.Annotator/DTO/AnnotationResult.cs b/Azaion.Annotator/DTO/AnnotationResult.cs index 38c0e23..96dcf3c 100644 --- a/Azaion.Annotator/DTO/AnnotationResult.cs +++ b/Azaion.Annotator/DTO/AnnotationResult.cs @@ -1,29 +1,72 @@ -using Newtonsoft.Json; +using System.Windows.Media; +using Newtonsoft.Json; namespace Azaion.Annotator.DTO; public class AnnotationResult { + private readonly Config _config = null!; + [JsonProperty(PropertyName = "f")] public string Image { get; set; } = null!; [JsonProperty(PropertyName = "t")] public TimeSpan Time { get; set; } - [JsonIgnore] - public string TimeStr => $"{Time:h\\:mm\\:ss}"; - [JsonProperty(PropertyName = "p")] public double Percentage { get; set; } - + public double Lat { get; set; } public double Lon { get; set; } public List Labels { get; set; } = new(); - + + #region For Display in the grid + + [JsonIgnore] + //For XAML Form + public string TimeStr => $"{Time:h\\:mm\\:ss}"; + + [JsonIgnore] + //For XAML Form + public string ClassName + { + get + { + if (Labels.Count == 0) + return ""; + + var groups = Labels.Select(x => x.ClassNumber).GroupBy(x => x).ToList(); + return groups.Count > 1 + ? string.Join(",", groups.Select(x => x.Key + 1)) + : _config.AnnotationClassesDict[groups.FirstOrDefault().Key].Name; + } + } + + [JsonIgnore] + //For XAML Form + public Color ClassColor + { + get + { + var defaultColor = (Color)ColorConverter.ConvertFromString("#404040"); + if (Labels.Count == 0) + return defaultColor; + + var groups = Labels.Select(x => x.ClassNumber).GroupBy(x => x).ToList(); + + return groups.Count > 1 + ? defaultColor + : _config.AnnotationClassesDict[groups.FirstOrDefault().Key].Color; + } + } + + + #endregion public AnnotationResult() { } - public AnnotationResult(TimeSpan time, string timeName, List labels) + public AnnotationResult(TimeSpan time, string timeName, List labels, Config config) { + _config = config; Labels = labels; Time = time; Image = $"{timeName}.jpg"; diff --git a/Azaion.Annotator/DTO/Config.cs b/Azaion.Annotator/DTO/Config.cs index 9aa092d..c4a9980 100644 --- a/Azaion.Annotator/DTO/Config.cs +++ b/Azaion.Annotator/DTO/Config.cs @@ -27,6 +27,10 @@ public class Config public string ResultsDirectory { get; set; } = DEFAULT_RESULTS_DIR; public List AnnotationClasses { get; set; } = []; + + private Dictionary? _annotationClassesDict; + public Dictionary AnnotationClassesDict => _annotationClassesDict ??= AnnotationClasses.ToDictionary(x => x.Id); + public Size WindowSize { get; set; } public Point WindowLocation { get; set; } public bool ShowHelpOnStart { get; set; } diff --git a/Azaion.Annotator/DTO/FormState.cs b/Azaion.Annotator/DTO/FormState.cs index b5dc947..327b15f 100644 --- a/Azaion.Annotator/DTO/FormState.cs +++ b/Azaion.Annotator/DTO/FormState.cs @@ -1,4 +1,5 @@ -using System.Globalization; +using System.Collections.ObjectModel; +using System.Globalization; using System.IO; using System.Windows; @@ -13,7 +14,7 @@ public class FormState public string VideoName => Path.GetFileNameWithoutExtension(CurrentFile).Replace(" ", ""); public TimeSpan CurrentVideoLength { get; set; } public int CurrentVolume { get; set; } = 100; - public List AnnotationResults { get; set; } = []; + public ObservableCollection AnnotationResults { get; set; } = []; public string GetTimeName(TimeSpan ts) => $"{VideoName}_{ts:hmmssf}"; diff --git a/Azaion.Annotator/DTO/Label.cs b/Azaion.Annotator/DTO/Label.cs index fde951c..4cb019c 100644 --- a/Azaion.Annotator/DTO/Label.cs +++ b/Azaion.Annotator/DTO/Label.cs @@ -38,8 +38,8 @@ public class CanvasLabel : Label ClassNumber = label.ClassNumber; - var left = X - Width / 2; - var top = Y - Height / 2; + var left = label.CenterX - label.Width / 2; + var top = label.CenterY - label.Height / 2; if (videoAr > canvasAr) //100% width { @@ -89,7 +89,6 @@ public class YoloLabel : Label public YoloLabel(CanvasLabel canvasLabel, Size canvasSize, Size videoSize) { - var cw = canvasSize.Width; var ch = canvasSize.Height; var canvasAr = cw / ch; @@ -117,8 +116,8 @@ public class YoloLabel : Label Width = canvasLabel.Width / realWidth; } - CenterX = left + canvasLabel.Width / 2.0; - CenterY = top + canvasLabel.Height / 2.0; + CenterX = left + Width / 2.0; + CenterY = top + Height / 2.0; } public static YoloLabel? Parse(string s) diff --git a/Azaion.Annotator/MainWindow.xaml b/Azaion.Annotator/MainWindow.xaml index 6496e2b..7ebfc8b 100644 --- a/Azaion.Annotator/MainWindow.xaml +++ b/Azaion.Annotator/MainWindow.xaml @@ -207,7 +207,7 @@ CanUserResizeColumns="False"> @@ -220,15 +220,29 @@ + + + + + + AnnotationClasses { get; set; } = new(); private bool _suspendLayout; - - public Dictionary> Annotations { get; set; } = new(); - public Dictionary> AnnotationResults { get; set; } = new(); - + + private readonly TimeSpan _thresholdBefore = TimeSpan.FromMilliseconds(100); + private readonly TimeSpan _thresholdAfter = TimeSpan.FromMilliseconds(300); + + public Dictionary> AnnotationsDict { get; set; } = new(); + private IntervalTree> Annotations { get; set; } = new(); + public MainWindow(LibVLC libVLC, MediaPlayer mediaPlayer, IMediator mediator, FormState formState, @@ -111,20 +117,7 @@ public partial class MainWindow _mediaPlayer.PositionChanged += (o, args) => { - Dispatcher.Invoke(() => VideoSlider.Value = _mediaPlayer.Position * VideoSlider.Maximum); - Dispatcher.Invoke(() => StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}"); - - var time = TimeSpan.FromMilliseconds(_mediaPlayer.Time); - if (!Annotations.TryGetValue(time, out var annotations)) - return; - - foreach (var ann in annotations) - { - var annClass = _config.AnnotationClasses[ann.ClassNumber]; - var annInfo = new CanvasLabel(ann, Editor.RenderSize, _formState.CurrentVideoSize); - Dispatcher.Invoke(() => Editor.CreateAnnotation(annClass, time, annInfo)); - } - Dispatcher.Invoke(() => Editor.ClearExpiredAnnotations(time)); + ShowTimeAnnotations(TimeSpan.FromMilliseconds(_mediaPlayer.Time)); }; VideoSlider.ValueChanged += (value, newValue) => @@ -147,59 +140,58 @@ public partial class MainWindow Editor.FormState = _formState; Editor.Mediator = _mediator; - } - - public void LoadExistingAnnotations() - { - Annotations = LoadAnnotations(); - _formState.AnnotationResults = LoadAnnotationResults(); DgAnnotations.ItemsSource = _formState.AnnotationResults; } - private Dictionary> LoadAnnotations() + private void ShowTimeAnnotations(TimeSpan time) { + Dispatcher.Invoke(() => VideoSlider.Value = _mediaPlayer.Position * VideoSlider.Maximum); + Dispatcher.Invoke(() => StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}"); + + Dispatcher.Invoke(() => Editor.ClearExpiredAnnotations(time)); + + var annotations = Annotations.Query(time).SelectMany(x => x).ToList(); + foreach (var ann in annotations) + { + var annClass = _config.AnnotationClasses[ann.ClassNumber]; + var annInfo = new CanvasLabel(ann, Editor.RenderSize, _formState.CurrentVideoSize); + Dispatcher.Invoke(() => Editor.CreateAnnotation(annClass, time, annInfo)); + } + } + + public void ReloadAnnotations() + { + _formState.AnnotationResults.Clear(); + AnnotationsDict.Clear(); + Annotations.Clear(); + var labelDir = new DirectoryInfo(_config.LabelsDirectory); if (!labelDir.Exists) - return new Dictionary>(); + return; var labelFiles = labelDir.GetFiles($"{_formState.VideoName}_*"); - return labelFiles.Select(x => + foreach (var file in labelFiles) { - var name = Path.GetFileNameWithoutExtension(x.Name); - return new - { - Name = x.FullName, - Time = _formState.GetTime(name) - }; - }).ToDictionary(f => f.Time!.Value, f => - { - var str = File.ReadAllText(f.Name); - return str.Split(Environment.NewLine).Select(YoloLabel.Parse) - .Where(x => x != null) - .ToList(); - })!; - } + var name = Path.GetFileNameWithoutExtension(file.Name); + var time = _formState.GetTime(name)!.Value; - private List LoadAnnotationResults() - { - var resultDir = new DirectoryInfo(_config!.ResultsDirectory); - if (!resultDir.Exists) - Directory.CreateDirectory(_config!.ResultsDirectory); - - var resultsFile = resultDir.GetFiles($"{_formState!.CurrentFile}*").FirstOrDefault(); - - List results; - if (resultsFile == null) - { - results = Annotations.Select(anns => new AnnotationResult(anns.Key, _formState.GetTimeName(anns.Key), anns.Value)) + var str = File.ReadAllText(file.FullName); + var annotations = str.Split(Environment.NewLine).Select(YoloLabel.Parse) + .Where(ann => ann != null) .ToList(); - File.WriteAllText($"{_config.ResultsDirectory}/{_formState.VideoName}.json", JsonConvert.SerializeObject(results, Formatting.Indented)); + + AddAnnotation(time, annotations!); } - else - results = JsonConvert.DeserializeObject>(File.ReadAllText(File.ReadAllText(resultsFile.FullName)))!; - return results; } - + + public async Task AddAnnotation(TimeSpan time, List annotations) + { + var fName = _formState.GetTimeName(time); + AnnotationsDict.Add(time, annotations!); + Annotations.Add(time.Subtract(_thresholdBefore), time.Add(_thresholdAfter), annotations); + _formState.AnnotationResults.Add(new AnnotationResult(time, fName, annotations, _config)); + await File.WriteAllTextAsync($"{_config.ResultsDirectory}/{fName}.json", JsonConvert.SerializeObject(_formState.AnnotationResults)); + } private void ReloadFiles() { @@ -288,4 +280,17 @@ public partial class MainWindow _helpWindow.Show(); _helpWindow.Activate(); } + + private void DgAnnotationsRowClick(object sender, MouseButtonEventArgs e) + { + DgAnnotations.MouseDoubleClick += (sender, args) => + { + Editor.RemoveAllAnns(); + var dgRow = ItemsControl.ContainerFromElement((DataGrid)sender, (args.OriginalSource as DependencyObject)!) as DataGridRow; + var res = (AnnotationResult)dgRow!.Item; + _mediaPlayer.SetPause(true); + _mediaPlayer.Time = (long)res.Time.TotalMilliseconds;// + 250; + ShowTimeAnnotations(res.Time); + }; + } } diff --git a/Azaion.Annotator/PlayerControlHandler.cs b/Azaion.Annotator/PlayerControlHandler.cs index a5e8a54..8cdbcc1 100644 --- a/Azaion.Annotator/PlayerControlHandler.cs +++ b/Azaion.Annotator/PlayerControlHandler.cs @@ -4,7 +4,6 @@ using System.Windows.Input; using Azaion.Annotator.DTO; using LibVLCSharp.Shared; using MediatR; -using Newtonsoft.Json; namespace Azaion.Annotator; @@ -179,7 +178,7 @@ public class PlayerControlHandler(LibVLC libVLC, MediaPlayer mediaPlayer, MainWi var fileInfo = (VideoFileInfo)mainWindow.LvFiles.SelectedItem; formState.CurrentFile = fileInfo.Name; - mainWindow.LoadExistingAnnotations(); + mainWindow.ReloadAnnotations(); mediaPlayer.Stop(); mediaPlayer.Play(new Media(libVLC, fileInfo.Path)); @@ -206,15 +205,10 @@ public class PlayerControlHandler(LibVLC libVLC, MediaPlayer mediaPlayer, MainWi Directory.CreateDirectory(config.ResultsDirectory); await File.WriteAllTextAsync($"{config.LabelsDirectory}/{fName}.txt", labels); - - formState.AnnotationResults.Add(new AnnotationResult(time, fName, currentAnns)); - await File.WriteAllTextAsync($"{config.ResultsDirectory}/{fName}.json", JsonConvert.SerializeObject(formState.AnnotationResults)); - var resultHeight = (uint)Math.Round(RESULT_WIDTH / formState.CurrentVideoSize.Width * formState.CurrentVideoSize.Height); - mediaPlayer.TakeSnapshot(0, $"{config.ImagesDirectory}/{fName}.jpg", RESULT_WIDTH, resultHeight); - mainWindow.Annotations[time] = currentAnns; + await mainWindow.AddAnnotation(time, currentAnns); mainWindow.Editor.RemoveAllAnns(); mediaPlayer.Play(); }