azaion/annotations
1
0
Fork 0
mirror of https://github.com/azaion/annotations.git synced 2026-04-22 22:46:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

remove fix, todo: test

Browse Source
This commit is contained in:
Alex Bezdieniezhnykh
2025-01-03 18:32:56 +02:00
parent 9aebfd787b
commit ae2c62350a
19 changed files with 353 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Azaion.Annotator/Annotator.xaml.cs
+88 -111
View File
@@ -9,25 +9,26 @@ using System.Windows.Media;
using System.Windows.Media.Imaging;
using Azaion.Annotator.DTO;
using Azaion.Annotator.Extensions;
using Azaion.Common;
using Azaion.Common.Database;
using Azaion.Common.DTO;
using Azaion.Common.DTO.Config;
using Azaion.Common.DTO.Queue;
using Azaion.Common.Events;
using Azaion.Common.Extensions;
using Azaion.Common.Services;
using LibVLCSharp.Shared;
using MediatR;
using Microsoft.WindowsAPICodePack.Dialogs;
using Newtonsoft.Json;
using Size = System.Windows.Size;
using IntervalTree;
using LinqToDB;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MediaPlayer = LibVLCSharp.Shared.MediaPlayer;
namespace Azaion.Annotator;
public partial class Annotator
public partial class Annotator : INotificationHandler<AnnotationsDeletedEvent>
{
private readonly AppConfig _appConfig;
private readonly LibVLC _libVLC;
@@ -41,7 +42,8 @@ public partial class Annotator
private readonly VLCFrameExtractor _vlcFrameExtractor;
private readonly IAIDetector _aiDetector;
private readonly AnnotationService _annotationService;
private readonly CancellationTokenSource _cancellationTokenSource = new();
private readonly IDbFactory _dbFactory;
private readonly CancellationTokenSource _ctSource = new();
private ObservableCollection<DetectionClass> AnnotationClasses { get; set; } = new();
private bool _suspendLayout;
@@ -53,20 +55,22 @@ public partial class Annotator
private ObservableCollection<MediaFileInfo> AllMediaFiles { get; set; } = new();
private ObservableCollection<MediaFileInfo> FilteredMediaFiles { get; set; } = new();
public IntervalTree<TimeSpan, List<Detection>> Detections { get; set; } = new();
public IntervalTree<TimeSpan, Annotation> TimedAnnotations { get; set; } = new();
private AutodetectDialog _autoDetectDialog = new() { Topmost = true };
public Annotator(
IConfigUpdater configUpdater,
IOptions<AppConfig> appConfig,
LibVLC libVLC, MediaPlayer mediaPlayer,
LibVLC libVLC,
MediaPlayer mediaPlayer,
IMediator mediator,
FormState formState,
HelpWindow helpWindow,
ILogger<Annotator> logger,
VLCFrameExtractor vlcFrameExtractor,
IAIDetector aiDetector,
AnnotationService annotationService)
AnnotationService annotationService,
IDbFactory dbFactory)
{
InitializeComponent();
_appConfig = appConfig.Value;
@@ -80,10 +84,27 @@ public partial class Annotator
_vlcFrameExtractor = vlcFrameExtractor;
_aiDetector = aiDetector;
_annotationService = annotationService;
_dbFactory = dbFactory;
Loaded += OnLoaded;
Closed += OnFormClosed;
Activated += (_, _) => _formState.ActiveWindow = WindowEnum.Annotator;
TbFolder.TextChanged += async (sender, args) =>
{
if (!Path.Exists(TbFolder.Text))
return;
try
{
_appConfig.DirectoriesConfig.VideosDirectory = TbFolder.Text;
ReloadFiles();
await SaveUserSettings();
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
}
};
Editor.GetTimeFunc = () => TimeSpan.FromMilliseconds(_mediaPlayer.Time);
}
@@ -98,8 +119,7 @@ public partial class Annotator
MainGrid.ColumnDefinitions.LastOrDefault()!.Width = new GridLength(_appConfig.AnnotationConfig.RightPanelWidth);
_suspendLayout = false;
ReloadFiles();
TbFolder.Text = _appConfig.DirectoriesConfig.VideosDirectory;
AnnotationClasses = new ObservableCollection<DetectionClass>(_appConfig.AnnotationConfig.AnnotationClasses);
LvClasses.ItemsSource = AnnotationClasses;
@@ -141,7 +161,7 @@ public partial class Annotator
_formState.CurrentVideoSize = new Size(vw, vh);
_formState.CurrentVideoLength = TimeSpan.FromMilliseconds(_mediaPlayer.Length);
await Dispatcher.Invoke(async () => await ReloadAnnotations(_cancellationTokenSource.Token));
await Dispatcher.Invoke(async () => await ReloadAnnotations());
if (_formState.CurrentMedia?.MediaType == MediaTypes.Image)
{
@@ -182,7 +202,7 @@ public partial class Annotator
OpenAnnotationResult((AnnotationResult)dgRow!.Item);
};
DgAnnotations.KeyUp += (sender, args) =>
DgAnnotations.KeyUp += async (sender, args) =>
{
switch (args.Key)
{
@@ -196,17 +216,9 @@ public partial class Annotator
return;
var res = DgAnnotations.SelectedItems.Cast<AnnotationResult>().ToList();
foreach (var annotationResult in res)
{
var imgName = Path.GetFileNameWithoutExtension(annotationResult.Image);
var thumbnailPath = Path.Combine(_appConfig.DirectoriesConfig.ThumbnailsDirectory, $"{imgName}{Constants.THUMBNAIL_PREFIX}.jpg");
File.Delete(annotationResult.Image);
var annotations = res.Select(x => x.Annotation).ToList();
File.Delete(Path.Combine(_appConfig.DirectoriesConfig.LabelsDirectory, $"{imgName}.txt"));
File.Delete(thumbnailPath);
_formState.AnnotationResults.Remove(annotationResult);
Detections.Remove(Detections.Query(annotationResult.Time));
}
await _mediator.Publish(new AnnotationsDeletedEvent(annotations));
break;
}
};
@@ -219,16 +231,16 @@ public partial class Annotator
{
_mediaPlayer.SetPause(true);
Editor.RemoveAllAnns();
_mediaPlayer.Time = (long)res.Time.TotalMilliseconds;
_mediaPlayer.Time = (long)res.Annotation.Time.TotalMilliseconds;
Dispatcher.Invoke(() =>
{
VideoSlider.Value = _mediaPlayer.Position * VideoSlider.Maximum;
StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}";
Editor.ClearExpiredAnnotations(res.Time);
Editor.ClearExpiredAnnotations(res.Annotation.Time);
});
AddAnnotationsToCanvas(res.Time, res.Detections, showImage: true);
ShowAnnotations(res.Annotation, showImage: true);
}
private async Task SaveUserSettings()
{
@@ -254,114 +266,73 @@ public partial class Annotator
Editor.ClearExpiredAnnotations(time);
});
var annotations = Detections.Query(time).SelectMany(x => x).Select(x => new Detection(_formState.GetTimeName(time), x));
AddAnnotationsToCanvas(time, annotations);
ShowAnnotations(TimedAnnotations.Query(time).FirstOrDefault());
}
private void AddAnnotationsToCanvas(TimeSpan? time, IEnumerable<Detection> labels, bool showImage = false)
private void ShowAnnotations(Annotation? annotation, bool showImage = false)
{
if (annotation == null)
return;
Dispatcher.Invoke(async () =>
{
var canvasSize = Editor.RenderSize;
var videoSize = _formState.CurrentVideoSize;
if (showImage)
{
var fName = _formState.GetTimeName(time);
var imgPath = Path.Combine(_appConfig.DirectoriesConfig.ImagesDirectory, $"{fName}.jpg");
if (File.Exists(imgPath))
if (File.Exists(annotation.ImagePath))
{
Editor.Background = new ImageBrush { ImageSource = await imgPath.OpenImage() };
_formState.BackgroundTime = time;
Editor.Background = new ImageBrush { ImageSource = await annotation.ImagePath.OpenImage() };
_formState.BackgroundTime = annotation.Time;
videoSize = Editor.RenderSize;
}
}
foreach (var label in labels)
foreach (var detection in annotation.Detections)
{
var annClass = _appConfig.AnnotationConfig.AnnotationClasses[label.ClassNumber];
var canvasLabel = new CanvasLabel(label, canvasSize, videoSize, label.Probability);
Editor.CreateAnnotation(annClass, time, canvasLabel);
var annClass = _appConfig.AnnotationConfig.AnnotationClasses[detection.ClassNumber];
var canvasLabel = new CanvasLabel(detection, canvasSize, videoSize, detection.Probability);
Editor.CreateDetectionControl(annClass, annotation.Time, canvasLabel);
}
});
}
private async Task ReloadAnnotations(CancellationToken ct = default)
private async Task ReloadAnnotations()
{
_formState.AnnotationResults.Clear();
Detections.Clear();
TimedAnnotations.Clear();
Editor.RemoveAllAnns();
var labelDir = new DirectoryInfo(_appConfig.DirectoriesConfig.LabelsDirectory);
if (!labelDir.Exists)
return;
var annotations = await _dbFactory.Run(async db =>
await db.Annotations.LoadWith(x => x.Detections)
.Where(x => x.Name.Contains(_formState.VideoName))
.ToListAsync(token: _ctSource.Token));
var labelFiles = labelDir.GetFiles($"{_formState.VideoName}_??????.txt");
foreach (var file in labelFiles)
await AddAnnotations(Path.GetFileNameWithoutExtension(file.Name), await YoloLabel.ReadFromFile(file.FullName, ct), ct);
foreach (var ann in annotations)
AddAnnotation(ann);
}
//Load from yolo label file
public async Task AddAnnotations(string name, List<YoloLabel> annotations, CancellationToken ct = default)
=> await AddAnnotations(name, annotations.Select(x => new Detection(name, x)).ToList(), ct);
//Add manually
public async Task AddAnnotations(string name, List<Detection> detections, CancellationToken ct = default)
public void AddAnnotation(Annotation annotation)
{
var time = Constants.GetTime(name);
var timeValue = time ?? TimeSpan.FromMinutes(0);
var previousAnnotations = Detections.Query(timeValue);
Detections.Remove(previousAnnotations);
Detections.Add(timeValue.Subtract(_thresholdBefore), timeValue.Add(_thresholdAfter), detections);
var time = annotation.Time;
var previousAnnotations = TimedAnnotations.Query(time);
TimedAnnotations.Remove(previousAnnotations);
TimedAnnotations.Add(time.Subtract(_thresholdBefore), time.Add(_thresholdAfter), annotation);
var existingResult = _formState.AnnotationResults.FirstOrDefault(x => x.Time == time);
var existingResult = _formState.AnnotationResults.FirstOrDefault(x => x.Annotation.Time == time);
if (existingResult != null)
_formState.AnnotationResults.Remove(existingResult);
var dict = _formState.AnnotationResults
.Select((x, i) => new { x.Time, Index = i })
.Select((x, i) => new { x.Annotation.Time, Index = i })
.ToDictionary(x => x.Time, x => x.Index);
var index = dict.Where(x => x.Key < timeValue)
.OrderBy(x => timeValue - x.Key)
var index = dict.Where(x => x.Key < time)
.OrderBy(x => time - x.Key)
.Select(x => x.Value + 1)
.FirstOrDefault();
_formState.AnnotationResults.Insert(index, CreateAnnotationReult(timeValue, detections));
await File.WriteAllTextAsync($"{_appConfig.DirectoriesConfig.ResultsDirectory}/{_formState.VideoName}.json", JsonConvert.SerializeObject(_formState.AnnotationResults), ct);
}
private AnnotationResult CreateAnnotationReult(TimeSpan timeValue, List<Detection> detections)
{
var annotationResult = new AnnotationResult
{
Time = timeValue,
Image = $"{_formState.GetTimeName(timeValue)}.jpg",
Detections = detections,
};
if (detections.Count <= 0)
return annotationResult;
Color GetAnnotationClass(List<int> detectionClasses, int colorNumber)
{
if (detections.Count == 0)
return (-1).ToColor();
return colorNumber >= detectionClasses.Count
? _appConfig.AnnotationConfig.DetectionClassesDict[detectionClasses.LastOrDefault()].Color
: _appConfig.AnnotationConfig.DetectionClassesDict[detectionClasses[colorNumber]].Color;
}
var detectionClasses = detections.Select(x => x.ClassNumber).Distinct().ToList();
annotationResult.ClassName = detectionClasses.Count > 1
? string.Join(", ", detectionClasses.Select(x => _appConfig.AnnotationConfig.DetectionClassesDict[x].ShortName))
: _appConfig.AnnotationConfig.DetectionClassesDict[detectionClasses.FirstOrDefault()].Name;
annotationResult.ClassColor0 = GetAnnotationClass(detectionClasses, 0);
annotationResult.ClassColor1 = GetAnnotationClass(detectionClasses, 1);
annotationResult.ClassColor2 = GetAnnotationClass(detectionClasses, 2);
annotationResult.ClassColor3 = GetAnnotationClass(detectionClasses, 3);
return annotationResult;
_formState.AnnotationResults.Insert(index, new AnnotationResult(_appConfig.AnnotationConfig.DetectionClassesDict, annotation));
}
private void ReloadFiles()
@@ -407,7 +378,6 @@ public partial class Annotator
AllMediaFiles = new ObservableCollection<MediaFileInfo>(videoFiles.Concat(imageFiles).ToList());
LvFiles.ItemsSource = AllMediaFiles;
TbFolder.Text = _appConfig.DirectoriesConfig.VideosDirectory;
BlinkHelp(AllMediaFiles.Count == 0
? HelpTexts.HelpTextsDict[HelpTextEnum.Initial]
@@ -458,16 +428,15 @@ public partial class Annotator
IsFolderPicker = true,
InitialDirectory = Path.GetDirectoryName(AppDomain.CurrentDomain.BaseDirectory)
};
if (dlg.ShowDialog() != CommonFileDialogResult.Ok)
var dialogResult = dlg.ShowDialog();
if (dialogResult != CommonFileDialogResult.Ok || string.IsNullOrEmpty(dlg.FileName))
return;
if (!string.IsNullOrEmpty(dlg.FileName))
{
_appConfig.DirectoriesConfig.VideosDirectory = dlg.FileName;
await SaveUserSettings();
}
_appConfig.DirectoriesConfig.VideosDirectory = dlg.FileName;
TbFolder.Text = dlg.FileName;
ReloadFiles();
await SaveUserSettings();
}
private void TbFilter_OnTextChanged(object sender, TextChangedEventArgs e)
@@ -546,7 +515,7 @@ public partial class Annotator
while (mediaInfo != null)
{
_formState.CurrentMedia = mediaInfo;
await Dispatcher.Invoke(async () => await ReloadAnnotations(token));
await Dispatcher.Invoke(async () => await ReloadAnnotations());
if (mediaInfo.MediaType == MediaTypes.Image)
{
@@ -632,7 +601,7 @@ public partial class Annotator
continue;
mediaInfo.HasAnnotations = true;
await ProcessDetection(timeframe, "jpg", detections, token);
await ProcessDetection(timeframe, ".jpg", detections, token);
}
catch (Exception ex)
{
@@ -697,17 +666,15 @@ public partial class Annotator
try
{
var time = timeframe.Time;
var fName = _formState.GetTimeName(timeframe.Time);
var imgPath = Path.Combine(_appConfig.DirectoriesConfig.ImagesDirectory, $"{fName}.{imageExtension}");
Editor.Background = new ImageBrush { ImageSource = await imgPath.OpenImage() };
var annotation = await _annotationService.SaveAnnotation(fName, imageExtension, detections, SourceEnum.AI, timeframe.Stream, token);
Editor.Background = new ImageBrush { ImageSource = await annotation.ImagePath.OpenImage() };
Editor.RemoveAllAnns();
AddAnnotationsToCanvas(time, detections, true);
await AddAnnotations(fName, detections, token);
ShowAnnotations(annotation, true);
AddAnnotation(annotation);
await _annotationService.SaveAnnotation(fName, imageExtension, detections, SourceEnum.AI, timeframe.Stream, token);
var log = string.Join(Environment.NewLine, detections.Select(det =>
$"{_appConfig.AnnotationConfig.DetectionClassesDict[det.ClassNumber].Name}: " +
$"xy=({det.CenterX:F2},{det.CenterY:F2}), " +
@@ -724,4 +691,14 @@ public partial class Annotator
}
});
}
public async Task Handle(AnnotationsDeletedEvent notification, CancellationToken cancellationToken)
{
var annResDict = _formState.AnnotationResults.ToDictionary(x => x.Annotation.Name, x => x);
foreach (var ann in notification.Annotations)
{
_formState.AnnotationResults.Remove(annResDict[ann.Name]);
TimedAnnotations.Remove(ann);
}
}
}