azaion/annotations
1
0
Fork 0
mirror of https://github.com/azaion/annotations.git synced 2026-04-22 16:16:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix ai detection bugs #1

Browse Source
This commit is contained in:
Alex Bezdieniezhnykh
2024-11-04 21:12:45 +02:00
parent addf7ccc11
commit d8f60d7491
10 changed files with 172 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Azaion.Annotator/MainWindow.xaml.cs
+59 -40
View File
@@ -18,6 +18,7 @@ using Size = System.Windows.Size;
using IntervalTree;
using Microsoft.Extensions.Logging;
using OpenTK.Graphics.OpenGL;
using ScottPlot.TickGenerators.TimeUnits;
using Serilog;
using MediaPlayer = LibVLCSharp.Shared.MediaPlayer;
@@ -155,6 +156,7 @@ public partial class MainWindow
{
VideoView.MediaPlayer = _mediaPlayer;
//On start playing media
_mediaPlayer.Playing += async (sender, args) =>
{
if (_formState.CurrentMrl == _mediaPlayer.Media?.Mrl)
@@ -167,13 +169,13 @@ public partial class MainWindow
_formState.CurrentVideoLength = TimeSpan.FromMilliseconds(_mediaPlayer.Length);
await Dispatcher.Invoke(async () => await ReloadAnnotations(_cancellationTokenSource.Token));
if (_formState.CurrentMedia?.MediaType != MediaTypes.Image)
return;
//if image show annotations, give 100ms to load the frame and set on pause
await Task.Delay(100);
ShowCurrentAnnotations();
_mediaPlayer.SetPause(true);
if (_formState.CurrentMedia?.MediaType == MediaTypes.Image)
{
await Task.Delay(100); //wait to load the frame and set on pause
ShowTimeAnnotations(TimeSpan.FromMilliseconds(_mediaPlayer.Time));
_mediaPlayer.SetPause(true);
}
};
LvFiles.MouseDoubleClick += async (_, _) => await _mediator.Publish(new PlaybackControlEvent(PlaybackControlEnum.Play));
@@ -203,13 +205,12 @@ public partial class MainWindow
DgAnnotations.MouseDoubleClick += (sender, args) =>
{
Editor.RemoveAllAnns();
var dgRow = ItemsControl.ContainerFromElement((DataGrid)sender, (args.OriginalSource as DependencyObject)!) as DataGridRow;
var res = (AnnotationResult)dgRow!.Item;
_mediaPlayer.SetPause(true);
Editor.RemoveAllAnns();
_mediaPlayer.Time = (long)res.Time.TotalMilliseconds;
ShowTimeAnnotations(res.Time);
ShowTimeAnnotations(res.Time, showImage: true);
};
DgAnnotations.KeyUp += (sender, args) =>
@@ -255,27 +256,47 @@ public partial class MainWindow
}, TimeSpan.FromSeconds(5));
}
public void ShowCurrentAnnotations() => ShowTimeAnnotations(TimeSpan.FromMilliseconds(_mediaPlayer.Time));
private void ShowTimeAnnotations(TimeSpan time)
private void ShowTimeAnnotations(TimeSpan time, bool showImage = false)
{
Dispatcher.Invoke(() => VideoSlider.Value = _mediaPlayer.Position * VideoSlider.Maximum);
Dispatcher.Invoke(() => StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}");
Dispatcher.Invoke(() =>
{
VideoSlider.Value = _mediaPlayer.Position * VideoSlider.Maximum;
StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}";
Editor.ClearExpiredAnnotations(time);
});
Dispatcher.Invoke(() => Editor.ClearExpiredAnnotations(time));
var annotations = Annotations.Query(time).SelectMany(x => x).ToList();
foreach (var ann in annotations)
AddAnnotationToCanvas(time, new CanvasLabel(ann, Editor.RenderSize, _formState.CurrentVideoSize));
var annotations = Annotations.Query(time).SelectMany(x => x).Select(x => new Detection(x));
AddAnnotationsToCanvas(time, annotations, showImage);
}
private void AddAnnotationToCanvas(TimeSpan? time, CanvasLabel canvasLabel)
private void AddAnnotationsToCanvas(TimeSpan? time, IEnumerable<Detection> labels, bool showImage = false)
{
var annClass = _config.AnnotationClasses[canvasLabel.ClassNumber];
Dispatcher.Invoke(() => Editor.CreateAnnotation(annClass, time, canvasLabel));
Dispatcher.Invoke(async () =>
{
var canvasSize = Editor.RenderSize;
var videoSize = _formState.CurrentVideoSize;
if (showImage)
{
var fName = _formState.GetTimeName(time);
var imgPath = Path.Combine(_config.ImagesDirectory, $"{fName}.jpg");
if (File.Exists(imgPath))
{
Editor.Background = new ImageBrush { ImageSource = await imgPath.OpenImage() };
_formState.BackgroundShown = true;
videoSize = Editor.RenderSize;
}
}
foreach (var label in labels)
{
var annClass = _config.AnnotationClasses[label.ClassNumber];
var canvasLabel = new CanvasLabel(label, canvasSize, videoSize, label.Probability);
Editor.CreateAnnotation(annClass, time, canvasLabel);
}
});
}
private async Task ReloadAnnotations(CancellationToken cancellationToken)
private async Task ReloadAnnotations(CancellationToken ct = default)
{
_formState.AnnotationResults.Clear();
Annotations.Clear();
@@ -290,11 +311,11 @@ public partial class MainWindow
{
var name = Path.GetFileNameWithoutExtension(file.Name);
var time = _formState.GetTime(name);
await AddAnnotations(time, await YoloLabel.ReadFromFile(file.FullName, cancellationToken));
await AddAnnotations(time, await YoloLabel.ReadFromFile(file.FullName, ct), ct);
}
}
public async Task AddAnnotations(TimeSpan? time, List<YoloLabel> annotations)
public async Task AddAnnotations(TimeSpan? time, List<YoloLabel> annotations, CancellationToken ct = default)
{
var timeValue = time ?? TimeSpan.FromMinutes(0);
var previousAnnotations = Annotations.Query(timeValue);
@@ -315,7 +336,7 @@ public partial class MainWindow
.FirstOrDefault();
_formState.AnnotationResults.Insert(index, new AnnotationResult(timeValue, _formState.GetTimeName(time), annotations, _config));
await File.WriteAllTextAsync($"{_config.ResultsDirectory}/{_formState.VideoName}.json", JsonConvert.SerializeObject(_formState.AnnotationResults));
await File.WriteAllTextAsync($"{_config.ResultsDirectory}/{_formState.VideoName}.json", JsonConvert.SerializeObject(_formState.AnnotationResults), ct);
}
private void ReloadFiles()
@@ -481,7 +502,7 @@ public partial class MainWindow
LvFilesContextMenu.DataContext = listItem.DataContext;
}
private (TimeSpan Time, List<(YoloLabel Label, float Probability)> Detections)? _previousDetection;
private (TimeSpan Time, List<Detection> Detections)? _previousDetection;
public void AutoDetect(object sender, RoutedEventArgs e)
{
@@ -540,14 +561,14 @@ public partial class MainWindow
await manualCancellationSource.CancelAsync();
}
}
_autoDetectDialog.Close();
Dispatcher.Invoke(() => _autoDetectDialog.Close());
}, token);
_autoDetectDialog.ShowDialog();
Dispatcher.Invoke(() => Editor.Background = new SolidColorBrush(Color.FromArgb(1, 0, 0, 0)));
}
private bool IsValidDetection(TimeSpan time, List<(YoloLabel Label, float Probability)> detections)
private bool IsValidDetection(TimeSpan time, List<Detection> detections)
{
// No AI detection, forbid
if (detections.Count == 0)
@@ -572,12 +593,12 @@ public partial class MainWindow
foreach (var det in detections)
{
var point = new Point(det.Label.CenterX, det.Label.CenterY);
var point = new Point(det.CenterX, det.CenterY);
var closestObject = prev.Detections
.Select(p => new
{
Point = p,
Distance = point.SqrDistance(new Point(p.Label.CenterX, p.Label.CenterY))
Distance = point.SqrDistance(new Point(p.CenterX, p.CenterY))
})
.OrderBy(x => x.Distance)
.First();
@@ -594,7 +615,7 @@ public partial class MainWindow
return false;
}
private async Task ProcessDetection((TimeSpan Time, Stream Stream) timeframe, List<(YoloLabel Label, float Probability)> detections, CancellationToken token = default)
private async Task ProcessDetection((TimeSpan Time, Stream Stream) timeframe, List<Detection> detections, CancellationToken token = default)
{
_previousDetection = (timeframe.Time, detections);
await Dispatcher.Invoke(async () =>
@@ -602,31 +623,29 @@ public partial class MainWindow
try
{
var time = timeframe.Time;
var labels = detections.Select(x => x.Label).ToList();
var fName = _formState.GetTimeName(timeframe.Time);
var imgPath = Path.Combine(_config.ImagesDirectory, $"{fName}.jpg");
var img = System.Drawing.Image.FromStream(timeframe.Stream);
img.Save(imgPath, ImageFormat.Jpeg);
await YoloLabel.WriteToFile(labels, Path.Combine(_config.LabelsDirectory, $"{fName}.txt"), token);
await YoloLabel.WriteToFile(detections, Path.Combine(_config.LabelsDirectory, $"{fName}.txt"), token);
Editor.Background = new ImageBrush { ImageSource = await imgPath.OpenImage() };
Editor.RemoveAllAnns();
foreach (var (label, probability) in detections)
AddAnnotationToCanvas(time, new CanvasLabel(label, Editor.RenderSize, Editor.RenderSize, probability));
await AddAnnotations(timeframe.Time, labels);
AddAnnotationsToCanvas(time, detections, true);
await AddAnnotations(timeframe.Time, detections.Cast<YoloLabel>().ToList(), token);
var log = string.Join(Environment.NewLine, detections.Select(det =>
$"{_config.AnnotationClassesDict[det.Label.ClassNumber].Name}: " +
$"xy=({det.Label.CenterX:F2},{det.Label.CenterY:F2}), " +
$"size=({det.Label.Width:F2}, {det.Label.Height:F2}), " +
$"{_config.AnnotationClassesDict[det.ClassNumber].Name}: " +
$"xy=({det.CenterX:F2},{det.CenterY:F2}), " +
$"size=({det.Width:F2}, {det.Height:F2}), " +
$"prob: {det.Probability:F1}%"));
Dispatcher.Invoke(() => _autoDetectDialog.Log(log));
var thumbnailDto = await _galleryManager.CreateThumbnail(imgPath, token);
if (thumbnailDto != null)
_datasetExplorer.AddThumbnail(thumbnailDto, labels.Select(x => x.ClassNumber));
_datasetExplorer.AddThumbnail(thumbnailDto, detections.Select(x => x.ClassNumber));
}
catch (Exception e)
{