mirror of
https://github.com/azaion/annotations.git
synced 2026-04-22 11:16:30 +00:00
fix ai detection bugs #1
This commit is contained in:
@@ -17,7 +17,6 @@ public class AnnotationControl : Border
|
|||||||
private readonly TextBlock _classNameLabel;
|
private readonly TextBlock _classNameLabel;
|
||||||
private readonly Label _probabilityLabel;
|
private readonly Label _probabilityLabel;
|
||||||
public TimeSpan? Time { get; set; }
|
public TimeSpan? Time { get; set; }
|
||||||
public double? Probability { get; set; }
|
|
||||||
|
|
||||||
private AnnotationClass _annotationClass = null!;
|
private AnnotationClass _annotationClass = null!;
|
||||||
public AnnotationClass AnnotationClass
|
public AnnotationClass AnnotationClass
|
||||||
@@ -84,7 +83,6 @@ public class AnnotationControl : Border
|
|||||||
{
|
{
|
||||||
_selectionFrame,
|
_selectionFrame,
|
||||||
_classNameLabel,
|
_classNameLabel,
|
||||||
_probabilityLabel,
|
|
||||||
AddRect("rLT", HorizontalAlignment.Left, VerticalAlignment.Top, Cursors.SizeNWSE),
|
AddRect("rLT", HorizontalAlignment.Left, VerticalAlignment.Top, Cursors.SizeNWSE),
|
||||||
AddRect("rCT", HorizontalAlignment.Center, VerticalAlignment.Top, Cursors.SizeNS),
|
AddRect("rCT", HorizontalAlignment.Center, VerticalAlignment.Top, Cursors.SizeNS),
|
||||||
AddRect("rRT", HorizontalAlignment.Right, VerticalAlignment.Top, Cursors.SizeNESW),
|
AddRect("rRT", HorizontalAlignment.Right, VerticalAlignment.Top, Cursors.SizeNESW),
|
||||||
@@ -95,6 +93,8 @@ public class AnnotationControl : Border
|
|||||||
AddRect("rRB", HorizontalAlignment.Right, VerticalAlignment.Bottom, Cursors.SizeNWSE)
|
AddRect("rRB", HorizontalAlignment.Right, VerticalAlignment.Bottom, Cursors.SizeNWSE)
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
if (probability.HasValue)
|
||||||
|
_grid.Children.Add(_probabilityLabel);
|
||||||
Child = _grid;
|
Child = _grid;
|
||||||
Cursor = Cursors.SizeAll;
|
Cursor = Cursors.SizeAll;
|
||||||
AnnotationClass = annotationClass;
|
AnnotationClass = annotationClass;
|
||||||
|
|||||||
@@ -46,6 +46,7 @@ public class AIRecognitionConfig
|
|||||||
public double FrameRecognitionSeconds { get; set; }
|
public double FrameRecognitionSeconds { get; set; }
|
||||||
public double TrackingDistanceConfidence { get; set; }
|
public double TrackingDistanceConfidence { get; set; }
|
||||||
public double TrackingProbabilityIncrease { get; set; }
|
public double TrackingProbabilityIncrease { get; set; }
|
||||||
|
public double TrackingIntersectionThreshold { get; set; }
|
||||||
}
|
}
|
||||||
|
|
||||||
public class WindowConfig
|
public class WindowConfig
|
||||||
@@ -83,6 +84,7 @@ public class FileConfigRepository(ILogger<FileConfigRepository> logger) : IConfi
|
|||||||
private const double DEFAULT_FRAME_RECOGNITION_SECONDS = 2;
|
private const double DEFAULT_FRAME_RECOGNITION_SECONDS = 2;
|
||||||
private const double TRACKING_DISTANCE_CONFIDENCE = 0.15;
|
private const double TRACKING_DISTANCE_CONFIDENCE = 0.15;
|
||||||
private const double TRACKING_PROBABILITY_INCREASE = 15;
|
private const double TRACKING_PROBABILITY_INCREASE = 15;
|
||||||
|
private const double TRACKING_INTERSECTION_THRESHOLD = 0.8;
|
||||||
|
|
||||||
private static readonly Size DefaultWindowSize = new(1280, 720);
|
private static readonly Size DefaultWindowSize = new(1280, 720);
|
||||||
private static readonly Point DefaultWindowLocation = new(100, 100);
|
private static readonly Point DefaultWindowLocation = new(100, 100);
|
||||||
@@ -131,7 +133,8 @@ public class FileConfigRepository(ILogger<FileConfigRepository> logger) : IConfi
|
|||||||
AIModelPath = "azaion.onnx",
|
AIModelPath = "azaion.onnx",
|
||||||
FrameRecognitionSeconds = DEFAULT_FRAME_RECOGNITION_SECONDS,
|
FrameRecognitionSeconds = DEFAULT_FRAME_RECOGNITION_SECONDS,
|
||||||
TrackingDistanceConfidence = TRACKING_DISTANCE_CONFIDENCE,
|
TrackingDistanceConfidence = TRACKING_DISTANCE_CONFIDENCE,
|
||||||
TrackingProbabilityIncrease = TRACKING_PROBABILITY_INCREASE
|
TrackingProbabilityIncrease = TRACKING_PROBABILITY_INCREASE,
|
||||||
|
TrackingIntersectionThreshold = TRACKING_INTERSECTION_THRESHOLD
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ public class FormState
|
|||||||
public Size CurrentVideoSize { get; set; }
|
public Size CurrentVideoSize { get; set; }
|
||||||
public TimeSpan CurrentVideoLength { get; set; }
|
public TimeSpan CurrentVideoLength { get; set; }
|
||||||
|
|
||||||
|
public bool BackgroundShown { get; set; }
|
||||||
public int CurrentVolume { get; set; } = 100;
|
public int CurrentVolume { get; set; } = 100;
|
||||||
public ObservableCollection<AnnotationResult> AnnotationResults { get; set; } = [];
|
public ObservableCollection<AnnotationResult> AnnotationResults { get; set; } = [];
|
||||||
public WindowsEnum ActiveWindow { get; set; }
|
public WindowsEnum ActiveWindow { get; set; }
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
using System.Globalization;
|
using System.Drawing;
|
||||||
|
using System.Globalization;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
using System.Windows;
|
|
||||||
using Newtonsoft.Json;
|
using Newtonsoft.Json;
|
||||||
|
using Size = System.Windows.Size;
|
||||||
|
|
||||||
namespace Azaion.Annotator.DTO;
|
namespace Azaion.Annotator.DTO;
|
||||||
|
|
||||||
@@ -98,6 +99,9 @@ public class YoloLabel : Label
|
|||||||
Height = height;
|
Height = height;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public RectangleF ToRectangle() =>
|
||||||
|
new((float)(CenterX - Width / 2.0), (float)(CenterY - Height / 2.0), (float)Width, (float)Height);
|
||||||
|
|
||||||
public YoloLabel(CanvasLabel canvasLabel, Size canvasSize, Size videoSize)
|
public YoloLabel(CanvasLabel canvasLabel, Size canvasSize, Size videoSize)
|
||||||
{
|
{
|
||||||
var cw = canvasSize.Width;
|
var cw = canvasSize.Width;
|
||||||
@@ -169,3 +173,17 @@ public class YoloLabel : Label
|
|||||||
|
|
||||||
public override string ToString() => $"{ClassNumber} {CenterX:F5} {CenterY:F5} {Width:F5} {Height:F5}".Replace(',', '.');
|
public override string ToString() => $"{ClassNumber} {CenterX:F5} {CenterY:F5} {Width:F5} {Height:F5}".Replace(',', '.');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public class Detection : YoloLabel
|
||||||
|
{
|
||||||
|
public Detection(YoloLabel label, double? probability = null)
|
||||||
|
{
|
||||||
|
ClassNumber = label.ClassNumber;
|
||||||
|
CenterX = label.CenterX;
|
||||||
|
CenterY = label.CenterY;
|
||||||
|
Height = label.Height;
|
||||||
|
Width = label.Width;
|
||||||
|
Probability = probability;
|
||||||
|
}
|
||||||
|
public double? Probability { get; set; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
using System.Drawing;
|
||||||
|
|
||||||
|
namespace Azaion.Annotator.Extensions;
|
||||||
|
|
||||||
|
public static class RectangleFExtensions
|
||||||
|
{
|
||||||
|
public static double Area(this RectangleF rectangle) => rectangle.Width * rectangle.Height;
|
||||||
|
}
|
||||||
@@ -36,13 +36,13 @@ public class VLCFrameExtractor(LibVLC libVLC)
|
|||||||
public async IAsyncEnumerable<(TimeSpan Time, Stream Stream)> ExtractFrames(string mediaPath,
|
public async IAsyncEnumerable<(TimeSpan Time, Stream Stream)> ExtractFrames(string mediaPath,
|
||||||
[EnumeratorCancellation] CancellationToken manualCancellationToken = default)
|
[EnumeratorCancellation] CancellationToken manualCancellationToken = default)
|
||||||
{
|
{
|
||||||
var videoFinishedCancellationToken = new CancellationTokenSource();
|
var videoFinishedCancellationSource = new CancellationTokenSource();
|
||||||
|
|
||||||
_mediaPlayer = new MediaPlayer(libVLC);
|
_mediaPlayer = new MediaPlayer(libVLC);
|
||||||
_mediaPlayer.Stopped += (s, e) => videoFinishedCancellationToken.CancelAfter(1);
|
_mediaPlayer.Stopped += (s, e) => videoFinishedCancellationSource.CancelAfter(1);
|
||||||
|
|
||||||
using var media = new Media(libVLC, mediaPath);
|
using var media = new Media(libVLC, mediaPath);
|
||||||
await media.Parse(cancellationToken: videoFinishedCancellationToken.Token);
|
await media.Parse(cancellationToken: videoFinishedCancellationSource.Token);
|
||||||
var videoTrack = media.Tracks.FirstOrDefault(x => x.Data.Video.Width != 0);
|
var videoTrack = media.Tracks.FirstOrDefault(x => x.Data.Video.Width != 0);
|
||||||
_width = videoTrack.Data.Video.Width;
|
_width = videoTrack.Data.Video.Width;
|
||||||
_height = videoTrack.Data.Video.Height;
|
_height = videoTrack.Data.Video.Height;
|
||||||
@@ -58,9 +58,9 @@ public class VLCFrameExtractor(LibVLC libVLC)
|
|||||||
_mediaPlayer.Play(media);
|
_mediaPlayer.Play(media);
|
||||||
_frameCounter = 0;
|
_frameCounter = 0;
|
||||||
var surface = SKSurface.Create(new SKImageInfo((int) _width, (int) _height));
|
var surface = SKSurface.Create(new SKImageInfo((int) _width, (int) _height));
|
||||||
var token = videoFinishedCancellationToken.Token;
|
var videoFinishedCT = videoFinishedCancellationSource.Token;
|
||||||
|
|
||||||
while (!(FramesQueue.IsEmpty && token.IsCancellationRequested) && !manualCancellationToken.IsCancellationRequested)
|
while ( !(FramesQueue.IsEmpty && videoFinishedCT.IsCancellationRequested || manualCancellationToken.IsCancellationRequested))
|
||||||
{
|
{
|
||||||
if (FramesQueue.TryDequeue(out var frameInfo))
|
if (FramesQueue.TryDequeue(out var frameInfo))
|
||||||
{
|
{
|
||||||
@@ -80,9 +80,10 @@ public class VLCFrameExtractor(LibVLC libVLC)
|
|||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
await Task.Delay(TimeSpan.FromSeconds(1), token);
|
await Task.Delay(TimeSpan.FromSeconds(1), videoFinishedCT);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
FramesQueue.Clear(); //clear queue in case of manual stop
|
||||||
_mediaPlayer.Stop();
|
_mediaPlayer.Stop();
|
||||||
_mediaPlayer.Dispose();
|
_mediaPlayer.Dispose();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ using Size = System.Windows.Size;
|
|||||||
using IntervalTree;
|
using IntervalTree;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
using OpenTK.Graphics.OpenGL;
|
using OpenTK.Graphics.OpenGL;
|
||||||
|
using ScottPlot.TickGenerators.TimeUnits;
|
||||||
using Serilog;
|
using Serilog;
|
||||||
using MediaPlayer = LibVLCSharp.Shared.MediaPlayer;
|
using MediaPlayer = LibVLCSharp.Shared.MediaPlayer;
|
||||||
|
|
||||||
@@ -155,6 +156,7 @@ public partial class MainWindow
|
|||||||
{
|
{
|
||||||
VideoView.MediaPlayer = _mediaPlayer;
|
VideoView.MediaPlayer = _mediaPlayer;
|
||||||
|
|
||||||
|
//On start playing media
|
||||||
_mediaPlayer.Playing += async (sender, args) =>
|
_mediaPlayer.Playing += async (sender, args) =>
|
||||||
{
|
{
|
||||||
if (_formState.CurrentMrl == _mediaPlayer.Media?.Mrl)
|
if (_formState.CurrentMrl == _mediaPlayer.Media?.Mrl)
|
||||||
@@ -167,13 +169,13 @@ public partial class MainWindow
|
|||||||
_formState.CurrentVideoLength = TimeSpan.FromMilliseconds(_mediaPlayer.Length);
|
_formState.CurrentVideoLength = TimeSpan.FromMilliseconds(_mediaPlayer.Length);
|
||||||
|
|
||||||
await Dispatcher.Invoke(async () => await ReloadAnnotations(_cancellationTokenSource.Token));
|
await Dispatcher.Invoke(async () => await ReloadAnnotations(_cancellationTokenSource.Token));
|
||||||
if (_formState.CurrentMedia?.MediaType != MediaTypes.Image)
|
|
||||||
return;
|
|
||||||
|
|
||||||
//if image show annotations, give 100ms to load the frame and set on pause
|
if (_formState.CurrentMedia?.MediaType == MediaTypes.Image)
|
||||||
await Task.Delay(100);
|
{
|
||||||
ShowCurrentAnnotations();
|
await Task.Delay(100); //wait to load the frame and set on pause
|
||||||
_mediaPlayer.SetPause(true);
|
ShowTimeAnnotations(TimeSpan.FromMilliseconds(_mediaPlayer.Time));
|
||||||
|
_mediaPlayer.SetPause(true);
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
LvFiles.MouseDoubleClick += async (_, _) => await _mediator.Publish(new PlaybackControlEvent(PlaybackControlEnum.Play));
|
LvFiles.MouseDoubleClick += async (_, _) => await _mediator.Publish(new PlaybackControlEvent(PlaybackControlEnum.Play));
|
||||||
@@ -203,13 +205,12 @@ public partial class MainWindow
|
|||||||
|
|
||||||
DgAnnotations.MouseDoubleClick += (sender, args) =>
|
DgAnnotations.MouseDoubleClick += (sender, args) =>
|
||||||
{
|
{
|
||||||
Editor.RemoveAllAnns();
|
|
||||||
var dgRow = ItemsControl.ContainerFromElement((DataGrid)sender, (args.OriginalSource as DependencyObject)!) as DataGridRow;
|
var dgRow = ItemsControl.ContainerFromElement((DataGrid)sender, (args.OriginalSource as DependencyObject)!) as DataGridRow;
|
||||||
var res = (AnnotationResult)dgRow!.Item;
|
var res = (AnnotationResult)dgRow!.Item;
|
||||||
_mediaPlayer.SetPause(true);
|
_mediaPlayer.SetPause(true);
|
||||||
Editor.RemoveAllAnns();
|
Editor.RemoveAllAnns();
|
||||||
_mediaPlayer.Time = (long)res.Time.TotalMilliseconds;
|
_mediaPlayer.Time = (long)res.Time.TotalMilliseconds;
|
||||||
ShowTimeAnnotations(res.Time);
|
ShowTimeAnnotations(res.Time, showImage: true);
|
||||||
};
|
};
|
||||||
|
|
||||||
DgAnnotations.KeyUp += (sender, args) =>
|
DgAnnotations.KeyUp += (sender, args) =>
|
||||||
@@ -255,27 +256,47 @@ public partial class MainWindow
|
|||||||
}, TimeSpan.FromSeconds(5));
|
}, TimeSpan.FromSeconds(5));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void ShowCurrentAnnotations() => ShowTimeAnnotations(TimeSpan.FromMilliseconds(_mediaPlayer.Time));
|
private void ShowTimeAnnotations(TimeSpan time, bool showImage = false)
|
||||||
|
|
||||||
private void ShowTimeAnnotations(TimeSpan time)
|
|
||||||
{
|
{
|
||||||
Dispatcher.Invoke(() => VideoSlider.Value = _mediaPlayer.Position * VideoSlider.Maximum);
|
Dispatcher.Invoke(() =>
|
||||||
Dispatcher.Invoke(() => StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}");
|
{
|
||||||
|
VideoSlider.Value = _mediaPlayer.Position * VideoSlider.Maximum;
|
||||||
|
StatusClock.Text = $"{TimeSpan.FromMilliseconds(_mediaPlayer.Time):mm\\:ss} / {_formState.CurrentVideoLength:mm\\:ss}";
|
||||||
|
Editor.ClearExpiredAnnotations(time);
|
||||||
|
});
|
||||||
|
|
||||||
Dispatcher.Invoke(() => Editor.ClearExpiredAnnotations(time));
|
var annotations = Annotations.Query(time).SelectMany(x => x).Select(x => new Detection(x));
|
||||||
|
AddAnnotationsToCanvas(time, annotations, showImage);
|
||||||
var annotations = Annotations.Query(time).SelectMany(x => x).ToList();
|
|
||||||
foreach (var ann in annotations)
|
|
||||||
AddAnnotationToCanvas(time, new CanvasLabel(ann, Editor.RenderSize, _formState.CurrentVideoSize));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void AddAnnotationToCanvas(TimeSpan? time, CanvasLabel canvasLabel)
|
private void AddAnnotationsToCanvas(TimeSpan? time, IEnumerable<Detection> labels, bool showImage = false)
|
||||||
{
|
{
|
||||||
var annClass = _config.AnnotationClasses[canvasLabel.ClassNumber];
|
Dispatcher.Invoke(async () =>
|
||||||
Dispatcher.Invoke(() => Editor.CreateAnnotation(annClass, time, canvasLabel));
|
{
|
||||||
|
var canvasSize = Editor.RenderSize;
|
||||||
|
var videoSize = _formState.CurrentVideoSize;
|
||||||
|
if (showImage)
|
||||||
|
{
|
||||||
|
var fName = _formState.GetTimeName(time);
|
||||||
|
var imgPath = Path.Combine(_config.ImagesDirectory, $"{fName}.jpg");
|
||||||
|
if (File.Exists(imgPath))
|
||||||
|
{
|
||||||
|
Editor.Background = new ImageBrush { ImageSource = await imgPath.OpenImage() };
|
||||||
|
_formState.BackgroundShown = true;
|
||||||
|
videoSize = Editor.RenderSize;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
foreach (var label in labels)
|
||||||
|
{
|
||||||
|
var annClass = _config.AnnotationClasses[label.ClassNumber];
|
||||||
|
var canvasLabel = new CanvasLabel(label, canvasSize, videoSize, label.Probability);
|
||||||
|
Editor.CreateAnnotation(annClass, time, canvasLabel);
|
||||||
|
}
|
||||||
|
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task ReloadAnnotations(CancellationToken cancellationToken)
|
private async Task ReloadAnnotations(CancellationToken ct = default)
|
||||||
{
|
{
|
||||||
_formState.AnnotationResults.Clear();
|
_formState.AnnotationResults.Clear();
|
||||||
Annotations.Clear();
|
Annotations.Clear();
|
||||||
@@ -290,11 +311,11 @@ public partial class MainWindow
|
|||||||
{
|
{
|
||||||
var name = Path.GetFileNameWithoutExtension(file.Name);
|
var name = Path.GetFileNameWithoutExtension(file.Name);
|
||||||
var time = _formState.GetTime(name);
|
var time = _formState.GetTime(name);
|
||||||
await AddAnnotations(time, await YoloLabel.ReadFromFile(file.FullName, cancellationToken));
|
await AddAnnotations(time, await YoloLabel.ReadFromFile(file.FullName, ct), ct);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task AddAnnotations(TimeSpan? time, List<YoloLabel> annotations)
|
public async Task AddAnnotations(TimeSpan? time, List<YoloLabel> annotations, CancellationToken ct = default)
|
||||||
{
|
{
|
||||||
var timeValue = time ?? TimeSpan.FromMinutes(0);
|
var timeValue = time ?? TimeSpan.FromMinutes(0);
|
||||||
var previousAnnotations = Annotations.Query(timeValue);
|
var previousAnnotations = Annotations.Query(timeValue);
|
||||||
@@ -315,7 +336,7 @@ public partial class MainWindow
|
|||||||
.FirstOrDefault();
|
.FirstOrDefault();
|
||||||
|
|
||||||
_formState.AnnotationResults.Insert(index, new AnnotationResult(timeValue, _formState.GetTimeName(time), annotations, _config));
|
_formState.AnnotationResults.Insert(index, new AnnotationResult(timeValue, _formState.GetTimeName(time), annotations, _config));
|
||||||
await File.WriteAllTextAsync($"{_config.ResultsDirectory}/{_formState.VideoName}.json", JsonConvert.SerializeObject(_formState.AnnotationResults));
|
await File.WriteAllTextAsync($"{_config.ResultsDirectory}/{_formState.VideoName}.json", JsonConvert.SerializeObject(_formState.AnnotationResults), ct);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void ReloadFiles()
|
private void ReloadFiles()
|
||||||
@@ -481,7 +502,7 @@ public partial class MainWindow
|
|||||||
LvFilesContextMenu.DataContext = listItem.DataContext;
|
LvFilesContextMenu.DataContext = listItem.DataContext;
|
||||||
}
|
}
|
||||||
|
|
||||||
private (TimeSpan Time, List<(YoloLabel Label, float Probability)> Detections)? _previousDetection;
|
private (TimeSpan Time, List<Detection> Detections)? _previousDetection;
|
||||||
|
|
||||||
public void AutoDetect(object sender, RoutedEventArgs e)
|
public void AutoDetect(object sender, RoutedEventArgs e)
|
||||||
{
|
{
|
||||||
@@ -540,14 +561,14 @@ public partial class MainWindow
|
|||||||
await manualCancellationSource.CancelAsync();
|
await manualCancellationSource.CancelAsync();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
_autoDetectDialog.Close();
|
Dispatcher.Invoke(() => _autoDetectDialog.Close());
|
||||||
}, token);
|
}, token);
|
||||||
_autoDetectDialog.ShowDialog();
|
_autoDetectDialog.ShowDialog();
|
||||||
|
|
||||||
Dispatcher.Invoke(() => Editor.Background = new SolidColorBrush(Color.FromArgb(1, 0, 0, 0)));
|
Dispatcher.Invoke(() => Editor.Background = new SolidColorBrush(Color.FromArgb(1, 0, 0, 0)));
|
||||||
}
|
}
|
||||||
|
|
||||||
private bool IsValidDetection(TimeSpan time, List<(YoloLabel Label, float Probability)> detections)
|
private bool IsValidDetection(TimeSpan time, List<Detection> detections)
|
||||||
{
|
{
|
||||||
// No AI detection, forbid
|
// No AI detection, forbid
|
||||||
if (detections.Count == 0)
|
if (detections.Count == 0)
|
||||||
@@ -572,12 +593,12 @@ public partial class MainWindow
|
|||||||
|
|
||||||
foreach (var det in detections)
|
foreach (var det in detections)
|
||||||
{
|
{
|
||||||
var point = new Point(det.Label.CenterX, det.Label.CenterY);
|
var point = new Point(det.CenterX, det.CenterY);
|
||||||
var closestObject = prev.Detections
|
var closestObject = prev.Detections
|
||||||
.Select(p => new
|
.Select(p => new
|
||||||
{
|
{
|
||||||
Point = p,
|
Point = p,
|
||||||
Distance = point.SqrDistance(new Point(p.Label.CenterX, p.Label.CenterY))
|
Distance = point.SqrDistance(new Point(p.CenterX, p.CenterY))
|
||||||
})
|
})
|
||||||
.OrderBy(x => x.Distance)
|
.OrderBy(x => x.Distance)
|
||||||
.First();
|
.First();
|
||||||
@@ -594,7 +615,7 @@ public partial class MainWindow
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task ProcessDetection((TimeSpan Time, Stream Stream) timeframe, List<(YoloLabel Label, float Probability)> detections, CancellationToken token = default)
|
private async Task ProcessDetection((TimeSpan Time, Stream Stream) timeframe, List<Detection> detections, CancellationToken token = default)
|
||||||
{
|
{
|
||||||
_previousDetection = (timeframe.Time, detections);
|
_previousDetection = (timeframe.Time, detections);
|
||||||
await Dispatcher.Invoke(async () =>
|
await Dispatcher.Invoke(async () =>
|
||||||
@@ -602,31 +623,29 @@ public partial class MainWindow
|
|||||||
try
|
try
|
||||||
{
|
{
|
||||||
var time = timeframe.Time;
|
var time = timeframe.Time;
|
||||||
var labels = detections.Select(x => x.Label).ToList();
|
|
||||||
|
|
||||||
var fName = _formState.GetTimeName(timeframe.Time);
|
var fName = _formState.GetTimeName(timeframe.Time);
|
||||||
var imgPath = Path.Combine(_config.ImagesDirectory, $"{fName}.jpg");
|
var imgPath = Path.Combine(_config.ImagesDirectory, $"{fName}.jpg");
|
||||||
var img = System.Drawing.Image.FromStream(timeframe.Stream);
|
var img = System.Drawing.Image.FromStream(timeframe.Stream);
|
||||||
img.Save(imgPath, ImageFormat.Jpeg);
|
img.Save(imgPath, ImageFormat.Jpeg);
|
||||||
await YoloLabel.WriteToFile(labels, Path.Combine(_config.LabelsDirectory, $"{fName}.txt"), token);
|
await YoloLabel.WriteToFile(detections, Path.Combine(_config.LabelsDirectory, $"{fName}.txt"), token);
|
||||||
|
|
||||||
Editor.Background = new ImageBrush { ImageSource = await imgPath.OpenImage() };
|
Editor.Background = new ImageBrush { ImageSource = await imgPath.OpenImage() };
|
||||||
Editor.RemoveAllAnns();
|
Editor.RemoveAllAnns();
|
||||||
foreach (var (label, probability) in detections)
|
AddAnnotationsToCanvas(time, detections, true);
|
||||||
AddAnnotationToCanvas(time, new CanvasLabel(label, Editor.RenderSize, Editor.RenderSize, probability));
|
await AddAnnotations(timeframe.Time, detections.Cast<YoloLabel>().ToList(), token);
|
||||||
await AddAnnotations(timeframe.Time, labels);
|
|
||||||
|
|
||||||
var log = string.Join(Environment.NewLine, detections.Select(det =>
|
var log = string.Join(Environment.NewLine, detections.Select(det =>
|
||||||
$"{_config.AnnotationClassesDict[det.Label.ClassNumber].Name}: " +
|
$"{_config.AnnotationClassesDict[det.ClassNumber].Name}: " +
|
||||||
$"xy=({det.Label.CenterX:F2},{det.Label.CenterY:F2}), " +
|
$"xy=({det.CenterX:F2},{det.CenterY:F2}), " +
|
||||||
$"size=({det.Label.Width:F2}, {det.Label.Height:F2}), " +
|
$"size=({det.Width:F2}, {det.Height:F2}), " +
|
||||||
$"prob: {det.Probability:F1}%"));
|
$"prob: {det.Probability:F1}%"));
|
||||||
|
|
||||||
Dispatcher.Invoke(() => _autoDetectDialog.Log(log));
|
Dispatcher.Invoke(() => _autoDetectDialog.Log(log));
|
||||||
|
|
||||||
var thumbnailDto = await _galleryManager.CreateThumbnail(imgPath, token);
|
var thumbnailDto = await _galleryManager.CreateThumbnail(imgPath, token);
|
||||||
if (thumbnailDto != null)
|
if (thumbnailDto != null)
|
||||||
_datasetExplorer.AddThumbnail(thumbnailDto, labels.Select(x => x.ClassNumber));
|
_datasetExplorer.AddThumbnail(thumbnailDto, detections.Select(x => x.ClassNumber));
|
||||||
}
|
}
|
||||||
catch (Exception e)
|
catch (Exception e)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -2,10 +2,12 @@
|
|||||||
using System.Windows;
|
using System.Windows;
|
||||||
using System.Windows.Controls;
|
using System.Windows.Controls;
|
||||||
using System.Windows.Input;
|
using System.Windows.Input;
|
||||||
|
using System.Windows.Media;
|
||||||
using Azaion.Annotator.DTO;
|
using Azaion.Annotator.DTO;
|
||||||
using LibVLCSharp.Shared;
|
using LibVLCSharp.Shared;
|
||||||
using MediatR;
|
using MediatR;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
using MediaPlayer = LibVLCSharp.Shared.MediaPlayer;
|
||||||
|
|
||||||
namespace Azaion.Annotator;
|
namespace Azaion.Annotator;
|
||||||
|
|
||||||
@@ -148,6 +150,11 @@ public class MainWindowEventHandler :
|
|||||||
_mediaPlayer.Pause();
|
_mediaPlayer.Pause();
|
||||||
if (!_mediaPlayer.IsPlaying)
|
if (!_mediaPlayer.IsPlaying)
|
||||||
_mainWindow.BlinkHelp(HelpTexts.HelpTextsDict[HelpTextEnum.AnnotationHelp]);
|
_mainWindow.BlinkHelp(HelpTexts.HelpTextsDict[HelpTextEnum.AnnotationHelp]);
|
||||||
|
if (_formState.BackgroundShown)
|
||||||
|
{
|
||||||
|
_mainWindow.Editor.Background = new SolidColorBrush(Color.FromArgb(1, 0, 0, 0));
|
||||||
|
_formState.BackgroundShown = false;
|
||||||
|
}
|
||||||
break;
|
break;
|
||||||
case PlaybackControlEnum.Stop:
|
case PlaybackControlEnum.Stop:
|
||||||
_mediaPlayer.Stop();
|
_mediaPlayer.Stop();
|
||||||
@@ -240,16 +247,13 @@ public class MainWindowEventHandler :
|
|||||||
return;
|
return;
|
||||||
|
|
||||||
var time = TimeSpan.FromMilliseconds(_mediaPlayer.Time);
|
var time = TimeSpan.FromMilliseconds(_mediaPlayer.Time);
|
||||||
var fName = _formState.GetTimeName(time);
|
var fName = _formState.GetTimeName(time);
|
||||||
|
|
||||||
var currentAnns = _mainWindow.Editor.CurrentAnns
|
var currentAnns = _mainWindow.Editor.CurrentAnns
|
||||||
.Select(x => new YoloLabel(x.Info, _mainWindow.Editor.RenderSize, _formState.CurrentVideoSize))
|
.Select(x => new YoloLabel(x.Info, _mainWindow.Editor.RenderSize, _formState.BackgroundShown ? _mainWindow.Editor.RenderSize : _formState.CurrentVideoSize))
|
||||||
.ToList();
|
.ToList();
|
||||||
|
|
||||||
await YoloLabel.WriteToFile(currentAnns, Path.Combine(_config.LabelsDirectory, $"{fName}.txt"));
|
await YoloLabel.WriteToFile(currentAnns, Path.Combine(_config.LabelsDirectory, $"{fName}.txt"));
|
||||||
|
|
||||||
var resultHeight = (uint)Math.Round(RESULT_WIDTH / _formState.CurrentVideoSize.Width * _formState.CurrentVideoSize.Height);
|
|
||||||
|
|
||||||
await _mainWindow.AddAnnotations(time, currentAnns);
|
await _mainWindow.AddAnnotations(time, currentAnns);
|
||||||
|
|
||||||
_formState.CurrentMedia.HasAnnotations = _mainWindow.Annotations.Count != 0;
|
_formState.CurrentMedia.HasAnnotations = _mainWindow.Annotations.Count != 0;
|
||||||
@@ -261,7 +265,18 @@ public class MainWindowEventHandler :
|
|||||||
_mainWindow.Editor.RemoveAllAnns();
|
_mainWindow.Editor.RemoveAllAnns();
|
||||||
if (isVideo)
|
if (isVideo)
|
||||||
{
|
{
|
||||||
_mediaPlayer.TakeSnapshot(0, destinationPath, RESULT_WIDTH, resultHeight);
|
if (_formState.BackgroundShown)
|
||||||
|
{
|
||||||
|
//no need to save image, it's already there, just remove background
|
||||||
|
_mainWindow.Editor.Background = new SolidColorBrush(Color.FromArgb(1, 0, 0, 0));
|
||||||
|
_formState.BackgroundShown = false;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
var resultHeight = (uint)Math.Round(RESULT_WIDTH / _formState.CurrentVideoSize.Width * _formState.CurrentVideoSize.Height);
|
||||||
|
_mediaPlayer.TakeSnapshot(0, destinationPath, RESULT_WIDTH, resultHeight);
|
||||||
|
}
|
||||||
|
|
||||||
_mediaPlayer.Play();
|
_mediaPlayer.Play();
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
|
|||||||
@@ -1,23 +1,24 @@
|
|||||||
using System.Drawing.Imaging;
|
using System.IO;
|
||||||
using System.IO;
|
|
||||||
using Azaion.Annotator.DTO;
|
using Azaion.Annotator.DTO;
|
||||||
|
using Azaion.Annotator.Extensions;
|
||||||
using Compunet.YoloV8;
|
using Compunet.YoloV8;
|
||||||
|
using Compunet.YoloV8.Data;
|
||||||
using SixLabors.ImageSharp;
|
using SixLabors.ImageSharp;
|
||||||
using SixLabors.ImageSharp.Formats.Jpeg;
|
|
||||||
using SixLabors.ImageSharp.PixelFormats;
|
using SixLabors.ImageSharp.PixelFormats;
|
||||||
|
using Detection = Azaion.Annotator.DTO.Detection;
|
||||||
|
|
||||||
namespace Azaion.Annotator;
|
namespace Azaion.Annotator;
|
||||||
|
|
||||||
public interface IAIDetector
|
public interface IAIDetector
|
||||||
{
|
{
|
||||||
List<(YoloLabel Label, float Probability)> Detect(Stream stream);
|
List<Detection> Detect(Stream stream);
|
||||||
}
|
}
|
||||||
|
|
||||||
public class YOLODetector(Config config) : IAIDetector, IDisposable
|
public class YOLODetector(Config config) : IAIDetector, IDisposable
|
||||||
{
|
{
|
||||||
private readonly YoloPredictor _predictor = new(config.AIRecognitionConfig.AIModelPath);
|
private readonly YoloPredictor _predictor = new(config.AIRecognitionConfig.AIModelPath);
|
||||||
|
|
||||||
public List<(YoloLabel Label, float Probability)> Detect(Stream stream)
|
public List<Detection> Detect(Stream stream)
|
||||||
{
|
{
|
||||||
stream.Seek(0, SeekOrigin.Begin);
|
stream.Seek(0, SeekOrigin.Begin);
|
||||||
var image = Image.Load<Rgb24>(stream);
|
var image = Image.Load<Rgb24>(stream);
|
||||||
@@ -25,11 +26,50 @@ public class YOLODetector(Config config) : IAIDetector, IDisposable
|
|||||||
|
|
||||||
var imageSize = new System.Windows.Size(image.Width, image.Height);
|
var imageSize = new System.Windows.Size(image.Width, image.Height);
|
||||||
|
|
||||||
return result.Select(d =>
|
var detections = result.Select(d =>
|
||||||
{
|
{
|
||||||
var label = new YoloLabel(new CanvasLabel(d.Name.Id, d.Bounds.X, d.Bounds.Y, d.Bounds.Width, d.Bounds.Height), imageSize, imageSize);
|
var label = new YoloLabel(new CanvasLabel(d.Name.Id, d.Bounds.X, d.Bounds.Y, d.Bounds.Width, d.Bounds.Height), imageSize, imageSize);
|
||||||
return (label, d.Confidence * 100);
|
return new Detection(label, (double?)d.Confidence * 100);
|
||||||
}).ToList();
|
}).ToList();
|
||||||
|
|
||||||
|
return FilterOverlapping(detections);
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<Detection> FilterOverlapping(List<Detection> detections)
|
||||||
|
{
|
||||||
|
var k = config.AIRecognitionConfig.TrackingIntersectionThreshold;
|
||||||
|
var filteredDetections = new List<Detection>();
|
||||||
|
for (var i = 0; i < detections.Count; i++)
|
||||||
|
{
|
||||||
|
var detectionSelected = false;
|
||||||
|
for (var j = i + 1; j < detections.Count; j++)
|
||||||
|
{
|
||||||
|
var intersect = detections[i].ToRectangle();
|
||||||
|
intersect.Intersect(detections[j].ToRectangle());
|
||||||
|
|
||||||
|
var maxArea = Math.Max(detections[i].ToRectangle().Area(), detections[j].ToRectangle().Area());
|
||||||
|
if (intersect.Area() > k * maxArea)
|
||||||
|
{
|
||||||
|
if (detections[i].Probability > detections[j].Probability)
|
||||||
|
{
|
||||||
|
filteredDetections.Add(detections[i]);
|
||||||
|
detections.RemoveAt(j);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
filteredDetections.Add(detections[j]);
|
||||||
|
detections.RemoveAt(i);
|
||||||
|
}
|
||||||
|
detectionSelected = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!detectionSelected)
|
||||||
|
filteredDetections.Add(detections[i]);
|
||||||
|
}
|
||||||
|
|
||||||
|
return filteredDetections;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void Dispose() => _predictor.Dispose();
|
public void Dispose() => _predictor.Dispose();
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
{
|
{
|
||||||
"VideosDirectory": "E:\\Azaion3\\VideosTest",
|
"VideosDirectory": "E:\\Azaion1\\Videos",
|
||||||
"LabelsDirectory": "E:\\labels",
|
"LabelsDirectory": "E:\\labels",
|
||||||
"ImagesDirectory": "E:\\images",
|
"ImagesDirectory": "E:\\images",
|
||||||
"ThumbnailsDirectory": "E:\\thumbnails",
|
"ThumbnailsDirectory": "E:\\thumbnails",
|
||||||
@@ -40,6 +40,7 @@
|
|||||||
"AIModelPath": "azaion.onnx",
|
"AIModelPath": "azaion.onnx",
|
||||||
"FrameRecognitionSeconds": 2,
|
"FrameRecognitionSeconds": 2,
|
||||||
"TrackingDistanceConfidence": 0.15,
|
"TrackingDistanceConfidence": 0.15,
|
||||||
"TrackingProbabilityIncrease": 15
|
"TrackingProbabilityIncrease": 15,
|
||||||
|
"TrackingIntersectionThreshold": 0.8
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Reference in New Issue
Block a user