mirror of
https://github.com/azaion/annotations.git
synced 2026-04-22 09:06:30 +00:00
add ramdisk, load AI model to ramdisk and start recognition from it
rewrite zmq to DEALER and ROUTER add GET_USER command to get CurrentUser from Python all auth is on the python side inference run and validate annotations on python
This commit is contained in:
Alex Bezdieniezhnykh
+148
-154
@@ -6,9 +6,7 @@ using System.Windows.Controls;
|
||||
using System.Windows.Controls.Primitives;
|
||||
using System.Windows.Input;
|
||||
using System.Windows.Media;
|
||||
using System.Windows.Media.Imaging;
|
||||
using Azaion.Annotator.DTO;
|
||||
using Azaion.Annotator.Extensions;
|
||||
using Azaion.Common.Database;
|
||||
using Azaion.Common.DTO;
|
||||
using Azaion.Common.DTO.Config;
|
||||
@@ -39,10 +37,9 @@ public partial class Annotator
|
||||
private readonly IConfigUpdater _configUpdater;
|
||||
private readonly HelpWindow _helpWindow;
|
||||
private readonly ILogger<Annotator> _logger;
|
||||
private readonly VLCFrameExtractor _vlcFrameExtractor;
|
||||
private readonly IAIDetector _aiDetector;
|
||||
private readonly AnnotationService _annotationService;
|
||||
private readonly IDbFactory _dbFactory;
|
||||
private readonly IInferenceService _inferenceService;
|
||||
private readonly CancellationTokenSource _ctSource = new();
|
||||
|
||||
private ObservableCollection<DetectionClass> AnnotationClasses { get; set; } = new();
|
||||
@@ -67,10 +64,9 @@ public partial class Annotator
|
||||
FormState formState,
|
||||
HelpWindow helpWindow,
|
||||
ILogger<Annotator> logger,
|
||||
VLCFrameExtractor vlcFrameExtractor,
|
||||
IAIDetector aiDetector,
|
||||
AnnotationService annotationService,
|
||||
IDbFactory dbFactory)
|
||||
IDbFactory dbFactory,
|
||||
IInferenceService inferenceService)
|
||||
{
|
||||
InitializeComponent();
|
||||
_appConfig = appConfig.Value;
|
||||
@@ -81,10 +77,9 @@ public partial class Annotator
|
||||
_formState = formState;
|
||||
_helpWindow = helpWindow;
|
||||
_logger = logger;
|
||||
_vlcFrameExtractor = vlcFrameExtractor;
|
||||
_aiDetector = aiDetector;
|
||||
_annotationService = annotationService;
|
||||
_dbFactory = dbFactory;
|
||||
_inferenceService = inferenceService;
|
||||
|
||||
Loaded += OnLoaded;
|
||||
Closed += OnFormClosed;
|
||||
@@ -304,11 +299,16 @@ public partial class Annotator
|
||||
|
||||
var annotations = await _dbFactory.Run(async db =>
|
||||
await db.Annotations.LoadWith(x => x.Detections)
|
||||
.Where(x => x.Name.Contains(_formState.VideoName))
|
||||
.Where(x => x.OriginalMediaName == _formState.VideoName)
|
||||
.ToListAsync(token: _ctSource.Token));
|
||||
|
||||
TimedAnnotations.Clear();
|
||||
_formState.AnnotationResults.Clear();
|
||||
foreach (var ann in annotations)
|
||||
AddAnnotation(ann);
|
||||
{
|
||||
TimedAnnotations.Add(ann.Time.Subtract(_thresholdBefore), ann.Time.Add(_thresholdAfter), ann);
|
||||
_formState.AnnotationResults.Add(new AnnotationResult(_appConfig.AnnotationConfig.DetectionClassesDict, ann));
|
||||
}
|
||||
}
|
||||
|
||||
//Add manually
|
||||
@@ -435,8 +435,6 @@ public partial class Annotator
|
||||
|
||||
_appConfig.DirectoriesConfig.VideosDirectory = dlg.FileName;
|
||||
TbFolder.Text = dlg.FileName;
|
||||
await ReloadFiles();
|
||||
await SaveUserSettings();
|
||||
}
|
||||
|
||||
private void TbFilter_OnTextChanged(object sender, TextChangedEventArgs e)
|
||||
@@ -487,11 +485,8 @@ public partial class Annotator
|
||||
if (LvFiles.SelectedIndex == -1)
|
||||
LvFiles.SelectedIndex = 0;
|
||||
|
||||
await _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.Play));
|
||||
_mediaPlayer.Stop();
|
||||
|
||||
var manualCancellationSource = new CancellationTokenSource();
|
||||
var token = manualCancellationSource.Token;
|
||||
var mct = new CancellationTokenSource();
|
||||
var token = mct.Token;
|
||||
|
||||
_autoDetectDialog = new AutodetectDialog
|
||||
{
|
||||
@@ -500,7 +495,7 @@ public partial class Annotator
|
||||
};
|
||||
_autoDetectDialog.Closing += (_, _) =>
|
||||
{
|
||||
manualCancellationSource.Cancel();
|
||||
mct.Cancel();
|
||||
_mediaPlayer.SeekTo(TimeSpan.Zero);
|
||||
Editor.RemoveAllAnns();
|
||||
};
|
||||
@@ -515,16 +510,17 @@ public partial class Annotator
|
||||
var mediaInfo = Dispatcher.Invoke(() => (MediaFileInfo)LvFiles.SelectedItem);
|
||||
while (mediaInfo != null)
|
||||
{
|
||||
_formState.CurrentMedia = mediaInfo;
|
||||
await Dispatcher.Invoke(async () => await ReloadAnnotations());
|
||||
|
||||
if (mediaInfo.MediaType == MediaTypes.Image)
|
||||
await Dispatcher.Invoke(async () =>
|
||||
{
|
||||
await DetectImage(mediaInfo, manualCancellationSource, token);
|
||||
await Task.Delay(70, token);
|
||||
}
|
||||
else
|
||||
await DetectVideo(mediaInfo, manualCancellationSource, token);
|
||||
await _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.Play), token);
|
||||
await ReloadAnnotations();
|
||||
});
|
||||
|
||||
await _inferenceService.RunInference(mediaInfo.Path, async (annotationImage, ct) =>
|
||||
{
|
||||
annotationImage.OriginalMediaName = mediaInfo.FName;
|
||||
await ProcessDetection(annotationImage, ct);
|
||||
}, token);
|
||||
|
||||
mediaInfo = Dispatcher.Invoke(() =>
|
||||
{
|
||||
@@ -533,6 +529,7 @@ public partial class Annotator
|
||||
LvFiles.SelectedIndex += 1;
|
||||
return (MediaFileInfo)LvFiles.SelectedItem;
|
||||
});
|
||||
LvFiles.Items.Refresh();
|
||||
}
|
||||
Dispatcher.Invoke(() =>
|
||||
{
|
||||
@@ -546,143 +543,140 @@ public partial class Annotator
|
||||
Dispatcher.Invoke(() => Editor.ResetBackground());
|
||||
}
|
||||
|
||||
private async Task DetectImage(MediaFileInfo mediaInfo, CancellationTokenSource manualCancellationSource, CancellationToken token)
|
||||
// private async Task DetectImage(MediaFileInfo mediaInfo, CancellationTokenSource manualCancellationSource, CancellationToken token)
|
||||
// {
|
||||
// try
|
||||
// {
|
||||
// var fName = Path.GetFileNameWithoutExtension(mediaInfo.Path);
|
||||
// var stream = new FileStream(mediaInfo.Path, FileMode.Open);
|
||||
// var detections = await _aiDetector.Detect(fName, stream, token);
|
||||
// await ProcessDetection((TimeSpan.FromMilliseconds(0), stream), Path.GetExtension(mediaInfo.Path), detections, token);
|
||||
// if (detections.Count != 0)
|
||||
// mediaInfo.HasAnnotations = true;
|
||||
// }
|
||||
// catch (Exception e)
|
||||
// {
|
||||
// _logger.LogError(e, e.Message);
|
||||
// await manualCancellationSource.CancelAsync();
|
||||
// }
|
||||
// }
|
||||
|
||||
// private async Task DetectVideo(MediaFileInfo mediaInfo, CancellationTokenSource manualCancellationSource, CancellationToken token)
|
||||
// {
|
||||
// var prevSeekTime = 0.0;
|
||||
// await foreach (var timeframe in _vlcFrameExtractor.ExtractFrames(mediaInfo.Path, token))
|
||||
// {
|
||||
// Console.WriteLine($"Detect time: {timeframe.Time}");
|
||||
// try
|
||||
// {
|
||||
// var fName = _formState.GetTimeName(timeframe.Time);
|
||||
// var detections = await _aiDetector.Detect(fName, timeframe.Stream, token);
|
||||
//
|
||||
// var isValid = IsValidDetection(timeframe.Time, detections);
|
||||
// Console.WriteLine($"Detection time: {timeframe.Time}");
|
||||
//
|
||||
// var log = string.Join(Environment.NewLine, detections.Select(det =>
|
||||
// $"{_appConfig.AnnotationConfig.DetectionClassesDict[det.ClassNumber].Name}: " +
|
||||
// $"xy=({det.CenterX:F2},{det.CenterY:F2}), " +
|
||||
// $"size=({det.Width:F2}, {det.Height:F2}), " +
|
||||
// $"prob: {det.Probability:F1}%"));
|
||||
//
|
||||
// log = $"Detection time: {timeframe.Time}, Valid: {isValid}. {Environment.NewLine} {log}";
|
||||
// Dispatcher.Invoke(() => _autoDetectDialog.Log(log));
|
||||
//
|
||||
// if (timeframe.Time.TotalMilliseconds > prevSeekTime + 250)
|
||||
// {
|
||||
// Dispatcher.Invoke(() => SeekTo(timeframe.Time));
|
||||
// prevSeekTime = timeframe.Time.TotalMilliseconds;
|
||||
// if (!isValid) //Show frame anyway
|
||||
// {
|
||||
// Dispatcher.Invoke(() =>
|
||||
// {
|
||||
// Editor.RemoveAllAnns();
|
||||
// Editor.Background = new ImageBrush
|
||||
// {
|
||||
// ImageSource = timeframe.Stream.OpenImage()
|
||||
// };
|
||||
// });
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// if (!isValid)
|
||||
// continue;
|
||||
//
|
||||
// mediaInfo.HasAnnotations = true;
|
||||
// await ProcessDetection(timeframe, ".jpg", detections, token);
|
||||
// await timeframe.Stream.DisposeAsync();
|
||||
// }
|
||||
// catch (Exception ex)
|
||||
// {
|
||||
// _logger.LogError(ex, ex.Message);
|
||||
// await manualCancellationSource.CancelAsync();
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
// private bool IsValidDetection(TimeSpan time, List<Detection> detections)
|
||||
// {
|
||||
// // No AI detection, forbid
|
||||
// if (detections.Count == 0)
|
||||
// return false;
|
||||
//
|
||||
// // Very first detection, allow
|
||||
// if (!_previousDetection.HasValue)
|
||||
// return true;
|
||||
//
|
||||
// var prev = _previousDetection.Value;
|
||||
//
|
||||
// // Time between detections is >= than Frame Recognition Seconds, allow
|
||||
// if (time >= prev.Time.Add(TimeSpan.FromSeconds(_appConfig.AIRecognitionConfig.FrameRecognitionSeconds)))
|
||||
// return true;
|
||||
//
|
||||
// // Detection is earlier than previous + FrameRecognitionSeconds.
|
||||
// // Look to the detections more in detail
|
||||
//
|
||||
// // More detected objects, allow
|
||||
// if (detections.Count > prev.Detections.Count)
|
||||
// return true;
|
||||
//
|
||||
// foreach (var det in detections)
|
||||
// {
|
||||
// var point = new Point(det.CenterX, det.CenterY);
|
||||
// var closestObject = prev.Detections
|
||||
// .Select(p => new
|
||||
// {
|
||||
// Point = p,
|
||||
// Distance = point.SqrDistance(new Point(p.CenterX, p.CenterY))
|
||||
// })
|
||||
// .OrderBy(x => x.Distance)
|
||||
// .First();
|
||||
//
|
||||
// // Closest object is farther than Tracking distance confidence, hence it's a different object, allow
|
||||
// if (closestObject.Distance > _appConfig.AIRecognitionConfig.TrackingDistanceConfidence)
|
||||
// return true;
|
||||
//
|
||||
// // Since closest object within distance confidence, then it is tracking of the same object. Then if recognition probability for the object > increase from previous
|
||||
// if (det.Probability >= closestObject.Point.Probability + _appConfig.AIRecognitionConfig.TrackingProbabilityIncrease)
|
||||
// return true;
|
||||
// }
|
||||
//
|
||||
// return false;
|
||||
// }
|
||||
|
||||
private async Task ProcessDetection(AnnotationImage annotationImage, CancellationToken token = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
var fName = Path.GetFileNameWithoutExtension(mediaInfo.Path);
|
||||
var stream = new FileStream(mediaInfo.Path, FileMode.Open);
|
||||
var detections = await _aiDetector.Detect(fName, stream, token);
|
||||
await ProcessDetection((TimeSpan.FromMilliseconds(0), stream), Path.GetExtension(mediaInfo.Path), detections, token);
|
||||
if (detections.Count != 0)
|
||||
mediaInfo.HasAnnotations = true;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, e.Message);
|
||||
await manualCancellationSource.CancelAsync();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task DetectVideo(MediaFileInfo mediaInfo, CancellationTokenSource manualCancellationSource, CancellationToken token)
|
||||
{
|
||||
var prevSeekTime = 0.0;
|
||||
await foreach (var timeframe in _vlcFrameExtractor.ExtractFrames(mediaInfo.Path, token))
|
||||
{
|
||||
Console.WriteLine($"Detect time: {timeframe.Time}");
|
||||
try
|
||||
{
|
||||
var fName = _formState.GetTimeName(timeframe.Time);
|
||||
var detections = await _aiDetector.Detect(fName, timeframe.Stream, token);
|
||||
|
||||
var isValid = IsValidDetection(timeframe.Time, detections);
|
||||
Console.WriteLine($"Detection time: {timeframe.Time}");
|
||||
|
||||
var log = string.Join(Environment.NewLine, detections.Select(det =>
|
||||
$"{_appConfig.AnnotationConfig.DetectionClassesDict[det.ClassNumber].Name}: " +
|
||||
$"xy=({det.CenterX:F2},{det.CenterY:F2}), " +
|
||||
$"size=({det.Width:F2}, {det.Height:F2}), " +
|
||||
$"prob: {det.Probability:F1}%"));
|
||||
|
||||
log = $"Detection time: {timeframe.Time}, Valid: {isValid}. {Environment.NewLine} {log}";
|
||||
Dispatcher.Invoke(() => _autoDetectDialog.Log(log));
|
||||
|
||||
if (timeframe.Time.TotalMilliseconds > prevSeekTime + 250)
|
||||
{
|
||||
Dispatcher.Invoke(() => SeekTo(timeframe.Time));
|
||||
prevSeekTime = timeframe.Time.TotalMilliseconds;
|
||||
if (!isValid) //Show frame anyway
|
||||
{
|
||||
Dispatcher.Invoke(() =>
|
||||
{
|
||||
Editor.RemoveAllAnns();
|
||||
Editor.Background = new ImageBrush
|
||||
{
|
||||
ImageSource = timeframe.Stream.OpenImage()
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (!isValid)
|
||||
continue;
|
||||
|
||||
mediaInfo.HasAnnotations = true;
|
||||
await ProcessDetection(timeframe, ".jpg", detections, token);
|
||||
await timeframe.Stream.DisposeAsync();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, ex.Message);
|
||||
await manualCancellationSource.CancelAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool IsValidDetection(TimeSpan time, List<Detection> detections)
|
||||
{
|
||||
// No AI detection, forbid
|
||||
if (detections.Count == 0)
|
||||
return false;
|
||||
|
||||
// Very first detection, allow
|
||||
if (!_previousDetection.HasValue)
|
||||
return true;
|
||||
|
||||
var prev = _previousDetection.Value;
|
||||
|
||||
// Time between detections is >= than Frame Recognition Seconds, allow
|
||||
if (time >= prev.Time.Add(TimeSpan.FromSeconds(_appConfig.AIRecognitionConfig.FrameRecognitionSeconds)))
|
||||
return true;
|
||||
|
||||
// Detection is earlier than previous + FrameRecognitionSeconds.
|
||||
// Look to the detections more in detail
|
||||
|
||||
// More detected objects, allow
|
||||
if (detections.Count > prev.Detections.Count)
|
||||
return true;
|
||||
|
||||
foreach (var det in detections)
|
||||
{
|
||||
var point = new Point(det.CenterX, det.CenterY);
|
||||
var closestObject = prev.Detections
|
||||
.Select(p => new
|
||||
{
|
||||
Point = p,
|
||||
Distance = point.SqrDistance(new Point(p.CenterX, p.CenterY))
|
||||
})
|
||||
.OrderBy(x => x.Distance)
|
||||
.First();
|
||||
|
||||
// Closest object is farther than Tracking distance confidence, hence it's a different object, allow
|
||||
if (closestObject.Distance > _appConfig.AIRecognitionConfig.TrackingDistanceConfidence)
|
||||
return true;
|
||||
|
||||
// Since closest object within distance confidence, then it is tracking of the same object. Then if recognition probability for the object > increase from previous
|
||||
if (det.Probability >= closestObject.Point.Probability + _appConfig.AIRecognitionConfig.TrackingProbabilityIncrease)
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private async Task ProcessDetection((TimeSpan Time, Stream Stream) timeframe, string imageExtension, List<Detection> detections, CancellationToken token = default)
|
||||
{
|
||||
_previousDetection = (timeframe.Time, detections);
|
||||
await Dispatcher.Invoke(async () =>
|
||||
{
|
||||
try
|
||||
{
|
||||
var fName = _formState.GetTimeName(timeframe.Time);
|
||||
|
||||
var annotation = await _annotationService.SaveAnnotation(fName, imageExtension, detections, SourceEnum.AI, timeframe.Stream, token);
|
||||
var annotation = await _annotationService.SaveAnnotation(annotationImage, token);
|
||||
|
||||
Editor.Background = new ImageBrush { ImageSource = await annotation.ImagePath.OpenImage() };
|
||||
Editor.RemoveAllAnns();
|
||||
ShowAnnotations(annotation, true);
|
||||
AddAnnotation(annotation);
|
||||
|
||||
var log = string.Join(Environment.NewLine, detections.Select(det =>
|
||||
var log = string.Join(Environment.NewLine, annotation.Detections.Select(det =>
|
||||
$"{_appConfig.AnnotationConfig.DetectionClassesDict[det.ClassNumber].Name}: " +
|
||||
$"xy=({det.CenterX:F2},{det.CenterY:F2}), " +
|
||||
$"size=({det.Width:F2}, {det.Height:F2}), " +
|
||||
|
||||
Reference in New Issue
Block a user