azaion/annotations
1
0
Fork 0
mirror of https://github.com/azaion/annotations.git synced 2026-04-22 13:06:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

add ramdisk, load AI model to ramdisk and start recognition from it

Browse Source 
rewrite zmq to DEALER and ROUTER
add GET_USER command to get CurrentUser from Python
all auth is on the python side
inference run and validate annotations on python
This commit is contained in:
Alex Bezdieniezhnykh
2025-01-29 17:45:26 +02:00
parent 82b3b526a7
commit 62623b7123
55 changed files with 945 additions and 895 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Azaion.Common/DTO/Config/AIRecognitionConfig.cs
+10 -5
View File
@@ -1,10 +1,15 @@
using MessagePack;
namespace Azaion.Common.DTO.Config;
[MessagePackObject]
public class AIRecognitionConfig
{
public double FrameRecognitionSeconds { get; set; }
public double TrackingDistanceConfidence { get; set; }
public double TrackingProbabilityIncrease { get; set; }
public double TrackingIntersectionThreshold { get; set; }
public int FramePeriodRecognition { get; set; }
[Key("FrameRecognitionSeconds")] public double FrameRecognitionSeconds { get; set; }
[Key("TrackingDistanceConfidence")] public double TrackingDistanceConfidence { get; set; }
[Key("TrackingProbabilityIncrease")] public double TrackingProbabilityIncrease { get; set; }
[Key("TrackingIntersectionThreshold")] public double TrackingIntersectionThreshold { get; set; }
[Key("FramePeriodRecognition")] public int FramePeriodRecognition { get; set; }
[Key("Data")] public byte[] Data { get; set; }
}
Azaion.Common/DTO/Config/AppConfig.cs
-9
View File
@@ -8,8 +8,6 @@ namespace Azaion.Common.DTO.Config;
public class AppConfig
{
public ApiConfig ApiConfig { get; set; } = null!;
public QueueConfig QueueConfig { get; set; } = null!;
public DirectoriesConfig DirectoriesConfig { get; set; } = null!;
@@ -39,13 +37,6 @@ public class ConfigUpdater : IConfigUpdater
var appConfig = new AppConfig
{
ApiConfig = new ApiConfig
{
Url = SecurityConstants.DEFAULT_API_URL,
RetryCount = SecurityConstants.DEFAULT_API_RETRY_COUNT,
TimeoutSeconds = SecurityConstants.DEFAULT_API_TIMEOUT_SECONDS
},
AnnotationConfig = new AnnotationConfig
{
AnnotationClasses = Constants.DefaultAnnotationClasses,
Azaion.Common/DTO/FormState.cs
-2
View File
@@ -16,6 +16,4 @@ public class FormState
public int CurrentVolume { get; set; } = 100;
public ObservableCollection<AnnotationResult> AnnotationResults { get; set; } = [];
public WindowEnum ActiveWindow { get; set; }
public string GetTimeName(TimeSpan? ts) => $"{VideoName}_{ts:hmmssf}";
}
Azaion.Common/DTO/Label.cs
+13 -11
View File
@@ -1,18 +1,18 @@
using System.Drawing;
using System.Globalization;
using System.IO;
using MessagePack;
using Newtonsoft.Json;
using Size = System.Windows.Size;
namespace Azaion.Common.DTO;
[MessagePackObject]
public abstract class Label
{
[JsonProperty(PropertyName = "cl")] public int ClassNumber { get; set; }
[JsonProperty(PropertyName = "cl")][Key("c")] public int ClassNumber { get; set; }
protected Label()
{
}
protected Label() { }
protected Label(int classNumber)
{
@@ -79,15 +79,16 @@ public class CanvasLabel : Label
}
}
[MessagePackObject]
public class YoloLabel : Label
{
[JsonProperty(PropertyName = "x")] public double CenterX { get; set; }
[JsonProperty(PropertyName = "x")][Key("x")] public double CenterX { get; set; }
[JsonProperty(PropertyName = "y")] public double CenterY { get; set; }
[JsonProperty(PropertyName = "y")][Key("y")] public double CenterY { get; set; }
[JsonProperty(PropertyName = "w")] public double Width { get; set; }
[JsonProperty(PropertyName = "w")][Key("w")] public double Width { get; set; }
[JsonProperty(PropertyName = "h")] public double Height { get; set; }
[JsonProperty(PropertyName = "h")][Key("h")] public double Height { get; set; }
public YoloLabel()
{
@@ -184,12 +185,13 @@ public class YoloLabel : Label
public override string ToString() => $"{ClassNumber} {CenterX:F5} {CenterY:F5} {Width:F5} {Height:F5}".Replace(',', '.');
}
[MessagePackObject]
public class Detection : YoloLabel
{
public string AnnotationName { get; set; } = null!;
public double? Probability { get; set; }
[IgnoreMember]public string AnnotationName { get; set; } = null!;
[Key("p")] public double? Probability { get; set; }
//For db
//For db & serialization
public Detection(){}
public Detection(string annotationName, YoloLabel label, double? probability = null)
Azaion.Common/DTO/MediaFileInfo.cs
+4 -2
View File
@@ -1,4 +1,6 @@
namespace Azaion.Common.DTO;
using Azaion.Common.Extensions;
namespace Azaion.Common.DTO;
public class MediaFileInfo
{
@@ -9,5 +11,5 @@ public class MediaFileInfo
public bool HasAnnotations { get; set; }
public MediaTypes MediaType { get; set; }
public string FName => System.IO.Path.GetFileNameWithoutExtension(Name).Replace(" ", "");
public string FName => Name.ToFName();
}
Azaion.Common/DTO/Queue/AnnotationCreatedMessage.cs
+11 -9
View File
@@ -7,15 +7,17 @@ using MessagePack;
[MessagePackObject]
public class AnnotationCreatedMessage
{
[Key(0)] public DateTime CreatedDate { get; set; }
[Key(1)] public string Name { get; set; } = null!;
[Key(2)] public string ImageExtension { get; set; } = null!;
[Key(3)] public string Detections { get; set; } = null!;
[Key(4)] public byte[] Image { get; set; } = null!;
[Key(5)] public RoleEnum CreatedRole { get; set; }
[Key(6)] public string CreatedEmail { get; set; } = null!;
[Key(7)] public SourceEnum Source { get; set; }
[Key(8)] public AnnotationStatus Status { get; set; }
[Key(0)] public DateTime CreatedDate { get; set; }
[Key(1)] public string Name { get; set; } = null!;
[Key(2)] public string OriginalMediaName { get; set; } = null!;
[Key(3)] public TimeSpan Time { get; set; }
[Key(4)] public string ImageExtension { get; set; } = null!;
[Key(5)] public string Detections { get; set; } = null!;
[Key(6)] public byte[] Image { get; set; } = null!;
[Key(7)] public RoleEnum CreatedRole { get; set; }
[Key(8)] public string CreatedEmail { get; set; } = null!;
[Key(9)] public SourceEnum Source { get; set; }
[Key(10)] public AnnotationStatus Status { get; set; }
}
[MessagePackObject]
Azaion.Common/Database/Annotation.cs
+24 -39
View File
@@ -3,9 +3,11 @@ using Azaion.Common.DTO;
using Azaion.Common.DTO.Config;
using Azaion.Common.DTO.Queue;
using Azaion.CommonSecurity.DTO;
using MessagePack;
namespace Azaion.Common.Database;
[MessagePackObject]
public class Annotation
{
private static string _labelsDir = null!;
@@ -19,53 +21,36 @@ public class Annotation
_thumbDir = config.ThumbnailsDirectory;
}
public string Name { get; set; } = null!;
public string ImageExtension { get; set; } = null!;
public DateTime CreatedDate { get; set; }
public string CreatedEmail { get; set; } = null!;
public RoleEnum CreatedRole { get; set; }
public SourceEnum Source { get; set; }
public AnnotationStatus AnnotationStatus { get; set; }
[IgnoreMember]public string Name { get; set; } = null!;
[IgnoreMember]public string OriginalMediaName { get; set; } = null!;
[IgnoreMember]public TimeSpan Time { get; set; }
[IgnoreMember]public string ImageExtension { get; set; } = null!;
[IgnoreMember]public DateTime CreatedDate { get; set; }
[IgnoreMember]public string CreatedEmail { get; set; } = null!;
[IgnoreMember]public RoleEnum CreatedRole { get; set; }
[IgnoreMember]public SourceEnum Source { get; set; }
[IgnoreMember]public AnnotationStatus AnnotationStatus { get; set; }
public IEnumerable<Detection> Detections { get; set; } = null!;
[Key("d")] public IEnumerable<Detection> Detections { get; set; } = null!;
[Key("t")] public long Milliseconds { get; set; }
public double Lat { get; set; }
public double Lon { get; set; }
[Key("lat")]public double Lat { get; set; }
[Key("lon")]public double Lon { get; set; }
#region Calculated
public List<int> Classes => Detections.Select(x => x.ClassNumber).ToList();
public string ImagePath => Path.Combine(_imagesDir, $"{Name}{ImageExtension}");
public string LabelPath => Path.Combine(_labelsDir, $"{Name}.txt");
public string ThumbPath => Path.Combine(_thumbDir, $"{Name}{Constants.THUMBNAIL_PREFIX}.jpg");
public string OriginalMediaName => $"{Name[..^7]}";
[IgnoreMember]public List<int> Classes => Detections.Select(x => x.ClassNumber).ToList();
[IgnoreMember]public string ImagePath => Path.Combine(_imagesDir, $"{Name}{ImageExtension}");
[IgnoreMember]public string LabelPath => Path.Combine(_labelsDir, $"{Name}.txt");
[IgnoreMember]public string ThumbPath => Path.Combine(_thumbDir, $"{Name}{Constants.THUMBNAIL_PREFIX}.jpg");
private TimeSpan? _time;
public TimeSpan Time
{
get
{
if (_time.HasValue)
return _time.Value;
var timeStr = Name.Split("_").LastOrDefault();
//For some reason, TimeSpan.ParseExact doesn't work on every platform.
if (!string.IsNullOrEmpty(timeStr) &&
timeStr.Length == 6 &&
int.TryParse(timeStr[..1], out var hours) &&
int.TryParse(timeStr[1..3], out var minutes) &&
int.TryParse(timeStr[3..5], out var seconds) &&
int.TryParse(timeStr[5..], out var milliseconds))
return new TimeSpan(0, hours, minutes, seconds, milliseconds * 100);
_time = TimeSpan.FromSeconds(0);
return _time.Value;
}
}
#endregion Calculated
}
[MessagePackObject]
public class AnnotationImage : Annotation
{
[Key("i")] public byte[] Image { get; set; }
}
public enum AnnotationStatus
{
Azaion.Common/Database/DbFactory.cs
+9 -6
View File
@@ -117,16 +117,19 @@ public static class AnnotationsDbSchemaHolder
MappingSchema = new MappingSchema();
var builder = new FluentMappingBuilder(MappingSchema);
builder.Entity<Annotation>()
.HasTableName(Constants.ANNOTATIONS_TABLENAME)
var annotationBuilder = builder.Entity<Annotation>();
annotationBuilder.HasTableName(Constants.ANNOTATIONS_TABLENAME)
.HasPrimaryKey(x => x.Name)
.Ignore(x => x.Time)
.Association(a => a.Detections, (a, d) => a.Name == d.AnnotationName)
.Property(x => x.Time).HasDataType(DataType.Int64).HasConversion(ts => ts.Ticks, t => new TimeSpan(t));
annotationBuilder
.Ignore(x => x.Milliseconds)
.Ignore(x => x.Classes)
.Ignore(x => x.Classes)
.Ignore(x => x.ImagePath)
.Ignore(x => x.LabelPath)
.Ignore(x => x.ThumbPath)
.Ignore(x => x.OriginalMediaName)
.Association(a => a.Detections, (a, d) => a.Name == d.AnnotationName);
.Ignore(x => x.ThumbPath);
builder.Entity<Detection>()
.HasTableName(Constants.DETECTIONS_TABLENAME);
Azaion.Common/Extensions/ParallelExt.cs
+1 -1
View File
@@ -24,7 +24,7 @@ public class ParallelExt
return Task.CompletedTask;
}
};
var threadsCount = (int)(Environment.ProcessorCount * parallelOptions.CpuUtilPercent / 100.0);
var threadsCount = (int)Math.Round(Environment.ProcessorCount * parallelOptions.CpuUtilPercent / 100.0);
var processedCount = 0;
var chunkSize = Math.Max(1, (int)(source.Count / (decimal)threadsCount));
Azaion.Common/Extensions/StringExtensions.cs
+12
View File
@@ -0,0 +1,12 @@
using System.IO;
namespace Azaion.Common.Extensions;
public static class StringExtensions
{
public static string ToFName(this string path) =>
Path.GetFileNameWithoutExtension(path).Replace(" ", "");
public static string ToTimeName(this string fName, TimeSpan? ts) =>
$"{fName}_{ts:hmmssf}";
}
Azaion.Common/Services/AnnotationService.cs
+29 -21
View File
@@ -22,29 +22,31 @@ namespace Azaion.Common.Services;
public class AnnotationService : INotificationHandler<AnnotationsDeletedEvent>
{
private readonly AzaionApiClient _apiClient;
private readonly IDbFactory _dbFactory;
private readonly FailsafeAnnotationsProducer _producer;
private readonly IGalleryService _galleryService;
private readonly IMediator _mediator;
private readonly IHardwareService _hardwareService;
private readonly IAuthProvider _authProvider;
private readonly QueueConfig _queueConfig;
private Consumer _consumer = null!;
public AnnotationService(AzaionApiClient apiClient,
public AnnotationService(
IResourceLoader resourceLoader,
IDbFactory dbFactory,
FailsafeAnnotationsProducer producer,
IOptions<QueueConfig> queueConfig,
IGalleryService galleryService,
IMediator mediator,
IHardwareService hardwareService)
IHardwareService hardwareService,
IAuthProvider authProvider)
{
_apiClient = apiClient;
_dbFactory = dbFactory;
_producer = producer;
_galleryService = galleryService;
_mediator = mediator;
_hardwareService = hardwareService;
_authProvider = authProvider;
_queueConfig = queueConfig.Value;
Task.Run(async () => await Init()).Wait();
@@ -73,7 +75,8 @@ public class AnnotationService : INotificationHandler<AnnotationsDeletedEvent>
await SaveAnnotationInner(
msg.CreatedDate,
msg.Name,
msg.OriginalMediaName,
msg.Time,
msg.ImageExtension,
JsonConvert.DeserializeObject<List<Detection>>(msg.Detections) ?? [],
msg.Source,
@@ -98,36 +101,39 @@ public class AnnotationService : INotificationHandler<AnnotationsDeletedEvent>
});
}
//AI / Manual
public async Task<Annotation> SaveAnnotation(string fName, string imageExtension, List<Detection> detections, SourceEnum source, Stream? stream = null, CancellationToken token = default) =>
await SaveAnnotationInner(DateTime.UtcNow, fName, imageExtension, detections, source, stream, _apiClient.User.Role, _apiClient.User.Email, generateThumbnail: true, token);
//AI
public async Task<Annotation> SaveAnnotation(AnnotationImage a, CancellationToken cancellationToken = default)
{
a.Time = TimeSpan.FromMilliseconds(a.Milliseconds);
a.Name = a.OriginalMediaName.ToTimeName(a.Time);
return await SaveAnnotationInner(DateTime.Now, a.OriginalMediaName, a.Time, ".jpg", a.Detections.ToList(),
a.Source, new MemoryStream(a.Image), a.CreatedRole, a.CreatedEmail, generateThumbnail: true, cancellationToken);
}
//Manual
public async Task<Annotation> SaveAnnotation(string originalMediaName, TimeSpan time, string imageExtension, List<Detection> detections, SourceEnum source, Stream? stream = null, CancellationToken token = default) =>
await SaveAnnotationInner(DateTime.UtcNow, originalMediaName, time, imageExtension, detections, source, stream,
_authProvider.CurrentUser.Role, _authProvider.CurrentUser.Email, generateThumbnail: true, token);
//Manual Validate existing
public async Task ValidateAnnotation(Annotation annotation, CancellationToken token = default) =>
await SaveAnnotationInner(DateTime.UtcNow, annotation.Name, annotation.ImageExtension, annotation.Detections.ToList(), SourceEnum.Manual, null, _apiClient.User.Role, _apiClient.User.Email,
generateThumbnail: false, token);
await SaveAnnotationInner(DateTime.UtcNow, annotation.OriginalMediaName, annotation.Time, annotation.ImageExtension, annotation.Detections.ToList(), SourceEnum.Manual, null,
_authProvider.CurrentUser.Role, _authProvider.CurrentUser.Email, generateThumbnail: false, token);
// //Queue (only from operators)
// public async Task Consume(AnnotationCreatedMessage message, CancellationToken cancellationToken = default)
// {
//
// }
private async Task<Annotation> SaveAnnotationInner(DateTime createdDate, string fName, string imageExtension, List<Detection> detections, SourceEnum source, Stream? stream,
private async Task<Annotation> SaveAnnotationInner(DateTime createdDate, string originalMediaName, TimeSpan time, string imageExtension, List<Detection> detections, SourceEnum source, Stream? stream,
RoleEnum userRole,
string createdEmail,
bool generateThumbnail = false,
CancellationToken token = default)
{
//Flow for roles:
// Operator or (AI from any role) -> Created
// Validator, Admin & Manual -> Validated
AnnotationStatus status;
var fName = originalMediaName.ToTimeName(time);
var annotation = await _dbFactory.Run(async db =>
{
var ann = await db.Annotations.FirstOrDefaultAsync(x => x.Name == fName, token: token);
// Manual Save from Validators -> Validated
// otherwise Created
status = userRole.IsValidator() && source == SourceEnum.Manual
? AnnotationStatus.Validated
: AnnotationStatus.Created;
@@ -149,6 +155,8 @@ public class AnnotationService : INotificationHandler<AnnotationsDeletedEvent>
{
CreatedDate = createdDate,
Name = fName,
OriginalMediaName = originalMediaName,
Time = time,
ImageExtension = imageExtension,
CreatedEmail = createdEmail,
CreatedRole = userRole,
Azaion.Common/Services/FailsafeProducer.cs
+3 -2
View File
@@ -76,7 +76,7 @@ public class FailsafeAnnotationsProducer
await _annotationConfirmProducer.Send(validatedMessages, CompressionType.Gzip);
await _dbFactory.Run(async db =>
await db.AnnotationsQueue.DeleteAsync(aq => messagesChunk.Any(x => aq.Name == x.Name), token: cancellationToken));
await db.AnnotationsQueue.DeleteAsync(aq => messagesChunk.Any(x => aq.Name == x.OriginalMediaName), token: cancellationToken));
sent = true;
}
catch (Exception e)
@@ -106,7 +106,8 @@ public class FailsafeAnnotationsProducer
var annCreateMessage = new AnnotationCreatedMessage
{
Name = annotation.Name,
OriginalMediaName = annotation.OriginalMediaName,
Time = annotation.Time,
CreatedRole = annotation.CreatedRole,
CreatedEmail = annotation.CreatedEmail,
CreatedDate = annotation.CreatedDate,
Azaion.Common/Services/GalleryService.cs
+45 -13
View File
@@ -8,6 +8,7 @@ using Azaion.Common.Database;
using Azaion.Common.DTO;
using Azaion.Common.DTO.Config;
using Azaion.Common.DTO.Queue;
using Azaion.Common.Extensions;
using Azaion.CommonSecurity.DTO;
using LinqToDB;
using LinqToDB.Data;
@@ -75,7 +76,6 @@ public class GalleryService(
var missedAnnotations = new ConcurrentBag<Annotation>();
try
{
var prefixLen = Constants.THUMBNAIL_PREFIX.Length;
var thumbnails = ThumbnailsDirectory.GetFiles()
@@ -105,9 +105,37 @@ public class GalleryService(
return;
var detections = (await YoloLabel.ReadFromFile(labelName, cancellationToken)).Select(x => new Detection(fName, x)).ToList();
//get names and time
var fileName = Path.GetFileNameWithoutExtension(file.Name);
var strings = fileName.Split("_");
var timeStr = strings.LastOrDefault();
string originalMediaName;
TimeSpan time;
//For some reason, TimeSpan.ParseExact doesn't work on every platform.
if (!string.IsNullOrEmpty(timeStr) &&
timeStr.Length == 6 &&
int.TryParse(timeStr[..1], out var hours) &&
int.TryParse(timeStr[1..3], out var minutes) &&
int.TryParse(timeStr[3..5], out var seconds) &&
int.TryParse(timeStr[5..], out var milliseconds))
{
time = new TimeSpan(0, hours, minutes, seconds, milliseconds * 100);
originalMediaName = fileName[..^7];
}
else
{
originalMediaName = fileName;
time = TimeSpan.FromSeconds(0);
}
var annotation = new Annotation
{
Name = fName,
Time = time,
OriginalMediaName = originalMediaName,
Name = file.Name.ToFName(),
ImageExtension = Path.GetExtension(file.Name),
Detections = detections,
CreatedDate = File.GetCreationTimeUtc(file.FullName),
@@ -129,18 +157,22 @@ public class GalleryService(
logger.LogError(e, $"Failed to generate thumbnail for {file.Name}! Error: {e.Message}");
}
},
new ParallelOptions
new ParallelOptions
{
ProgressFn = async num =>
{
ProgressFn = async num =>
{
Console.WriteLine($"Processed {num} item by Thread {Environment.CurrentManagedThreadId}");
ProcessedThumbnailsPercentage = imagesCount == 0 ? 0 : Math.Min(100, num * 100 / (double)imagesCount);
ThumbnailsUpdate?.Invoke(ProcessedThumbnailsPercentage);
await Task.CompletedTask;
},
CpuUtilPercent = 100,
ProgressUpdateInterval = 200
});
Console.WriteLine($"Processed {num} item by Thread {Environment.CurrentManagedThreadId}");
ProcessedThumbnailsPercentage = imagesCount == 0 ? 0 : Math.Min(100, num * 100 / (double)imagesCount);
ThumbnailsUpdate?.Invoke(ProcessedThumbnailsPercentage);
await Task.CompletedTask;
},
CpuUtilPercent = 100,
ProgressUpdateInterval = 200
});
}
catch (Exception e)
{
logger.LogError(e, $"Failed to refresh thumbnails! Error: {e.Message}");
}
finally
{
Azaion.Common/Services/InferenceService.cs
+53
View File
@@ -0,0 +1,53 @@
using System.Text;
using Azaion.Common.Database;
using Azaion.Common.DTO.Config;
using Azaion.CommonSecurity;
using Azaion.CommonSecurity.DTO.Commands;
using Azaion.CommonSecurity.Services;
using MessagePack;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using NetMQ;
using NetMQ.Sockets;
namespace Azaion.Common.Services;
public interface IInferenceService
{
Task RunInference(string mediaPath, Func<AnnotationImage, CancellationToken, Task> processAnnotation, CancellationToken ct = default);
}
public class PythonInferenceService(ILogger<PythonInferenceService> logger, IOptions<AIRecognitionConfig> aiConfigOptions) : IInferenceService
{
public async Task RunInference(string mediaPath, Func<AnnotationImage, CancellationToken, Task> processAnnotation, CancellationToken ct = default)
{
using var dealer = new DealerSocket();
var clientId = Guid.NewGuid();
dealer.Options.Identity = Encoding.UTF8.GetBytes(clientId.ToString("N"));
dealer.Connect($"tcp://{SecurityConstants.ZMQ_HOST}:{SecurityConstants.ZMQ_PORT}");
var data = MessagePackSerializer.Serialize(aiConfigOptions.Value);
dealer.SendFrame(MessagePackSerializer.Serialize(new RemoteCommand(CommandType.Inference, mediaPath, data)));
while (true)
{
byte[] bytes = [];
try
{
var annotationStream = dealer.Get<AnnotationImage>(out bytes);
if (annotationStream == null)
{
if (bytes.Length == 4 && Encoding.UTF8.GetString(bytes) == "DONE")
break;
continue;
}
await processAnnotation(annotationStream, ct);
}
catch (Exception e)
{
logger.LogError(e, e.Message);
}
}
}
}