huge queue refactoring:

3 queues -> 1 queue
send delete validate updates

Azaion.Common/Services/AnnotationService.cs

@@ -30,13 +30,16 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
     private readonly QueueConfig _queueConfig;
     private Consumer _consumer = null!;
     private readonly UIConfig _uiConfig;
+    private readonly DirectoriesConfig _dirConfig;
     private static readonly Guid SaveTaskId = Guid.NewGuid();
+    private static readonly Guid SaveQueueOffsetTaskId = Guid.NewGuid();
 
     public AnnotationService(
         IDbFactory dbFactory,
         FailsafeAnnotationsProducer producer,
         IOptions<QueueConfig> queueConfig,
         IOptions<UIConfig> uiConfig,
+        IOptions<DirectoriesConfig> directoriesConfig,
         IGalleryService galleryService,
         IMediator mediator,
         IAzaionApi api)
@@ -48,11 +51,12 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
         _api = api;
         _queueConfig = queueConfig.Value;
         _uiConfig = uiConfig.Value;
+        _dirConfig = directoriesConfig.Value;
 
-        Task.Run(async () => await Init()).Wait();
+        Task.Run(async () => await InitQueueConsumer()).Wait();
     }
 
-    private async Task Init(CancellationToken cancellationToken = default)
+    private async Task InitQueueConsumer(CancellationToken cancellationToken = default)
     {
         if (!_api.CurrentUser.Role.IsValidator())
             return;
@@ -72,29 +76,40 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
             OffsetSpec = new OffsetTypeOffset(offsets.AnnotationsOffset + 1),
             MessageHandler = async (_, _, context, message) =>
             {
-                var msg = MessagePackSerializer.Deserialize<AnnotationCreatedMessage>(message.Data.Contents);
+                var email = (string)message.ApplicationProperties[nameof(User.Email)]!;
+                if (email == _api.CurrentUser.Email) //Don't process messages by yourself
+                    return;
+                var annotationStatus = (AnnotationStatus)message.ApplicationProperties[nameof(AnnotationStatus)];
+                if (annotationStatus.In(AnnotationStatus.Created, AnnotationStatus.ValidatedEdited))
+                {
+                    var msg = MessagePackSerializer.Deserialize<AnnotationMessage>(message.Data.Contents);
+                    await SaveAnnotationInner(
+                        msg.CreatedDate,
+                        msg.OriginalMediaName,
+                        msg.Time,
+                        JsonConvert.DeserializeObject<List<Detection>>(msg.Detections) ?? [],
+                        msg.Source,
+                        new MemoryStream(msg.Image),
+                        msg.Role,
+                        msg.Email,
+                        fromQueue: true,
+                        token: cancellationToken);
+                }
+                else
+                {
+                    var msg = MessagePackSerializer.Deserialize<AnnotationBulkMessage>(message.Data.Contents);
+                    if (annotationStatus == AnnotationStatus.Validated)
+                        await ValidateAnnotations(msg.AnnotationNames.ToList(), true, cancellationToken);
+                    if (annotationStatus == AnnotationStatus.Deleted)
+                        await _mediator.Publish(new AnnotationsDeletedEvent(msg.AnnotationNames.ToList(), fromQueue:true), cancellationToken);
+                }
 
                 offsets.AnnotationsOffset = context.Offset;
                 ThrottleExt.Throttle(() =>
                 {
                     _api.UpdateOffsets(offsets);
                     return Task.CompletedTask;
-                }, SaveTaskId, TimeSpan.FromSeconds(10), scheduleCallAfterCooldown: true);
-
-                if (msg.CreatedEmail == _api.CurrentUser.Email) //Don't process messages by yourself
-                    return;
-
-                await SaveAnnotationInner(
-                    msg.CreatedDate,
-                    msg.OriginalMediaName,
-                    msg.Time,
-                    JsonConvert.DeserializeObject<List<Detection>>(msg.Detections) ?? [],
-                    msg.Source,
-                    new MemoryStream(msg.Image),
-                    msg.CreatedRole,
-                    msg.CreatedEmail,
-                    fromQueue: true,
-                    token: cancellationToken);
+                }, SaveQueueOffsetTaskId, TimeSpan.FromSeconds(10), scheduleCallAfterCooldown: true);
             }
         });
     }
@@ -120,7 +135,7 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
         bool fromQueue = false,
         CancellationToken token = default)
     {
-        AnnotationStatus status;
+        var status = AnnotationStatus.Created;
         var fName = originalMediaName.ToTimeName(time);
         var annotation = await _dbFactory.Run(async db =>
         {
@@ -128,25 +143,41 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
                 .LoadWith(x => x.Detections)
                 .FirstOrDefaultAsync(x => x.Name == fName, token: token);
 
-            status = userRole.IsValidator() && source == SourceEnum.Manual
-                ? AnnotationStatus.Validated
-                : AnnotationStatus.Created;
+            if (userRole.IsValidator() && source == SourceEnum.Manual)
+                status = AnnotationStatus.Validated;
 
             if (fromQueue && ann is { AnnotationStatus: AnnotationStatus.Validated })
                 return ann;
 
             await db.Detections.DeleteAsync(x => x.AnnotationName == fName, token: token);
 
-            if (ann != null)
+            if (ann != null) //Annotation is already exists
             {
-                await db.Annotations
+                var annotationUpdatable = db.Annotations
                     .Where(x => x.Name == fName)
                     .Set(x => x.Source, source)
+                    .Set(x => x.CreatedRole, userRole);
+
+                if (status == AnnotationStatus.Validated)
+                {
+                    if (status == AnnotationStatus.Validated)
+                        status = AnnotationStatus.ValidatedEdited; //For further processing mark Annotations *edited* by Validator, not just simply Validated by button.
+
+                    annotationUpdatable = annotationUpdatable
+                        .Set(x => x.ValidateDate, createdDate)
+                        .Set(x => x.ValidateEmail, createdEmail);
+                }
+                else
+                {
+                    annotationUpdatable = annotationUpdatable
+                        .Set(x => x.CreatedDate, createdDate)
+                        .Set(x => x.CreatedEmail, createdEmail);
+                }
+
+                await annotationUpdatable
                     .Set(x => x.AnnotationStatus, status)
-                    .Set(x => x.CreatedDate, createdDate)
-                    .Set(x => x.CreatedEmail, createdEmail)
-                    .Set(x => x.CreatedRole, userRole)
                     .UpdateAsync(token: token);
+
                 ann.Detections = detections;
             }
             else
@@ -184,10 +215,11 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
         if (_uiConfig.GenerateAnnotatedImage)
             await _galleryService.CreateAnnotatedImage(annotation, token);
 
-        if (!fromQueue && !_uiConfig.SilentDetection) //Send to queue only if we're not getting from queue already
-            await _producer.SendToInnerQueue(annotation, token);
-
         await _mediator.Publish(new AnnotationCreatedEvent(annotation), token);
+
+        if (!fromQueue && !_uiConfig.SilentDetection) //Send to queue only if we're not getting from queue already
+            await _producer.SendToInnerQueue([annotation.Name], status, token);
+
         ThrottleExt.Throttle(async () =>
         {
             _dbFactory.SaveToDisk();
@@ -196,12 +228,12 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
         return annotation;
     }
 
-    public async Task ValidateAnnotations(List<Annotation> annotations, CancellationToken token = default)
+    public async Task ValidateAnnotations(List<string> annotationNames, bool fromQueue = false, CancellationToken token = default)
     {
         if (!_api.CurrentUser.Role.IsValidator())
             return;
 
-        var annNames = annotations.Select(x => x.Name).ToHashSet();
+        var annNames = annotationNames.ToHashSet();
         await _dbFactory.Run(async db =>
         {
             await db.Annotations
@@ -211,6 +243,9 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
                 .Set(x => x.ValidateEmail, _api.CurrentUser.Email)
                 .UpdateAsync(token: token);
         });
+        if (!fromQueue)
+            await _producer.SendToInnerQueue(annotationNames, AnnotationStatus.Validated, token);
+
         ThrottleExt.Throttle(async () =>
         {
             _dbFactory.SaveToDisk();
@@ -218,15 +253,25 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
         }, SaveTaskId, TimeSpan.FromSeconds(5), true);
     }
 
-    public async Task Handle(AnnotationsDeletedEvent notification, CancellationToken cancellationToken)
+    public async Task Handle(AnnotationsDeletedEvent notification, CancellationToken ct)
     {
-        await _dbFactory.DeleteAnnotations(notification.Annotations, cancellationToken);
-        foreach (var annotation in notification.Annotations)
+        await _dbFactory.DeleteAnnotations(notification.AnnotationNames, ct);
+        foreach (var name in notification.AnnotationNames)
         {
-            File.Delete(annotation.ImagePath);
-            File.Delete(annotation.LabelPath);
-            File.Delete(annotation.ThumbPath);
+            File.Delete(Path.Combine(_dirConfig.ImagesDirectory, $"{name}{Constants.JPG_EXT}"));
+            File.Delete(Path.Combine(_dirConfig.LabelsDirectory, $"{name}{Constants.TXT_EXT}"));
+            File.Delete(Path.Combine(_dirConfig.ThumbnailsDirectory, $"{name}{Constants.THUMBNAIL_PREFIX}{Constants.JPG_EXT}"));
+            File.Delete(Path.Combine(_dirConfig.ResultsDirectory, $"{name}{Constants.RESULT_PREFIX}{Constants.JPG_EXT}"));
         }
+
+        if (!notification.FromQueue)
+            await _producer.SendToInnerQueue(notification.AnnotationNames, AnnotationStatus.Deleted, ct);
+
+        ThrottleExt.Throttle(async () =>
+        {
+            _dbFactory.SaveToDisk();
+            await Task.CompletedTask;
+        }, SaveTaskId, TimeSpan.FromSeconds(5), true);
     }
 }
 
@@ -234,6 +279,5 @@ public interface IAnnotationService
 {
     Task<Annotation> SaveAnnotation(AnnotationImage a, CancellationToken ct = default);
     Task<Annotation> SaveAnnotation(string originalMediaName, TimeSpan time, List<Detection> detections, Stream? stream = null, CancellationToken token = default);
-    Task ValidateAnnotations(List<Annotation> annotations, CancellationToken token = default);
-
+    Task ValidateAnnotations(List<string> annotationNames, bool fromQueue = false, CancellationToken token = default);
 }