add offset

fixes
add visual validation border and validate functionality

Azaion.Common/Constants.cs

@@ -85,11 +85,8 @@ public class Constants
 
     #region Queue
 
-    public const string MQ_DIRECT_TYPE = "direct";
     public const string MQ_ANNOTATIONS_QUEUE = "azaion-annotations";
     public const string MQ_ANNOTATIONS_CONFIRM_QUEUE = "azaion-annotations-confirm";
-    public const string ANNOTATION_PRODUCER = "AnnotationsProducer";
-    public const string ANNOTATION_CONFIRM_PRODUCER = "AnnotationsConfirmProducer";
 
     #endregion
 

Azaion.Common/DTO/AnnotationCreatedEvent.cs

@@ -1,4 +1,5 @@
-using MediatR;
+using Azaion.Common.Database;
+using MediatR;
 
 namespace Azaion.Common.DTO;
 

Azaion.Common/DTO/AnnotationImageView.cs

@@ -2,6 +2,7 @@
 using System.IO;
 using System.Runtime.CompilerServices;
 using System.Windows.Media.Imaging;
+using Azaion.Common.Database;
 using Azaion.Common.Extensions;
 
 namespace Azaion.Common.DTO;
@@ -27,6 +28,7 @@ public class AnnotationImageView(Annotation annotation) : INotifyPropertyChanged
     }
 
     public string ImageName => Path.GetFileName(Annotation.ImagePath);
+    public bool IsSeed => Annotation.AnnotationStatus == AnnotationStatus.Created;
 
     public void Delete()
     {

Azaion.Common/DTO/Annotator2ControlEvent.cs

@@ -0,0 +1,13 @@
+using MediatR;
+
+namespace Azaion.Common.DTO;
+
+public class AnnotatorControlEvent(PlaybackControlEnum playbackControlEnum) : INotification
+{
+    public PlaybackControlEnum PlaybackControl { get; set; } = playbackControlEnum;
+}
+
+public class DatasetExplorerControlEvent(PlaybackControlEnum playbackControlEnum) : INotification
+{
+    public PlaybackControlEnum PlaybackControl { get; set; } = playbackControlEnum;
+}

Azaion.Common/DTO/Label.cs

@@ -186,7 +186,7 @@ public class YoloLabel : Label
 
 public class Detection : YoloLabel
 {
-    public string AnnotationName { get; set; }
+    public string AnnotationName { get; set; } = null!;
     public double? Probability { get; set; }
 
     //For db

Azaion.Common/DTO/PlaybackControlEnum.cs

@@ -15,5 +15,6 @@ public enum PlaybackControlEnum
     TurnOnVolume = 10,
     Previous = 11,
     Next = 12,
-    Close = 13
+    Close = 13,
+    ValidateAnnotations = 15
 }

Azaion.Common/DTO/Queue/AnnotationCreatedMessage.cs

@@ -1,4 +1,5 @@
-using Azaion.CommonSecurity.DTO;
+using Azaion.Common.Database;
+using Azaion.CommonSecurity.DTO;
 
 namespace Azaion.Common.DTO.Queue;
 using MessagePack;

Azaion.Common/Database/Annotation.cs

@@ -1,11 +1,10 @@
 using System.IO;
-using System.Windows.Media.Imaging;
+using Azaion.Common.DTO;
 using Azaion.Common.DTO.Config;
 using Azaion.Common.DTO.Queue;
-using Azaion.Common.Extensions;
 using Azaion.CommonSecurity.DTO;
 
-namespace Azaion.Common.DTO;
+namespace Azaion.Common.Database;
 
 public class Annotation
 {
@@ -46,9 +45,4 @@ public enum AnnotationStatus
     None = 0,
     Created = 10,
     Validated = 20
-}
-
-public class AnnotationName
-{
-    public string Name { get; set; } = null!;
 }

Azaion.Common/Database/AnnotationName.cs

@@ -0,0 +1,6 @@
+namespace Azaion.Common.Database;
+
+public class AnnotationName
+{
+    public string Name { get; set; } = null!;
+}

Azaion.Common/Database/AnnotationsDb.cs

@@ -9,4 +9,5 @@ public class AnnotationsDb(DataOptions dataOptions) : DataConnection(dataOptions
     public ITable<Annotation> Annotations => this.GetTable<Annotation>();
     public ITable<AnnotationName> AnnotationsQueue => this.GetTable<AnnotationName>();
     public ITable<Detection> Detections => this.GetTable<Detection>();
+    public ITable<QueueOffset> QueueOffsets => this.GetTable<QueueOffset>();
 }

Azaion.Common/Database/DbFactory.cs

@@ -4,6 +4,7 @@ using System.IO;
 using Azaion.Common.DTO;
 using Azaion.Common.DTO.Config;
 using LinqToDB;
+using LinqToDB.Data;
 using LinqToDB.DataProvider.SQLite;
 using LinqToDB.Mapping;
 using Microsoft.Extensions.Logging;
@@ -63,6 +64,20 @@ public class DbFactory : IDbFactory
         db.CreateTable<Annotation>();
         db.CreateTable<AnnotationName>();
         db.CreateTable<Detection>();
+        db.CreateTable<QueueOffset>();
+        db.QueueOffsets.BulkCopy(new List<QueueOffset>
+        {
+            new()
+            {
+                Offset = 0,
+                QueueName = Constants.MQ_ANNOTATIONS_QUEUE
+            },
+            new()
+            {
+                Offset = 0,
+                QueueName = Constants.MQ_ANNOTATIONS_CONFIRM_QUEUE
+            }
+        });
     }
 
     public async Task<T> Run<T>(Func<AnnotationsDb, Task<T>> func)

Azaion.Common/Database/QueueOffset.cs

@@ -0,0 +1,7 @@
+namespace Azaion.Common.Database;
+
+public class QueueOffset
+{
+    public string QueueName { get; set; } = null!;
+    public ulong Offset { get; set; }
+}

Azaion.Common/Extensions/IEnumerableExtensions.cs

@@ -1,7 +0,0 @@
-namespace Azaion.Common.Extensions;
-
-public static class EnumerableExtensions
-{
-    public static bool In<T>(this T obj, params T[] objects) =>
-        objects.Contains(obj);
-}

Azaion.Common/Extensions/ThrottleExtensions.cs

@@ -2,18 +2,34 @@
 
 public static class ThrottleExt
 {
-    private static bool _throttleOn;
-    public static async Task Throttle(this Func<Task> func, TimeSpan? throttleTime = null, CancellationToken cancellationToken = default)
+    private static bool _throttleRunFirstOn;
+    public static async Task ThrottleRunFirst(this Func<Task> func, TimeSpan? throttleTime = null, CancellationToken cancellationToken = default)
     {
-        if (_throttleOn)
+        if (_throttleRunFirstOn)
             return;
 
-        _throttleOn = true;
+        _throttleRunFirstOn = true;
         await func();
         _ = Task.Run(async () =>
         {
             await Task.Delay(throttleTime ?? TimeSpan.FromMilliseconds(500), cancellationToken);
-            _throttleOn = false;
+            _throttleRunFirstOn = false;
         }, cancellationToken);
     }
+
+    private static bool _throttleRunAfter;
+    public static async Task ThrottleRunAfter(this Func<Task> func, TimeSpan? throttleTime = null, CancellationToken cancellationToken = default)
+    {
+        if (_throttleRunAfter)
+            return;
+
+        _throttleRunAfter = true;
+        _ = Task.Run(async () =>
+        {
+            await Task.Delay(throttleTime ?? TimeSpan.FromMilliseconds(500), cancellationToken);
+            await func();
+            _throttleRunAfter = false;
+        }, cancellationToken);
+    }
+
 }

Azaion.Common/Services/AnnotationService.cs

@@ -5,6 +5,7 @@ using Azaion.Common.Database;
 using Azaion.Common.DTO;
 using Azaion.Common.DTO.Config;
 using Azaion.Common.DTO.Queue;
+using Azaion.Common.Extensions;
 using Azaion.CommonSecurity.DTO;
 using Azaion.CommonSecurity.Services;
 using LinqToDB;
@@ -25,6 +26,7 @@ public class AnnotationService
     private readonly FailsafeAnnotationsProducer _producer;
     private readonly IGalleryService _galleryService;
     private readonly IMediator _mediator;
+    private readonly IHardwareService _hardwareService;
     private readonly QueueConfig _queueConfig;
     private Consumer _consumer = null!;
 
@@ -33,19 +35,21 @@ public class AnnotationService
         FailsafeAnnotationsProducer producer,
         IOptions<QueueConfig> queueConfig,
         IGalleryService galleryService,
-        IMediator mediator)
+        IMediator mediator,
+        IHardwareService hardwareService)
     {
         _apiClient = apiClient;
         _dbFactory = dbFactory;
         _producer = producer;
         _galleryService = galleryService;
         _mediator = mediator;
+        _hardwareService = hardwareService;
         _queueConfig = queueConfig.Value;
 
         Task.Run(async () => await Init()).Wait();
     }
 
-    private async Task Init()
+    private async Task Init(CancellationToken cancellationToken = default)
     {
         var consumerSystem = await StreamSystem.Create(new StreamSystemConfig
         {
@@ -53,11 +57,28 @@ public class AnnotationService
             UserName = _queueConfig.ConsumerUsername,
             Password = _queueConfig.ConsumerPassword
         });
+        var offset = (await _dbFactory.Run(db => db.QueueOffsets.FirstOrDefaultAsync(
+            x => x.QueueName == Constants.MQ_ANNOTATIONS_QUEUE, token: cancellationToken))
+            )?.Offset ?? 0;
         _consumer = await Consumer.Create(new ConsumerConfig(consumerSystem, Constants.MQ_ANNOTATIONS_QUEUE)
         {
-            OffsetSpec = new OffsetTypeFirst(),
-            MessageHandler = async (stream, _, _, message) =>
-                await Consume(MessagePackSerializer.Deserialize<AnnotationCreatedMessage>(message.Data.Contents)),
+            Reference = _hardwareService.GetHardware().Hash,
+            OffsetSpec = new OffsetTypeOffset(offset + 1),
+            MessageHandler = async (stream, consumer, context, message) =>
+            {
+                await Consume(MessagePackSerializer.Deserialize<AnnotationCreatedMessage>(message.Data.Contents), cancellationToken);
+                await _dbFactory.Run(async db => await db.QueueOffsets
+                    .Where(x => x.QueueName == Constants.MQ_ANNOTATIONS_QUEUE)
+                    .Set(x => x.Offset, context.Offset)
+                    .UpdateAsync(token: cancellationToken));
+
+                await ThrottleExt.ThrottleRunAfter(() =>
+                {
+                    _dbFactory.SaveToDisk();
+                    return Task.CompletedTask;
+                }, TimeSpan.FromSeconds(3), cancellationToken);
+
+            }
         });
     }
 
@@ -65,6 +86,10 @@ public class AnnotationService
     public async Task SaveAnnotation(string fName, string imageExtension, List<Detection> detections, SourceEnum source, Stream? stream = null, CancellationToken token = default) =>
         await SaveAnnotationInner(DateTime.UtcNow, fName, imageExtension, detections, source, stream, _apiClient.User.Role, _apiClient.User.Email, token);
 
+    //Manual
+    public async Task ValidateAnnotation(Annotation annotation, CancellationToken token = default) =>
+        await SaveAnnotationInner(DateTime.UtcNow, annotation.Name, annotation.ImageExtension, annotation.Detections.ToList(), SourceEnum.Manual, null, _apiClient.User.Role, _apiClient.User.Email, token);
+
     //Queue (only from operators)
     public async Task Consume(AnnotationCreatedMessage message, CancellationToken cancellationToken = default)
     {
@@ -84,24 +109,20 @@ public class AnnotationService
     }
 
     private async Task SaveAnnotationInner(DateTime createdDate, string fName, string imageExtension, List<Detection> detections, SourceEnum source, Stream? stream,
-        RoleEnum createdRole,
+        RoleEnum userRole,
         string createdEmail,
         CancellationToken token = default)
     {
         //Flow for roles:
-        // Operator:
-        //   sourceEnum: (manual, ai) <AnnotationCreatedMessage>
-        // Validator:
-        //   sourceEnum: (manual)     if was in received.json then <AnnotationValidatedMessage> else <AnnotationCreatedMessage>
-        //   sourceEnum: (queue, AI)  if queue CreatedMessage with the same user - do nothing Add to received.json
+        // Operator or (AI from any role) -> Created
+        // Validator, Admin & Manual -> Validated
 
-        var classes = detections.Select(x => x.ClassNumber).Distinct().ToList() ?? [];
         AnnotationStatus status;
 
         var annotation = await _dbFactory.Run(async db =>
         {
             var ann = await db.Annotations.FirstOrDefaultAsync(x => x.Name == fName, token: token);
-            status = ann?.AnnotationStatus == AnnotationStatus.Created && createdRole == RoleEnum.Validator
+            status = userRole.IsValidator() && source == SourceEnum.Manual
                 ? AnnotationStatus.Validated
                 : AnnotationStatus.Created;
 
@@ -110,7 +131,6 @@ public class AnnotationService
             if (ann != null)
                 await db.Annotations
                     .Where(x => x.Name == fName)
-                    .Set(x => x.Classes, classes)
                     .Set(x => x.Source, source)
                     .Set(x => x.AnnotationStatus, status)
                     .UpdateAsync(token: token);
@@ -122,7 +142,7 @@ public class AnnotationService
                     Name = fName,
                     ImageExtension = imageExtension,
                     CreatedEmail = createdEmail,
-                    CreatedRole = createdRole,
+                    CreatedRole = userRole,
                     AnnotationStatus = status,
                     Source = source,
                     Detections = detections
@@ -142,5 +162,10 @@ public class AnnotationService
         await _producer.SendToQueue(annotation, token);
 
         await _mediator.Publish(new AnnotationCreatedEvent(annotation), token);
+        await ThrottleExt.ThrottleRunAfter(() =>
+        {
+            _dbFactory.SaveToDisk();
+            return Task.CompletedTask;
+        }, TimeSpan.FromSeconds(5), token);
     }
 }

Azaion.Common/Services/GalleryService.cs

@@ -150,11 +150,6 @@ public class GalleryService(
             };
             await dbFactory.Run(async db =>
             {
-                var xx = missedAnnotations.GroupBy(x => x.Name)
-                    .Where(gr => gr.Count() > 1)
-                    .ToList();
-                foreach (var gr in xx)
-                    Console.WriteLine(gr.Key);
                 await db.BulkCopyAsync(copyOptions, missedAnnotations);
                 await db.BulkCopyAsync(copyOptions, missedAnnotations.SelectMany(x => x.Detections));
             });