azaion/annotations
1
0
Fork 0
mirror of https://github.com/azaion/annotations.git synced 2026-04-22 08:36:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixed bugs with queue handling. At least most of them

Browse Source
This commit is contained in:
Alex Bezdieniezhnykh
2025-05-18 20:11:19 +03:00
parent cf563571c8
commit c5e81ebcc6
15 changed files with 135 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Azaion.Annotator/Annotator.xaml.cs
+3 -2
View File
@@ -417,7 +417,8 @@ public partial class Annotator
mediaFile.HasAnnotations = labelsDict.ContainsKey(mediaFile.FName);
AllMediaFiles = new ObservableCollection<MediaFileInfo>(allFiles);
MediaFilesDict = AllMediaFiles.ToDictionary(x => x.FName);
MediaFilesDict = AllMediaFiles.GroupBy(x => x.Name)
.ToDictionary(gr => gr.Key, gr => gr.First());
LvFiles.ItemsSource = AllMediaFiles;
DataContext = this;
}
@@ -463,7 +464,7 @@ public partial class Annotator
{
Title = "Open Video folder",
IsFolderPicker = true,
InitialDirectory = Path.GetDirectoryName(AppDomain.CurrentDomain.BaseDirectory)
InitialDirectory = Path.GetDirectoryName(_appConfig.DirectoriesConfig.VideosDirectory)
};
var dialogResult = dlg.ShowDialog();
Azaion.Annotator/AnnotatorEventHandler.cs
+14 -2
View File
@@ -1,6 +1,7 @@
using System.IO;
using System.Windows;
using System.Windows.Input;
using System.Windows.Threading;
using Azaion.Annotator.DTO;
using Azaion.Common;
using Azaion.Common.DTO;
@@ -276,7 +277,11 @@ public class AnnotatorEventHandler(
mainWindow.AddAnnotation(annotation);
}
public async Task Handle(AnnotationsDeletedEvent notification, CancellationToken cancellationToken)
public Task Handle(AnnotationsDeletedEvent notification, CancellationToken cancellationToken)
{
try
{
mainWindow.Dispatcher.Invoke(() =>
{
var namesSet = notification.AnnotationNames.ToHashSet();
@@ -300,7 +305,14 @@ public class AnnotatorEventHandler(
mainWindow.LvFiles.Items.Refresh();
}
}
await Task.CompletedTask;
});
}
catch (Exception e)
{
logger.LogError(e, e.Message);
throw;
}
return Task.CompletedTask;
}
public Task Handle(AnnotationAddedEvent e, CancellationToken cancellationToken)
Azaion.Common/Constants.cs
-5
View File
@@ -97,12 +97,7 @@ public class Constants
#endregion
#region Queue
public const string MQ_ANNOTATIONS_QUEUE = "azaion-annotations";
public const string MQ_ANNOTATIONS_CONFIRM_QUEUE = "azaion-annotations-confirm";
#endregion
#region Database
Azaion.Common/Database/Annotation.cs
+4 -3
View File
@@ -59,7 +59,8 @@ public enum AnnotationStatus
{
None = 0,
Created = 10,
Validated = 20,
ValidatedEdited = 25,
Deleted = 30
ValidatedEdited = 20,
Validated = 30,
Deleted = 40
}
Azaion.Common/Database/AnnotationQueueRecord.cs
+1
View File
@@ -2,6 +2,7 @@ namespace Azaion.Common.Database;
public class AnnotationQueueRecord
{
public Guid Id { get; set; }
public DateTime DateTime { get; set; }
public AnnotationStatus Operation { get; set; }
public List<string> AnnotationNames { get; set; } = null!;
Azaion.Common/Database/AnnotationsDb.cs
-1
View File
@@ -9,5 +9,4 @@ public class AnnotationsDb(DataOptions dataOptions) : DataConnection(dataOptions
public ITable<Annotation> Annotations => this.GetTable<Annotation>();
public ITable<AnnotationQueueRecord> AnnotationsQueueRecords => this.GetTable<AnnotationQueueRecord>();
public ITable<Detection> Detections => this.GetTable<Detection>();
public ITable<QueueOffset> QueueOffsets => this.GetTable<QueueOffset>();
}
Azaion.Common/Database/DbFactory.cs
+11 -18
View File
@@ -53,32 +53,24 @@ public class DbFactory : IDbFactory
.UseMappingSchema(AnnotationsDbSchemaHolder.MappingSchema);
if (!File.Exists(_annConfig.AnnotationsDbFile))
CreateDb();
SQLiteConnection.CreateFile(_annConfig.AnnotationsDbFile);
RecreateTables();
_fileConnection.Open();
_fileConnection.BackupDatabase(_memoryConnection, "main", "main", -1, null, -1);
}
private void CreateDb()
private void RecreateTables()
{
SQLiteConnection.CreateFile(_annConfig.AnnotationsDbFile);
using var db = new AnnotationsDb(_fileDataOptions);
var schema = db.DataProvider.GetSchemaProvider().GetSchema(db);
var existingTables = schema.Tables.Select(x => x.TableName).ToHashSet();
if (!existingTables.Contains(Constants.ANNOTATIONS_TABLENAME))
db.CreateTable<Annotation>();
db.CreateTable<AnnotationQueueRecord>();
if (!existingTables.Contains(Constants.DETECTIONS_TABLENAME))
db.CreateTable<Detection>();
db.CreateTable<QueueOffset>();
db.QueueOffsets.BulkCopy(new List<QueueOffset>
{
new()
{
Offset = 0,
QueueName = Constants.MQ_ANNOTATIONS_QUEUE
},
new()
{
Offset = 0,
QueueName = Constants.MQ_ANNOTATIONS_CONFIRM_QUEUE
}
});
if (!existingTables.Contains(Constants.ANNOTATIONS_QUEUE_TABLENAME))
db.CreateTable<AnnotationQueueRecord>();
}
public async Task<T> Run<T>(Func<AnnotationsDb, Task<T>> func)
@@ -138,6 +130,7 @@ public static class AnnotationsDbSchemaHolder
builder.Entity<AnnotationQueueRecord>()
.HasTableName(Constants.ANNOTATIONS_QUEUE_TABLENAME)
.HasPrimaryKey(x => x.Id)
.Property(x => x.AnnotationNames)
.HasDataType(DataType.NVarChar)
.HasConversion(list => JsonConvert.SerializeObject(list), str => JsonConvert.DeserializeObject<List<string>>(str) ?? new List<string>());
Azaion.Common/Database/QueueOffset.cs
-7
View File
@@ -1,7 +0,0 @@
namespace Azaion.Common.Database;
public class QueueOffset
{
public string QueueName { get; set; } = null!;
public ulong Offset { get; set; }
}
Azaion.Common/Services/AnnotationService.cs
+12 -18
View File
@@ -77,15 +77,17 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
_consumer = await Consumer.Create(new ConsumerConfig(consumerSystem, Constants.MQ_ANNOTATIONS_QUEUE)
{
Reference = _api.CurrentUser.Email,
OffsetSpec = new OffsetTypeOffset(offsets.AnnotationsOffset + 1),
OffsetSpec = new OffsetTypeOffset(offsets.AnnotationsOffset),
MessageHandler = async (_, _, context, message) =>
{
try
{
var email = (string)message.ApplicationProperties[nameof(User.Email)]!;
if (email == _api.CurrentUser.Email) //Don't process messages by yourself
if (!Enum.TryParse<AnnotationStatus>((string)message.ApplicationProperties[nameof(AnnotationStatus)], out var annotationStatus))
return;
var annotationStatus = (AnnotationStatus)message.ApplicationProperties[nameof(AnnotationStatus)];
if (email != _api.CurrentUser.Email) //Don't process messages by yourself
{
if (annotationStatus.In(AnnotationStatus.Created, AnnotationStatus.ValidatedEdited))
{
var msg = MessagePackSerializer.Deserialize<AnnotationMessage>(message.Data.Contents);
@@ -109,8 +111,9 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
if (annotationStatus == AnnotationStatus.Deleted)
await _mediator.Publish(new AnnotationsDeletedEvent(msg.AnnotationNames.ToList(), fromQueue:true), cancellationToken);
}
}
offsets.AnnotationsOffset = context.Offset;
offsets.AnnotationsOffset = context.Offset + 1; //to consume on the next launch from the next message
ThrottleExt.Throttle(() =>
{
_api.UpdateOffsets(offsets);
@@ -154,12 +157,6 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
.LoadWith(x => x.Detections)
.FirstOrDefaultAsync(x => x.Name == fName, token: token);
if (userRole.IsValidator() && source == SourceEnum.Manual)
status = AnnotationStatus.Validated;
if (fromQueue && ann is { AnnotationStatus: AnnotationStatus.Validated })
return ann;
await db.Detections.DeleteAsync(x => x.AnnotationName == fName, token: token);
if (ann != null) //Annotation is already exists
@@ -169,10 +166,9 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
.Set(x => x.Source, source)
.Set(x => x.CreatedRole, userRole);
if (status == AnnotationStatus.Validated)
if (userRole.IsValidator() && source == SourceEnum.Manual)
{
if (status == AnnotationStatus.Validated)
status = AnnotationStatus.ValidatedEdited; //For further processing mark Annotations *edited* by Validator, not just simply Validated by button.
status = AnnotationStatus.ValidatedEdited;
annotationUpdatable = annotationUpdatable
.Set(x => x.ValidateDate, createdDate)
@@ -212,9 +208,6 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
return ann;
});
if (fromQueue && annotation is { AnnotationStatus: AnnotationStatus.Validated })
return annotation;
if (stream != null)
{
var img = System.Drawing.Image.FromStream(stream);
@@ -228,7 +221,7 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
await _mediator.Publish(new AnnotationCreatedEvent(annotation), token);
if (!fromQueue && !_uiConfig.SilentDetection) //Send to queue only if we're not getting from queue already
if (!fromQueue) //Send to queue only if we're not getting from queue already
await _producer.SendToInnerQueue([annotation.Name], status, token);
ThrottleExt.Throttle(async () =>
@@ -275,7 +268,8 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
File.Delete(Path.Combine(_dirConfig.ResultsDirectory, $"{name}{Constants.RESULT_PREFIX}{Constants.JPG_EXT}"));
}
if (!notification.FromQueue)
//Only validators can send Delete to the queue
if (!notification.FromQueue && _api.CurrentUser.Role.IsValidator())
await _producer.SendToInnerQueue(notification.AnnotationNames, AnnotationStatus.Deleted, ct);
ThrottleExt.Throttle(async () =>
Azaion.Common/Services/FailsafeProducer.cs
+16 -4
View File
@@ -23,14 +23,22 @@ public class FailsafeAnnotationsProducer
private readonly IDbFactory _dbFactory;
private readonly IAzaionApi _azaionApi;
private readonly QueueConfig _queueConfig;
private readonly UIConfig _uiConfig;
private Producer _annotationProducer = null!;
public FailsafeAnnotationsProducer(ILogger<FailsafeAnnotationsProducer> logger, IDbFactory dbFactory, IOptions<QueueConfig> queueConfig, IAzaionApi azaionApi)
public FailsafeAnnotationsProducer(ILogger<FailsafeAnnotationsProducer> logger,
IDbFactory dbFactory,
IOptions<QueueConfig> queueConfig,
IOptions<UIConfig> uiConfig,
IAzaionApi azaionApi)
{
_logger = logger;
_dbFactory = dbFactory;
_azaionApi = azaionApi;
_queueConfig = queueConfig.Value;
_uiConfig = uiConfig.Value;
Task.Run(async () => await ProcessQueue());
}
@@ -71,7 +79,7 @@ public class FailsafeAnnotationsProducer
{
var appProperties = new ApplicationProperties
{
{ nameof(AnnotationStatus), record.Operation },
{ nameof(AnnotationStatus), record.Operation.ToString() },
{ nameof(User.Email), _azaionApi.CurrentUser.Email }
};
@@ -86,7 +94,7 @@ public class FailsafeAnnotationsProducer
}
else
{
var annotation = annotationsDict.GetValueOrDefault(record.AnnotationNames.FirstOrDefault());
var annotation = annotationsDict!.GetValueOrDefault(record.AnnotationNames.FirstOrDefault());
if (annotation == null)
continue;
@@ -118,7 +126,8 @@ public class FailsafeAnnotationsProducer
if (result.messages.Any())
{
await _annotationProducer.Send(result.messages, CompressionType.Gzip);
await _dbFactory.Run(async db => await db.DeleteAsync(result.records, token: ct));
var ids = result.records.Select(x => x.Id).ToList();
var removed = await _dbFactory.Run(async db => await db.AnnotationsQueueRecords.DeleteAsync(x => ids.Contains(x.Id), token: ct));
sent = true;
_dbFactory.SaveToDisk();
}
@@ -136,9 +145,12 @@ public class FailsafeAnnotationsProducer
public async Task SendToInnerQueue(List<string> annotationNames, AnnotationStatus status, CancellationToken cancellationToken = default)
{
if (_uiConfig.SilentDetection)
return;
await _dbFactory.Run(async db =>
await db.InsertAsync(new AnnotationQueueRecord
{
Id = Guid.NewGuid(),
DateTime = DateTime.UtcNow,
Operation = status,
AnnotationNames = annotationNames
Azaion.Common/Services/GPSMatcherService.cs
+1 -1
View File
@@ -41,7 +41,7 @@ public class GpsMatcherService(IGpsMatcherClient gpsMatcherClient, ISatelliteDow
var indexOffset = 0;
while (routeFiles.Any())
{
//await satelliteTileDownloader.GetTiles(currentLat, currentLon, SATELLITE_RADIUS_M, ZOOM_LEVEL, detectToken);
await satelliteTileDownloader.GetTiles(currentLat, currentLon, SATELLITE_RADIUS_M, ZOOM_LEVEL, detectToken);
gpsMatcherClient.StartMatching(new StartMatchingEvent
{
ImagesCount = POINTS_COUNT,
Azaion.Common/Services/InferenceClient.cs
+1 -1
View File
@@ -126,7 +126,7 @@ public class InferenceClient : IInferenceClient, IResourceLoader
_waitFileCancelSource.Cancel();
}
bytes = command.Data;
bytes = command.Data!;
_waitFileCancelSource.Cancel();
}
_waitFileCancelSource.Token.WaitForCancel(timeout ?? TimeSpan.FromSeconds(15));
Azaion.Dataset/DatasetExplorerEventHandler.cs
+15 -8
View File
@@ -1,19 +1,17 @@
using System.IO;
using System.Windows;
using System.Windows.Input;
using System.Windows.Threading;
using System.Windows.Input;
using Azaion.Common.Database;
using Azaion.Common.DTO;
using Azaion.Common.DTO.Queue;
using Azaion.Common.Events;
using Azaion.Common.Services;
using Azaion.CommonSecurity.DTO;
using Azaion.CommonSecurity.Services;
using MediatR;
using Microsoft.Extensions.Logging;
namespace Azaion.Dataset;
public class DatasetExplorerEventHandler(
ILogger<DatasetExplorerEventHandler> logger,
DatasetExplorer datasetExplorer,
IAnnotationService annotationService,
IAzaionApi azaionApi) :
@@ -22,8 +20,6 @@ public class DatasetExplorerEventHandler(
INotificationHandler<AnnotationCreatedEvent>,
INotificationHandler<AnnotationsDeletedEvent>
{
private readonly IAzaionApi _azaionApi = azaionApi;
private readonly Dictionary<Key, PlaybackControlEnum> _keysControlEnumDict = new()
{
{ Key.Enter, PlaybackControlEnum.SaveAnnotations },
@@ -127,7 +123,7 @@ public class DatasetExplorerEventHandler(
if (annotation.Classes.Contains(selectedClass) || selectedClass == -1)
{
var index = 0;
var annThumb = new AnnotationThumbnail(annotation, _azaionApi.CurrentUser.Role.IsValidator());
var annThumb = new AnnotationThumbnail(annotation, azaionApi.CurrentUser.Role.IsValidator());
if (datasetExplorer.SelectedAnnotationDict.ContainsKey(annThumb.Annotation.Name))
{
datasetExplorer.SelectedAnnotationDict.Remove(annThumb.Annotation.Name);
@@ -147,6 +143,10 @@ public class DatasetExplorerEventHandler(
}
public async Task Handle(AnnotationsDeletedEvent notification, CancellationToken cancellationToken)
{
try
{
datasetExplorer.Dispatcher.Invoke(() =>
{
var annThumbs = datasetExplorer.SelectedAnnotationDict
.Where(x => notification.AnnotationNames.Contains(x.Key))
@@ -157,6 +157,13 @@ public class DatasetExplorerEventHandler(
datasetExplorer.SelectedAnnotations.Remove(annThumb);
datasetExplorer.SelectedAnnotationDict.Remove(annThumb.Annotation.Name);
}
});
}
catch (Exception e)
{
logger.LogError(e, e.Message);
throw;
}
await Task.CompletedTask;
}
}
Azaion.Suite/App.xaml.cs
+3
View File
@@ -145,6 +145,9 @@ public partial class App
StartMain();
_host.Start();
EventManager.RegisterClassHandler(typeof(UIElement), UIElement.PreviewKeyDownEvent, new RoutedEventHandler(GlobalKeyHandler));
var datasetExplorer = _host.Services.GetRequiredService<DatasetExplorer>();
datasetExplorer.Show();
datasetExplorer.Hide();
_host.Services.GetRequiredService<MainSuite>().Show();
};
login.Closed += (sender, args) =>
Azaion.Suite/config.json
+1 -1
View File
@@ -27,6 +27,6 @@
"LeftPanelWidth": 220.0,
"RightPanelWidth": 230.0,
"GenerateAnnotatedImage": true,
"SilentDetection": true
"SilentDetection": false
}
}