azaion/annotations
1
0
Fork 0
mirror of https://github.com/azaion/annotations.git synced 2026-04-22 12:46:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

gps matcher async

Browse Source 
put cryptography lib to fixed version
fix race condition bug in queue handler
add lock to db writing and backup to file db on each write
This commit is contained in:
Alex Bezdieniezhnykh
2025-05-29 00:35:35 +03:00
parent 34ea821fb3
commit d842466594
12 changed files with 245 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Azaion.Common/Services/AnnotationService.cs
+43 -52
View File
@@ -1,4 +1,5 @@
using System.Drawing.Imaging;
using System.Drawing;
using System.Drawing.Imaging;
using System.IO;
using System.Net;
using Azaion.Common.Database;
@@ -21,7 +22,8 @@ using RabbitMQ.Stream.Client.Reliable;
namespace Azaion.Common.Services;
public class AnnotationService : IAnnotationService, INotificationHandler<AnnotationsDeletedEvent>
// SHOULD BE ONLY ONE INSTANCE OF AnnotationService. Do not add ANY NotificationHandler to it!
public class AnnotationService : IAnnotationService
{
private readonly IDbFactory _dbFactory;
private readonly FailsafeAnnotationsProducer _producer;
@@ -32,16 +34,16 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
private readonly QueueConfig _queueConfig;
private Consumer _consumer = null!;
private readonly UIConfig _uiConfig;
private readonly DirectoriesConfig _dirConfig;
private static readonly Guid SaveTaskId = Guid.NewGuid();
private readonly SemaphoreSlim _imageAccessSemaphore = new(1, 1);
private readonly SemaphoreSlim _messageProcessingSemaphore = new(1, 1);
private static readonly Guid SaveQueueOffsetTaskId = Guid.NewGuid();
public AnnotationService(
IDbFactory dbFactory,
FailsafeAnnotationsProducer producer,
IOptions<QueueConfig> queueConfig,
IOptions<UIConfig> uiConfig,
IOptions<DirectoriesConfig> directoriesConfig,
IGalleryService galleryService,
IMediator mediator,
IAzaionApi api,
@@ -55,7 +57,6 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
_logger = logger;
_queueConfig = queueConfig.Value;
_uiConfig = uiConfig.Value;
_dirConfig = directoriesConfig.Value;
Task.Run(async () => await InitQueueConsumer()).Wait();
}
@@ -80,6 +81,7 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
OffsetSpec = new OffsetTypeOffset(offsets.AnnotationsOffset),
MessageHandler = async (_, _, context, message) =>
{
await _messageProcessingSemaphore.WaitAsync(cancellationToken);
try
{
var email = (string)message.ApplicationProperties[nameof(User.Email)]!;
@@ -100,7 +102,7 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
msg.Image == null ? null : new MemoryStream(msg.Image),
msg.Role,
msg.Email,
fromQueue: true,
context.Offset,
token: cancellationToken);
}
else
@@ -124,6 +126,10 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
{
_logger.LogError(e, e.Message);
}
finally
{
_messageProcessingSemaphore.Release();
}
}
});
}
@@ -145,12 +151,12 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
List<Detection> detections, SourceEnum source, Stream? stream,
RoleEnum userRole,
string createdEmail,
bool fromQueue = false,
ulong? offset = null,
CancellationToken token = default)
{
var status = AnnotationStatus.Created;
var fName = originalMediaName.ToTimeName(time);
var annotation = await _dbFactory.Run(async db =>
var annotation = await _dbFactory.RunWrite(async db =>
{
var ann = await db.Annotations
.LoadWith(x => x.Detections)
@@ -200,27 +206,40 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
return ann;
});
if (stream != null)
//Save image should be done in 1 thread only
await _imageAccessSemaphore.WaitAsync(token);
try
{
var img = System.Drawing.Image.FromStream(stream);
img.Save(annotation.ImagePath, ImageFormat.Jpeg); //todo: check png images coming from queue
}
await YoloLabel.WriteToFile(detections, annotation.LabelPath, token);
Image image = null!;
if (stream != null)
{
image = Image.FromStream(stream);
if (File.Exists(annotation.ImagePath))
ResilienceExt.WithRetry(() => File.Delete(annotation.ImagePath));
image.Save(annotation.ImagePath, ImageFormat.Jpeg); //todo: check png images coming from queue
}
await _galleryService.CreateThumbnail(annotation, token);
if (_uiConfig.GenerateAnnotatedImage)
await _galleryService.CreateAnnotatedImage(annotation, token);
await YoloLabel.WriteToFile(detections, annotation.LabelPath, token);
await _galleryService.CreateThumbnail(annotation, image, token);
if (_uiConfig.GenerateAnnotatedImage)
await _galleryService.CreateAnnotatedImage(annotation, image, token);
}
catch (Exception e)
{
_logger.LogError(e, $"Try to save {annotation.ImagePath}, Error: {e.Message}");
throw;
}
finally
{
_imageAccessSemaphore.Release();
}
await _mediator.Publish(new AnnotationCreatedEvent(annotation), token);
if (!fromQueue) //Send to queue only if we're not getting from queue already
if (!offset.HasValue) //Send to queue only if we're not getting from queue already
await _producer.SendToInnerQueue([annotation.Name], status, token);
ThrottleExt.Throttle(async () =>
{
_dbFactory.SaveToDisk();
await Task.CompletedTask;
}, SaveTaskId, TimeSpan.FromSeconds(5), true);
return annotation;
}
@@ -230,7 +249,7 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
return;
var annNames = annotationNames.ToHashSet();
await _dbFactory.Run(async db =>
await _dbFactory.RunWrite(async db =>
{
await db.Annotations
.Where(x => annNames.Contains(x.Name))
@@ -241,34 +260,6 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
});
if (!fromQueue)
await _producer.SendToInnerQueue(annotationNames, AnnotationStatus.Validated, token);
ThrottleExt.Throttle(async () =>
{
_dbFactory.SaveToDisk();
await Task.CompletedTask;
}, SaveTaskId, TimeSpan.FromSeconds(5), true);
}
public async Task Handle(AnnotationsDeletedEvent notification, CancellationToken ct)
{
await _dbFactory.DeleteAnnotations(notification.AnnotationNames, ct);
foreach (var name in notification.AnnotationNames)
{
File.Delete(Path.Combine(_dirConfig.ImagesDirectory, $"{name}{Constants.JPG_EXT}"));
File.Delete(Path.Combine(_dirConfig.LabelsDirectory, $"{name}{Constants.TXT_EXT}"));
File.Delete(Path.Combine(_dirConfig.ThumbnailsDirectory, $"{name}{Constants.THUMBNAIL_PREFIX}{Constants.JPG_EXT}"));
File.Delete(Path.Combine(_dirConfig.ResultsDirectory, $"{name}{Constants.RESULT_PREFIX}{Constants.JPG_EXT}"));
}
//Only validators can send Delete to the queue
if (!notification.FromQueue && _api.CurrentUser.Role.IsValidator())
await _producer.SendToInnerQueue(notification.AnnotationNames, AnnotationStatus.Deleted, ct);
ThrottleExt.Throttle(async () =>
{
_dbFactory.SaveToDisk();
await Task.CompletedTask;
}, SaveTaskId, TimeSpan.FromSeconds(5), true);
}
}