azaion/annotations
1
0
Fork 0
mirror of https://github.com/azaion/annotations.git synced 2026-04-22 13:06:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixed bugs with queue handling. At least most of them

Browse Source
This commit is contained in:
Alex Bezdieniezhnykh
2025-05-18 20:11:19 +03:00
parent cf563571c8
commit c5e81ebcc6
15 changed files with 135 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Azaion.Common/Services/AnnotationService.cs
+35 -41
View File
@@ -77,40 +77,43 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
_consumer = await Consumer.Create(new ConsumerConfig(consumerSystem, Constants.MQ_ANNOTATIONS_QUEUE)
{
Reference = _api.CurrentUser.Email,
OffsetSpec = new OffsetTypeOffset(offsets.AnnotationsOffset + 1),
OffsetSpec = new OffsetTypeOffset(offsets.AnnotationsOffset),
MessageHandler = async (_, _, context, message) =>
{
try
{
var email = (string)message.ApplicationProperties[nameof(User.Email)]!;
if (email == _api.CurrentUser.Email) //Don't process messages by yourself
if (!Enum.TryParse<AnnotationStatus>((string)message.ApplicationProperties[nameof(AnnotationStatus)], out var annotationStatus))
return;
var annotationStatus = (AnnotationStatus)message.ApplicationProperties[nameof(AnnotationStatus)];
if (annotationStatus.In(AnnotationStatus.Created, AnnotationStatus.ValidatedEdited))
{
var msg = MessagePackSerializer.Deserialize<AnnotationMessage>(message.Data.Contents);
await SaveAnnotationInner(
msg.CreatedDate,
msg.OriginalMediaName,
msg.Time,
JsonConvert.DeserializeObject<List<Detection>>(msg.Detections) ?? [],
msg.Source,
new MemoryStream(msg.Image),
msg.Role,
msg.Email,
fromQueue: true,
token: cancellationToken);
}
else
{
var msg = MessagePackSerializer.Deserialize<AnnotationBulkMessage>(message.Data.Contents);
if (annotationStatus == AnnotationStatus.Validated)
await ValidateAnnotations(msg.AnnotationNames.ToList(), true, cancellationToken);
if (annotationStatus == AnnotationStatus.Deleted)
await _mediator.Publish(new AnnotationsDeletedEvent(msg.AnnotationNames.ToList(), fromQueue:true), cancellationToken);
}
offsets.AnnotationsOffset = context.Offset;
if (email != _api.CurrentUser.Email) //Don't process messages by yourself
{
if (annotationStatus.In(AnnotationStatus.Created, AnnotationStatus.ValidatedEdited))
{
var msg = MessagePackSerializer.Deserialize<AnnotationMessage>(message.Data.Contents);
await SaveAnnotationInner(
msg.CreatedDate,
msg.OriginalMediaName,
msg.Time,
JsonConvert.DeserializeObject<List<Detection>>(msg.Detections) ?? [],
msg.Source,
new MemoryStream(msg.Image),
msg.Role,
msg.Email,
fromQueue: true,
token: cancellationToken);
}
else
{
var msg = MessagePackSerializer.Deserialize<AnnotationBulkMessage>(message.Data.Contents);
if (annotationStatus == AnnotationStatus.Validated)
await ValidateAnnotations(msg.AnnotationNames.ToList(), true, cancellationToken);
if (annotationStatus == AnnotationStatus.Deleted)
await _mediator.Publish(new AnnotationsDeletedEvent(msg.AnnotationNames.ToList(), fromQueue:true), cancellationToken);
}
}
offsets.AnnotationsOffset = context.Offset + 1; //to consume on the next launch from the next message
ThrottleExt.Throttle(() =>
{
_api.UpdateOffsets(offsets);
@@ -154,12 +157,6 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
.LoadWith(x => x.Detections)
.FirstOrDefaultAsync(x => x.Name == fName, token: token);
if (userRole.IsValidator() && source == SourceEnum.Manual)
status = AnnotationStatus.Validated;
if (fromQueue && ann is { AnnotationStatus: AnnotationStatus.Validated })
return ann;
await db.Detections.DeleteAsync(x => x.AnnotationName == fName, token: token);
if (ann != null) //Annotation is already exists
@@ -169,10 +166,9 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
.Set(x => x.Source, source)
.Set(x => x.CreatedRole, userRole);
if (status == AnnotationStatus.Validated)
if (userRole.IsValidator() && source == SourceEnum.Manual)
{
if (status == AnnotationStatus.Validated)
status = AnnotationStatus.ValidatedEdited; //For further processing mark Annotations *edited* by Validator, not just simply Validated by button.
status = AnnotationStatus.ValidatedEdited;
annotationUpdatable = annotationUpdatable
.Set(x => x.ValidateDate, createdDate)
@@ -212,9 +208,6 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
return ann;
});
if (fromQueue && annotation is { AnnotationStatus: AnnotationStatus.Validated })
return annotation;
if (stream != null)
{
var img = System.Drawing.Image.FromStream(stream);
@@ -228,7 +221,7 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
await _mediator.Publish(new AnnotationCreatedEvent(annotation), token);
if (!fromQueue && !_uiConfig.SilentDetection) //Send to queue only if we're not getting from queue already
if (!fromQueue) //Send to queue only if we're not getting from queue already
await _producer.SendToInnerQueue([annotation.Name], status, token);
ThrottleExt.Throttle(async () =>
@@ -275,7 +268,8 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
File.Delete(Path.Combine(_dirConfig.ResultsDirectory, $"{name}{Constants.RESULT_PREFIX}{Constants.JPG_EXT}"));
}
if (!notification.FromQueue)
//Only validators can send Delete to the queue
if (!notification.FromQueue && _api.CurrentUser.Role.IsValidator())
await _producer.SendToInnerQueue(notification.AnnotationNames, AnnotationStatus.Deleted, ct);
ThrottleExt.Throttle(async () =>
Azaion.Common/Services/FailsafeProducer.cs
+16 -4
View File
@@ -23,14 +23,22 @@ public class FailsafeAnnotationsProducer
private readonly IDbFactory _dbFactory;
private readonly IAzaionApi _azaionApi;
private readonly QueueConfig _queueConfig;
private readonly UIConfig _uiConfig;
private Producer _annotationProducer = null!;
public FailsafeAnnotationsProducer(ILogger<FailsafeAnnotationsProducer> logger, IDbFactory dbFactory, IOptions<QueueConfig> queueConfig, IAzaionApi azaionApi)
public FailsafeAnnotationsProducer(ILogger<FailsafeAnnotationsProducer> logger,
IDbFactory dbFactory,
IOptions<QueueConfig> queueConfig,
IOptions<UIConfig> uiConfig,
IAzaionApi azaionApi)
{
_logger = logger;
_dbFactory = dbFactory;
_azaionApi = azaionApi;
_queueConfig = queueConfig.Value;
_uiConfig = uiConfig.Value;
Task.Run(async () => await ProcessQueue());
}
@@ -71,7 +79,7 @@ public class FailsafeAnnotationsProducer
{
var appProperties = new ApplicationProperties
{
{ nameof(AnnotationStatus), record.Operation },
{ nameof(AnnotationStatus), record.Operation.ToString() },
{ nameof(User.Email), _azaionApi.CurrentUser.Email }
};
@@ -86,7 +94,7 @@ public class FailsafeAnnotationsProducer
}
else
{
var annotation = annotationsDict.GetValueOrDefault(record.AnnotationNames.FirstOrDefault());
var annotation = annotationsDict!.GetValueOrDefault(record.AnnotationNames.FirstOrDefault());
if (annotation == null)
continue;
@@ -118,7 +126,8 @@ public class FailsafeAnnotationsProducer
if (result.messages.Any())
{
await _annotationProducer.Send(result.messages, CompressionType.Gzip);
await _dbFactory.Run(async db => await db.DeleteAsync(result.records, token: ct));
var ids = result.records.Select(x => x.Id).ToList();
var removed = await _dbFactory.Run(async db => await db.AnnotationsQueueRecords.DeleteAsync(x => ids.Contains(x.Id), token: ct));
sent = true;
_dbFactory.SaveToDisk();
}
@@ -136,9 +145,12 @@ public class FailsafeAnnotationsProducer
public async Task SendToInnerQueue(List<string> annotationNames, AnnotationStatus status, CancellationToken cancellationToken = default)
{
if (_uiConfig.SilentDetection)
return;
await _dbFactory.Run(async db =>
await db.InsertAsync(new AnnotationQueueRecord
{
Id = Guid.NewGuid(),
DateTime = DateTime.UtcNow,
Operation = status,
AnnotationNames = annotationNames
Azaion.Common/Services/GPSMatcherService.cs
+1 -1
View File
@@ -41,7 +41,7 @@ public class GpsMatcherService(IGpsMatcherClient gpsMatcherClient, ISatelliteDow
var indexOffset = 0;
while (routeFiles.Any())
{
//await satelliteTileDownloader.GetTiles(currentLat, currentLon, SATELLITE_RADIUS_M, ZOOM_LEVEL, detectToken);
await satelliteTileDownloader.GetTiles(currentLat, currentLon, SATELLITE_RADIUS_M, ZOOM_LEVEL, detectToken);
gpsMatcherClient.StartMatching(new StartMatchingEvent
{
ImagesCount = POINTS_COUNT,
Azaion.Common/Services/InferenceClient.cs
+1 -1
View File
@@ -126,7 +126,7 @@ public class InferenceClient : IInferenceClient, IResourceLoader
_waitFileCancelSource.Cancel();
}
bytes = command.Data;
bytes = command.Data!;
_waitFileCancelSource.Cancel();
}
_waitFileCancelSource.Token.WaitForCancel(timeout ?? TimeSpan.FromSeconds(15));