azaion/annotations
1
0
Fork 0
mirror of https://github.com/azaion/annotations.git synced 2026-04-22 10:56:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

fixed bugs with queue handling. At least most of them

Browse Source
This commit is contained in:
Alex Bezdieniezhnykh
2025-05-18 20:11:19 +03:00
parent cf563571c8
commit c5e81ebcc6
15 changed files with 135 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Azaion.Common/Constants.cs
-5
View File
@@ -97,12 +97,7 @@ public class Constants
#endregion
#region Queue
public const string MQ_ANNOTATIONS_QUEUE = "azaion-annotations";
public const string MQ_ANNOTATIONS_CONFIRM_QUEUE = "azaion-annotations-confirm";
#endregion
#region Database
Azaion.Common/Database/Annotation.cs
+4 -3
View File
@@ -59,7 +59,8 @@ public enum AnnotationStatus
{
None = 0,
Created = 10,
Validated = 20,
ValidatedEdited = 25,
Deleted = 30
ValidatedEdited = 20,
Validated = 30,
Deleted = 40
}
Azaion.Common/Database/AnnotationQueueRecord.cs
+3 -2
View File
@@ -2,7 +2,8 @@ namespace Azaion.Common.Database;
public class AnnotationQueueRecord
{
public DateTime DateTime { get; set; }
public AnnotationStatus Operation { get; set; }
public Guid Id { get; set; }
public DateTime DateTime { get; set; }
public AnnotationStatus Operation { get; set; }
public List<string> AnnotationNames { get; set; } = null!;
}
Azaion.Common/Database/AnnotationsDb.cs
-1
View File
@@ -9,5 +9,4 @@ public class AnnotationsDb(DataOptions dataOptions) : DataConnection(dataOptions
public ITable<Annotation> Annotations => this.GetTable<Annotation>();
public ITable<AnnotationQueueRecord> AnnotationsQueueRecords => this.GetTable<AnnotationQueueRecord>();
public ITable<Detection> Detections => this.GetTable<Detection>();
public ITable<QueueOffset> QueueOffsets => this.GetTable<QueueOffset>();
}
Azaion.Common/Database/DbFactory.cs
+13 -20
View File
@@ -53,32 +53,24 @@ public class DbFactory : IDbFactory
.UseMappingSchema(AnnotationsDbSchemaHolder.MappingSchema);
if (!File.Exists(_annConfig.AnnotationsDbFile))
CreateDb();
SQLiteConnection.CreateFile(_annConfig.AnnotationsDbFile);
RecreateTables();
_fileConnection.Open();
_fileConnection.BackupDatabase(_memoryConnection, "main", "main", -1, null, -1);
}
private void CreateDb()
private void RecreateTables()
{
SQLiteConnection.CreateFile(_annConfig.AnnotationsDbFile);
using var db = new AnnotationsDb(_fileDataOptions);
db.CreateTable<Annotation>();
db.CreateTable<AnnotationQueueRecord>();
db.CreateTable<Detection>();
db.CreateTable<QueueOffset>();
db.QueueOffsets.BulkCopy(new List<QueueOffset>
{
new()
{
Offset = 0,
QueueName = Constants.MQ_ANNOTATIONS_QUEUE
},
new()
{
Offset = 0,
QueueName = Constants.MQ_ANNOTATIONS_CONFIRM_QUEUE
}
});
var schema = db.DataProvider.GetSchemaProvider().GetSchema(db);
var existingTables = schema.Tables.Select(x => x.TableName).ToHashSet();
if (!existingTables.Contains(Constants.ANNOTATIONS_TABLENAME))
db.CreateTable<Annotation>();
if (!existingTables.Contains(Constants.DETECTIONS_TABLENAME))
db.CreateTable<Detection>();
if (!existingTables.Contains(Constants.ANNOTATIONS_QUEUE_TABLENAME))
db.CreateTable<AnnotationQueueRecord>();
}
public async Task<T> Run<T>(Func<AnnotationsDb, Task<T>> func)
@@ -138,6 +130,7 @@ public static class AnnotationsDbSchemaHolder
builder.Entity<AnnotationQueueRecord>()
.HasTableName(Constants.ANNOTATIONS_QUEUE_TABLENAME)
.HasPrimaryKey(x => x.Id)
.Property(x => x.AnnotationNames)
.HasDataType(DataType.NVarChar)
.HasConversion(list => JsonConvert.SerializeObject(list), str => JsonConvert.DeserializeObject<List<string>>(str) ?? new List<string>());
Azaion.Common/Database/QueueOffset.cs
-7
View File
@@ -1,7 +0,0 @@
namespace Azaion.Common.Database;
public class QueueOffset
{
public string QueueName { get; set; } = null!;
public ulong Offset { get; set; }
}
Azaion.Common/Services/AnnotationService.cs
+35 -41
View File
@@ -77,40 +77,43 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
_consumer = await Consumer.Create(new ConsumerConfig(consumerSystem, Constants.MQ_ANNOTATIONS_QUEUE)
{
Reference = _api.CurrentUser.Email,
OffsetSpec = new OffsetTypeOffset(offsets.AnnotationsOffset + 1),
OffsetSpec = new OffsetTypeOffset(offsets.AnnotationsOffset),
MessageHandler = async (_, _, context, message) =>
{
try
{
var email = (string)message.ApplicationProperties[nameof(User.Email)]!;
if (email == _api.CurrentUser.Email) //Don't process messages by yourself
if (!Enum.TryParse<AnnotationStatus>((string)message.ApplicationProperties[nameof(AnnotationStatus)], out var annotationStatus))
return;
var annotationStatus = (AnnotationStatus)message.ApplicationProperties[nameof(AnnotationStatus)];
if (annotationStatus.In(AnnotationStatus.Created, AnnotationStatus.ValidatedEdited))
{
var msg = MessagePackSerializer.Deserialize<AnnotationMessage>(message.Data.Contents);
await SaveAnnotationInner(
msg.CreatedDate,
msg.OriginalMediaName,
msg.Time,
JsonConvert.DeserializeObject<List<Detection>>(msg.Detections) ?? [],
msg.Source,
new MemoryStream(msg.Image),
msg.Role,
msg.Email,
fromQueue: true,
token: cancellationToken);
}
else
{
var msg = MessagePackSerializer.Deserialize<AnnotationBulkMessage>(message.Data.Contents);
if (annotationStatus == AnnotationStatus.Validated)
await ValidateAnnotations(msg.AnnotationNames.ToList(), true, cancellationToken);
if (annotationStatus == AnnotationStatus.Deleted)
await _mediator.Publish(new AnnotationsDeletedEvent(msg.AnnotationNames.ToList(), fromQueue:true), cancellationToken);
}
offsets.AnnotationsOffset = context.Offset;
if (email != _api.CurrentUser.Email) //Don't process messages by yourself
{
if (annotationStatus.In(AnnotationStatus.Created, AnnotationStatus.ValidatedEdited))
{
var msg = MessagePackSerializer.Deserialize<AnnotationMessage>(message.Data.Contents);
await SaveAnnotationInner(
msg.CreatedDate,
msg.OriginalMediaName,
msg.Time,
JsonConvert.DeserializeObject<List<Detection>>(msg.Detections) ?? [],
msg.Source,
new MemoryStream(msg.Image),
msg.Role,
msg.Email,
fromQueue: true,
token: cancellationToken);
}
else
{
var msg = MessagePackSerializer.Deserialize<AnnotationBulkMessage>(message.Data.Contents);
if (annotationStatus == AnnotationStatus.Validated)
await ValidateAnnotations(msg.AnnotationNames.ToList(), true, cancellationToken);
if (annotationStatus == AnnotationStatus.Deleted)
await _mediator.Publish(new AnnotationsDeletedEvent(msg.AnnotationNames.ToList(), fromQueue:true), cancellationToken);
}
}
offsets.AnnotationsOffset = context.Offset + 1; //to consume on the next launch from the next message
ThrottleExt.Throttle(() =>
{
_api.UpdateOffsets(offsets);
@@ -154,12 +157,6 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
.LoadWith(x => x.Detections)
.FirstOrDefaultAsync(x => x.Name == fName, token: token);
if (userRole.IsValidator() && source == SourceEnum.Manual)
status = AnnotationStatus.Validated;
if (fromQueue && ann is { AnnotationStatus: AnnotationStatus.Validated })
return ann;
await db.Detections.DeleteAsync(x => x.AnnotationName == fName, token: token);
if (ann != null) //Annotation is already exists
@@ -169,10 +166,9 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
.Set(x => x.Source, source)
.Set(x => x.CreatedRole, userRole);
if (status == AnnotationStatus.Validated)
if (userRole.IsValidator() && source == SourceEnum.Manual)
{
if (status == AnnotationStatus.Validated)
status = AnnotationStatus.ValidatedEdited; //For further processing mark Annotations *edited* by Validator, not just simply Validated by button.
status = AnnotationStatus.ValidatedEdited;
annotationUpdatable = annotationUpdatable
.Set(x => x.ValidateDate, createdDate)
@@ -212,9 +208,6 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
return ann;
});
if (fromQueue && annotation is { AnnotationStatus: AnnotationStatus.Validated })
return annotation;
if (stream != null)
{
var img = System.Drawing.Image.FromStream(stream);
@@ -228,7 +221,7 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
await _mediator.Publish(new AnnotationCreatedEvent(annotation), token);
if (!fromQueue && !_uiConfig.SilentDetection) //Send to queue only if we're not getting from queue already
if (!fromQueue) //Send to queue only if we're not getting from queue already
await _producer.SendToInnerQueue([annotation.Name], status, token);
ThrottleExt.Throttle(async () =>
@@ -275,7 +268,8 @@ public class AnnotationService : IAnnotationService, INotificationHandler<Annota
File.Delete(Path.Combine(_dirConfig.ResultsDirectory, $"{name}{Constants.RESULT_PREFIX}{Constants.JPG_EXT}"));
}
if (!notification.FromQueue)
//Only validators can send Delete to the queue
if (!notification.FromQueue && _api.CurrentUser.Role.IsValidator())
await _producer.SendToInnerQueue(notification.AnnotationNames, AnnotationStatus.Deleted, ct);
ThrottleExt.Throttle(async () =>
Azaion.Common/Services/FailsafeProducer.cs
+16 -4
View File
@@ -23,14 +23,22 @@ public class FailsafeAnnotationsProducer
private readonly IDbFactory _dbFactory;
private readonly IAzaionApi _azaionApi;
private readonly QueueConfig _queueConfig;
private readonly UIConfig _uiConfig;
private Producer _annotationProducer = null!;
public FailsafeAnnotationsProducer(ILogger<FailsafeAnnotationsProducer> logger, IDbFactory dbFactory, IOptions<QueueConfig> queueConfig, IAzaionApi azaionApi)
public FailsafeAnnotationsProducer(ILogger<FailsafeAnnotationsProducer> logger,
IDbFactory dbFactory,
IOptions<QueueConfig> queueConfig,
IOptions<UIConfig> uiConfig,
IAzaionApi azaionApi)
{
_logger = logger;
_dbFactory = dbFactory;
_azaionApi = azaionApi;
_queueConfig = queueConfig.Value;
_uiConfig = uiConfig.Value;
Task.Run(async () => await ProcessQueue());
}
@@ -71,7 +79,7 @@ public class FailsafeAnnotationsProducer
{
var appProperties = new ApplicationProperties
{
{ nameof(AnnotationStatus), record.Operation },
{ nameof(AnnotationStatus), record.Operation.ToString() },
{ nameof(User.Email), _azaionApi.CurrentUser.Email }
};
@@ -86,7 +94,7 @@ public class FailsafeAnnotationsProducer
}
else
{
var annotation = annotationsDict.GetValueOrDefault(record.AnnotationNames.FirstOrDefault());
var annotation = annotationsDict!.GetValueOrDefault(record.AnnotationNames.FirstOrDefault());
if (annotation == null)
continue;
@@ -118,7 +126,8 @@ public class FailsafeAnnotationsProducer
if (result.messages.Any())
{
await _annotationProducer.Send(result.messages, CompressionType.Gzip);
await _dbFactory.Run(async db => await db.DeleteAsync(result.records, token: ct));
var ids = result.records.Select(x => x.Id).ToList();
var removed = await _dbFactory.Run(async db => await db.AnnotationsQueueRecords.DeleteAsync(x => ids.Contains(x.Id), token: ct));
sent = true;
_dbFactory.SaveToDisk();
}
@@ -136,9 +145,12 @@ public class FailsafeAnnotationsProducer
public async Task SendToInnerQueue(List<string> annotationNames, AnnotationStatus status, CancellationToken cancellationToken = default)
{
if (_uiConfig.SilentDetection)
return;
await _dbFactory.Run(async db =>
await db.InsertAsync(new AnnotationQueueRecord
{
Id = Guid.NewGuid(),
DateTime = DateTime.UtcNow,
Operation = status,
AnnotationNames = annotationNames
Azaion.Common/Services/GPSMatcherService.cs
+1 -1
View File
@@ -41,7 +41,7 @@ public class GpsMatcherService(IGpsMatcherClient gpsMatcherClient, ISatelliteDow
var indexOffset = 0;
while (routeFiles.Any())
{
//await satelliteTileDownloader.GetTiles(currentLat, currentLon, SATELLITE_RADIUS_M, ZOOM_LEVEL, detectToken);
await satelliteTileDownloader.GetTiles(currentLat, currentLon, SATELLITE_RADIUS_M, ZOOM_LEVEL, detectToken);
gpsMatcherClient.StartMatching(new StartMatchingEvent
{
ImagesCount = POINTS_COUNT,
Azaion.Common/Services/InferenceClient.cs
+1 -1
View File
@@ -126,7 +126,7 @@ public class InferenceClient : IInferenceClient, IResourceLoader
_waitFileCancelSource.Cancel();
}
bytes = command.Data;
bytes = command.Data!;
_waitFileCancelSource.Cancel();
}
_waitFileCancelSource.Token.WaitForCancel(timeout ?? TimeSpan.FromSeconds(15));