mirror of
https://github.com/azaion/annotations.git
synced 2026-04-23 00:16:30 +00:00
huge queue refactoring:
3 queues -> 1 queue send delete validate updates
This commit is contained in:
Alex Bezdieniezhnykh
@@ -3,12 +3,16 @@ using System.Net;
|
||||
using Azaion.Common.Database;
|
||||
using Azaion.Common.DTO.Config;
|
||||
using Azaion.Common.DTO.Queue;
|
||||
using Azaion.Common.Extensions;
|
||||
using Azaion.CommonSecurity.DTO;
|
||||
using Azaion.CommonSecurity.Services;
|
||||
using LinqToDB;
|
||||
using MessagePack;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Newtonsoft.Json;
|
||||
using RabbitMQ.Stream.Client;
|
||||
using RabbitMQ.Stream.Client.AMQP;
|
||||
using RabbitMQ.Stream.Client.Reliable;
|
||||
|
||||
namespace Azaion.Common.Services;
|
||||
@@ -17,16 +21,15 @@ public class FailsafeAnnotationsProducer
|
||||
{
|
||||
private readonly ILogger<FailsafeAnnotationsProducer> _logger;
|
||||
private readonly IDbFactory _dbFactory;
|
||||
private readonly IAzaionApi _azaionApi;
|
||||
private readonly QueueConfig _queueConfig;
|
||||
|
||||
private Producer _annotationProducer = null!;
|
||||
private Producer _annotationConfirmProducer = null!;
|
||||
|
||||
|
||||
public FailsafeAnnotationsProducer(ILogger<FailsafeAnnotationsProducer> logger, IDbFactory dbFactory, IOptions<QueueConfig> queueConfig)
|
||||
public FailsafeAnnotationsProducer(ILogger<FailsafeAnnotationsProducer> logger, IDbFactory dbFactory, IOptions<QueueConfig> queueConfig, IAzaionApi azaionApi)
|
||||
{
|
||||
_logger = logger;
|
||||
_dbFactory = dbFactory;
|
||||
_azaionApi = azaionApi;
|
||||
_queueConfig = queueConfig.Value;
|
||||
Task.Run(async () => await ProcessQueue());
|
||||
}
|
||||
@@ -41,107 +44,104 @@ public class FailsafeAnnotationsProducer
|
||||
});
|
||||
}
|
||||
|
||||
private async Task Init(CancellationToken cancellationToken = default)
|
||||
private async Task ProcessQueue(CancellationToken ct = default)
|
||||
{
|
||||
_annotationProducer = await Producer.Create(new ProducerConfig(await GetProducerQueueConfig(), Constants.MQ_ANNOTATIONS_QUEUE));
|
||||
_annotationConfirmProducer = await Producer.Create(new ProducerConfig(await GetProducerQueueConfig(), Constants.MQ_ANNOTATIONS_CONFIRM_QUEUE));
|
||||
}
|
||||
|
||||
private async Task ProcessQueue(CancellationToken cancellationToken = default)
|
||||
{
|
||||
await Init(cancellationToken);
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
var messages = await GetFromInnerQueue(cancellationToken);
|
||||
foreach (var messagesChunk in messages.Chunk(10)) //Sending by 10
|
||||
{
|
||||
var sent = false;
|
||||
while (!sent || cancellationToken.IsCancellationRequested) //Waiting for send
|
||||
{
|
||||
try
|
||||
{
|
||||
var createdMessages = messagesChunk
|
||||
.Where(x => x.Status == AnnotationStatus.Created)
|
||||
.Select(x => new Message(MessagePackSerializer.Serialize(x)))
|
||||
.ToList();
|
||||
if (createdMessages.Any())
|
||||
await _annotationProducer.Send(createdMessages, CompressionType.Gzip);
|
||||
|
||||
var validatedMessages = messagesChunk
|
||||
.Where(x => x.Status == AnnotationStatus.Validated)
|
||||
.Select(x => new Message(MessagePackSerializer.Serialize(x)))
|
||||
.ToList();
|
||||
if (validatedMessages.Any())
|
||||
await _annotationConfirmProducer.Send(validatedMessages, CompressionType.Gzip);
|
||||
|
||||
await _dbFactory.Run(async db =>
|
||||
await db.AnnotationsQueue.DeleteAsync(aq => messagesChunk.Any(x => aq.Name == x.Name), token: cancellationToken));
|
||||
sent = true;
|
||||
_dbFactory.SaveToDisk();
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, e.Message);
|
||||
await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken);
|
||||
}
|
||||
await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken);
|
||||
}
|
||||
}
|
||||
await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<List<AnnotationCreatedMessage>> GetFromInnerQueue(CancellationToken cancellationToken = default)
|
||||
{
|
||||
return await _dbFactory.Run(async db =>
|
||||
{
|
||||
var annotations = await db.AnnotationsQueue.Join(
|
||||
db.Annotations.LoadWith(x => x.Detections), aq => aq.Name, a => a.Name, (aq, a) => a)
|
||||
.ToListAsync(token: cancellationToken);
|
||||
|
||||
var messages = new List<AnnotationCreatedMessage>();
|
||||
var badImages = new List<string>();
|
||||
foreach (var annotation in annotations)
|
||||
var sent = false;
|
||||
while (!sent || !ct.IsCancellationRequested) //Waiting for send
|
||||
{
|
||||
try
|
||||
{
|
||||
var image = await File.ReadAllBytesAsync(annotation.ImagePath, cancellationToken);
|
||||
var annCreateMessage = new AnnotationCreatedMessage
|
||||
var result = await _dbFactory.Run(async db =>
|
||||
{
|
||||
Name = annotation.Name,
|
||||
OriginalMediaName = annotation.OriginalMediaName,
|
||||
Time = annotation.Time,
|
||||
CreatedRole = annotation.CreatedRole,
|
||||
CreatedEmail = annotation.CreatedEmail,
|
||||
CreatedDate = annotation.CreatedDate,
|
||||
Status = annotation.AnnotationStatus,
|
||||
var records = await db.AnnotationsQueueRecords.OrderBy(x => x.DateTime).ToListAsync(token: ct);
|
||||
var editedCreatedNames = records
|
||||
.Where(x => x.Operation.In(AnnotationStatus.Created, AnnotationStatus.ValidatedEdited))
|
||||
.Select(x => x.AnnotationNames.FirstOrDefault())
|
||||
.ToList();
|
||||
|
||||
ImageExtension = annotation.ImageExtension,
|
||||
Image = image,
|
||||
Detections = JsonConvert.SerializeObject(annotation.Detections),
|
||||
Source = annotation.Source,
|
||||
};
|
||||
messages.Add(annCreateMessage);
|
||||
var annotationsDict = await db.Annotations.LoadWith(x => x.Detections)
|
||||
.Where(x => editedCreatedNames.Contains(x.Name))
|
||||
.ToDictionaryAsync(a => a.Name, token: ct);
|
||||
|
||||
var messages = new List<Message>();
|
||||
foreach (var record in records)
|
||||
{
|
||||
var appProperties = new ApplicationProperties
|
||||
{
|
||||
{ nameof(AnnotationStatus), record.Operation },
|
||||
{ nameof(User.Email), _azaionApi.CurrentUser.Email }
|
||||
};
|
||||
|
||||
if (record.Operation.In(AnnotationStatus.Validated, AnnotationStatus.Deleted))
|
||||
{
|
||||
var message = new Message(MessagePackSerializer.Serialize(new AnnotationBulkMessage
|
||||
{
|
||||
AnnotationNames = record.AnnotationNames.ToArray()
|
||||
})) { ApplicationProperties = appProperties };
|
||||
|
||||
messages.Add(message);
|
||||
}
|
||||
else
|
||||
{
|
||||
var annotation = annotationsDict.GetValueOrDefault(record.AnnotationNames.FirstOrDefault());
|
||||
if (annotation == null)
|
||||
continue;
|
||||
|
||||
var image = await File.ReadAllBytesAsync(annotation.ImagePath, ct);
|
||||
var annMessage = new AnnotationMessage
|
||||
{
|
||||
Name = annotation.Name,
|
||||
OriginalMediaName = annotation.OriginalMediaName,
|
||||
Time = annotation.Time,
|
||||
Role = annotation.CreatedRole,
|
||||
Email = annotation.CreatedEmail,
|
||||
CreatedDate = annotation.CreatedDate,
|
||||
Status = annotation.AnnotationStatus,
|
||||
|
||||
ImageExtension = annotation.ImageExtension,
|
||||
Image = image,
|
||||
Detections = JsonConvert.SerializeObject(annotation.Detections),
|
||||
Source = annotation.Source,
|
||||
};
|
||||
var message = new Message(MessagePackSerializer.Serialize(annMessage)) { ApplicationProperties = appProperties };
|
||||
|
||||
messages.Add(message);
|
||||
}
|
||||
}
|
||||
|
||||
return (messages, records);
|
||||
});
|
||||
|
||||
if (result.messages.Any())
|
||||
{
|
||||
await _annotationProducer.Send(result.messages, CompressionType.Gzip);
|
||||
await _dbFactory.Run(async db => await db.DeleteAsync(result.records, token: ct));
|
||||
sent = true;
|
||||
_dbFactory.SaveToDisk();
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, e.Message);
|
||||
badImages.Add(annotation.Name);
|
||||
await Task.Delay(TimeSpan.FromSeconds(10), ct);
|
||||
}
|
||||
await Task.Delay(TimeSpan.FromSeconds(10), ct);
|
||||
}
|
||||
|
||||
if (badImages.Any())
|
||||
{
|
||||
await db.AnnotationsQueue.Where(x => badImages.Contains(x.Name)).DeleteAsync(token: cancellationToken);
|
||||
_dbFactory.SaveToDisk();
|
||||
}
|
||||
return messages;
|
||||
});
|
||||
}
|
||||
await Task.Delay(TimeSpan.FromSeconds(5), ct);
|
||||
}
|
||||
|
||||
public async Task SendToInnerQueue(Annotation annotation, CancellationToken cancellationToken = default)
|
||||
public async Task SendToInnerQueue(List<string> annotationNames, AnnotationStatus status, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await _dbFactory.Run(async db =>
|
||||
await db.InsertAsync(new AnnotationName { Name = annotation.Name }, token: cancellationToken));
|
||||
await db.InsertAsync(new AnnotationQueueRecord
|
||||
{
|
||||
DateTime = DateTime.UtcNow,
|
||||
Operation = status,
|
||||
AnnotationNames = annotationNames
|
||||
}, token: cancellationToken));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user