diff --git a/Azaion.Annotator/Annotator.xaml.cs b/Azaion.Annotator/Annotator.xaml.cs index 9feeba4..9e4c0c9 100644 --- a/Azaion.Annotator/Annotator.xaml.cs +++ b/Azaion.Annotator/Annotator.xaml.cs @@ -14,6 +14,7 @@ using Azaion.Common.DTO.Config; using Azaion.Common.Events; using Azaion.Common.Extensions; using Azaion.Common.Services; +using Azaion.Common.Services.Inference; using LibVLCSharp.Shared; using MediatR; using Microsoft.WindowsAPICodePack.Dialogs; @@ -106,38 +107,6 @@ public partial class Annotator _logger.LogError(e, e.Message); } }; - _inferenceClient.AIAvailabilityReceived += (_, command) => - { - Dispatcher.Invoke(() => - { - _logger.LogInformation(command.Message); - var aiEnabled = command.Message == "enabled"; - AIDetectBtn.IsEnabled = aiEnabled; - var aiDisabledText = "Будь ласка, зачекайте, наразі розпізнавання AI недоступне"; - var messagesDict = new Dictionary - { - { "disabled", aiDisabledText }, - { "downloading", "Будь ласка зачекайте, йде завантаження AI для Вашої відеокарти" }, - { "converting", "Будь ласка зачекайте, йде налаштування AI під Ваше залізо. (5-12 хвилин в залежності від моделі відеокарти, до 50 хв на старих GTX1650)" }, - { "uploading", "Будь ласка зачекайте, йде зберігання" }, - { "enabled", "AI готовий для розпізнавання" } - }; - - if (command.Message?.StartsWith("Error") ?? false) - { - _logger.LogError(command.Message); - StatusHelp.Text = command.Message; - } - - else - StatusHelp.Text = messagesDict!.GetValueOrDefault(command.Message, aiDisabledText); - - if (aiEnabled) - StatusHelp.Foreground = aiEnabled ? Brushes.White : Brushes.Red; - }); - }; - _inferenceClient.Send(RemoteCommand.Create(CommandType.AIAvailabilityCheck)); - Editor.GetTimeFunc = () => TimeSpan.FromMilliseconds(_mediaPlayer.Time); MapMatcherComponent.Init(_appConfig, gpsMatcherService); } diff --git a/Azaion.Annotator/AnnotatorEventHandler.cs b/Azaion.Annotator/AnnotatorEventHandler.cs index f345f6f..5a0c4ec 100644 --- a/Azaion.Annotator/AnnotatorEventHandler.cs +++ b/Azaion.Annotator/AnnotatorEventHandler.cs @@ -12,6 +12,7 @@ using Azaion.Common.DTO.Config; using Azaion.Common.Events; using Azaion.Common.Extensions; using Azaion.Common.Services; +using Azaion.Common.Services.Inference; using GMap.NET; using GMap.NET.WindowsPresentation; using LibVLCSharp.Shared; @@ -43,7 +44,8 @@ public class AnnotatorEventHandler( INotificationHandler, INotificationHandler, INotificationHandler, - INotificationHandler + INotificationHandler, + INotificationHandler { private const int STEP = 20; private const int LARGE_STEP = 5000; @@ -472,4 +474,14 @@ public class AnnotatorEventHandler( map.SatelliteMap.Position = pointLatLon; map.SatelliteMap.ZoomAndCenterMarkers(null); } + + public async Task Handle(AIAvailabilityStatusEvent e, CancellationToken cancellationToken) + { + mainWindow.Dispatcher.Invoke(() => + { + logger.LogInformation(e.ToString()); + mainWindow.AIDetectBtn.IsEnabled = e.Status == AIAvailabilityEnum.Enabled; + mainWindow.StatusHelp.Text = e.ToString(); + }); + } } \ No newline at end of file diff --git a/Azaion.Common/DTO/AIAvailabilityStatus.cs b/Azaion.Common/DTO/AIAvailabilityStatus.cs new file mode 100644 index 0000000..b45f865 --- /dev/null +++ b/Azaion.Common/DTO/AIAvailabilityStatus.cs @@ -0,0 +1,33 @@ +using MediatR; +using MessagePack; + +namespace Azaion.Common.DTO; + +public enum AIAvailabilityEnum +{ + None = 0, + Downloading = 10, + Converting = 20, + Uploading = 30, + Enabled = 200, + Error = 500 +} + +[MessagePackObject] +public class AIAvailabilityStatusEvent : INotification +{ + [Key("s")] public AIAvailabilityEnum Status { get; set; } + [Key("m")] public string? ErrorMessage { get; set; } + + public override string ToString() => $"{StatusMessageDict.GetValueOrDefault(Status, "Помилка")} {ErrorMessage}"; + + private static readonly Dictionary StatusMessageDict = new() + { + { AIAvailabilityEnum.Downloading, "Йде завантаження AI для Вашої відеокарти" }, + { AIAvailabilityEnum.Converting, "Йде налаштування AI під Ваше залізо. (5-12 хвилин в залежності від моделі відеокарти, до 50 хв на старих GTX1650)" }, + { AIAvailabilityEnum.Uploading, "Йде зберігання AI" }, + { AIAvailabilityEnum.Enabled, "AI готовий для розпізнавання" }, + { AIAvailabilityEnum.Error, "Помилка під час налаштування AI" } + }; + +} diff --git a/Azaion.Common/Services/AnnotationService.cs b/Azaion.Common/Services/AnnotationService.cs index 190c396..d0381df 100644 --- a/Azaion.Common/Services/AnnotationService.cs +++ b/Azaion.Common/Services/AnnotationService.cs @@ -21,6 +21,7 @@ using RabbitMQ.Stream.Client.Reliable; namespace Azaion.Common.Services; // SHOULD BE ONLY ONE INSTANCE OF AnnotationService. Do not add ANY NotificationHandler to it! +// Queue consumer should be created only once. public class AnnotationService : IAnnotationService { private readonly IDbFactory _dbFactory; diff --git a/Azaion.Common/Services/GPSMatcherEventHandler.cs b/Azaion.Common/Services/GpsMatcher/GPSMatcherEventHandler.cs similarity index 100% rename from Azaion.Common/Services/GPSMatcherEventHandler.cs rename to Azaion.Common/Services/GpsMatcher/GPSMatcherEventHandler.cs diff --git a/Azaion.Common/Services/GPSMatcherEvents.cs b/Azaion.Common/Services/GpsMatcher/GPSMatcherEvents.cs similarity index 100% rename from Azaion.Common/Services/GPSMatcherEvents.cs rename to Azaion.Common/Services/GpsMatcher/GPSMatcherEvents.cs diff --git a/Azaion.Common/Services/GPSMatcherService.cs b/Azaion.Common/Services/GpsMatcher/GPSMatcherService.cs similarity index 100% rename from Azaion.Common/Services/GPSMatcherService.cs rename to Azaion.Common/Services/GpsMatcher/GPSMatcherService.cs diff --git a/Azaion.Common/Services/GpsMatcherClient.cs b/Azaion.Common/Services/GpsMatcher/GpsMatcherClient.cs similarity index 100% rename from Azaion.Common/Services/GpsMatcherClient.cs rename to Azaion.Common/Services/GpsMatcher/GpsMatcherClient.cs diff --git a/Azaion.Common/Services/InferenceClient.cs b/Azaion.Common/Services/Inference/InferenceClient.cs similarity index 75% rename from Azaion.Common/Services/InferenceClient.cs rename to Azaion.Common/Services/Inference/InferenceClient.cs index 7e59620..2e612b9 100644 --- a/Azaion.Common/Services/InferenceClient.cs +++ b/Azaion.Common/Services/Inference/InferenceClient.cs @@ -1,18 +1,17 @@ using System.Diagnostics; using System.Text; using Azaion.Common.DTO; +using MediatR; using MessagePack; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using NetMQ; using NetMQ.Sockets; -namespace Azaion.Common.Services; +namespace Azaion.Common.Services.Inference; public interface IInferenceClient : IDisposable { - event EventHandler? InferenceDataReceived; - event EventHandler? AIAvailabilityReceived; void Send(RemoteCommand create); void Stop(); } @@ -20,21 +19,22 @@ public interface IInferenceClient : IDisposable public class InferenceClient : IInferenceClient { private readonly ILogger _logger; - public event EventHandler? BytesReceived; - public event EventHandler? InferenceDataReceived; - public event EventHandler? AIAvailabilityReceived; private readonly DealerSocket _dealer = new(); private readonly NetMQPoller _poller = new(); private readonly Guid _clientId = Guid.NewGuid(); private readonly InferenceClientConfig _inferenceClientConfig; private readonly LoaderClientConfig _loaderClientConfig; + private readonly IMediator _mediator; - public InferenceClient(ILogger logger, IOptions inferenceConfig, IOptions loaderConfig) + public InferenceClient(ILogger logger, IOptions inferenceConfig, + IMediator mediator, + IOptions loaderConfig) { _logger = logger; _inferenceClientConfig = inferenceConfig.Value; _loaderClientConfig = loaderConfig.Value; + _mediator = mediator; Start(); } @@ -59,32 +59,31 @@ public class InferenceClient : IInferenceClient _dealer.Options.Identity = Encoding.UTF8.GetBytes(_clientId.ToString("N")); _dealer.Connect($"tcp://{_inferenceClientConfig.ZeroMqHost}:{_inferenceClientConfig.ZeroMqPort}"); - _dealer.ReceiveReady += (_, e) => ProcessClientCommand(e.Socket); + _dealer.ReceiveReady += async (_, e) => await ProcessClientCommand(e.Socket); _poller.Add(_dealer); _ = Task.Run(() => _poller.RunAsync()); } - private void ProcessClientCommand(NetMQSocket socket, CancellationToken ct = default) + private async Task ProcessClientCommand(NetMQSocket socket, CancellationToken ct = default) { while (socket.TryReceiveFrameBytes(TimeSpan.Zero, out var bytes)) { - if (bytes?.Length == 0) + if (bytes.Length == 0) continue; var remoteCommand = MessagePackSerializer.Deserialize(bytes, cancellationToken: ct); switch (remoteCommand.CommandType) { - case CommandType.DataBytes: - BytesReceived?.Invoke(this, remoteCommand); - break; case CommandType.InferenceData: - InferenceDataReceived?.Invoke(this, remoteCommand); + await _mediator.Publish(new InferenceDataEvent(remoteCommand), ct); break; case CommandType.AIAvailabilityResult: - AIAvailabilityReceived?.Invoke(this, remoteCommand); + var aiAvailabilityStatus = MessagePackSerializer.Deserialize(remoteCommand.Data, cancellationToken: ct); + await _mediator.Publish(aiAvailabilityStatus, ct); break; + default: + throw new ArgumentOutOfRangeException(); } - } } diff --git a/Azaion.Common/Services/Inference/InferenceService.cs b/Azaion.Common/Services/Inference/InferenceService.cs new file mode 100644 index 0000000..d94ae4d --- /dev/null +++ b/Azaion.Common/Services/Inference/InferenceService.cs @@ -0,0 +1,56 @@ +using Azaion.Common.DTO; +using Azaion.Common.DTO.Config; +using Azaion.Common.Extensions; +using Microsoft.Extensions.Options; + +namespace Azaion.Common.Services.Inference; + +public interface IInferenceService +{ + Task RunInference(List mediaPaths, CancellationToken ct = default); + CancellationTokenSource InferenceCancelTokenSource { get; set; } + void StopInference(); +} + +// SHOULD BE ONLY ONE INSTANCE OF InferenceService. Do not add ANY NotificationHandler to it! +// _inferenceCancelTokenSource should be created only once. +public class InferenceService : IInferenceService +{ + private readonly IInferenceClient _client; + private readonly IAzaionApi _azaionApi; + private readonly IOptions _aiConfigOptions; + public CancellationTokenSource InferenceCancelTokenSource { get; set; } = new(); + public CancellationTokenSource CheckAIAvailabilityTokenSource { get; set; } = new(); + + public InferenceService(IInferenceClient client, IAzaionApi azaionApi, IOptions aiConfigOptions) + { + _client = client; + _azaionApi = azaionApi; + _aiConfigOptions = aiConfigOptions; + } + + public async Task CheckAIAvailabilityStatus() + { + CheckAIAvailabilityTokenSource = new CancellationTokenSource(); + while (!CheckAIAvailabilityTokenSource.IsCancellationRequested) + { + _client.Send(RemoteCommand.Create(CommandType.AIAvailabilityCheck)); + await Task.Delay(10000, CheckAIAvailabilityTokenSource.Token); + } + } + + public async Task RunInference(List mediaPaths, CancellationToken ct = default) + { + InferenceCancelTokenSource = new CancellationTokenSource(); + _client.Send(RemoteCommand.Create(CommandType.Login, _azaionApi.Credentials)); + + var aiConfig = _aiConfigOptions.Value; + aiConfig.Paths = mediaPaths; + _client.Send(RemoteCommand.Create(CommandType.Inference, aiConfig)); + + using var combinedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(ct, InferenceCancelTokenSource.Token); + await combinedTokenSource.Token.AsTask(); + } + + public void StopInference() => _client.Stop(); +} \ No newline at end of file diff --git a/Azaion.Common/Services/Inference/InferenceServiceEventHandler.cs b/Azaion.Common/Services/Inference/InferenceServiceEventHandler.cs new file mode 100644 index 0000000..092edfb --- /dev/null +++ b/Azaion.Common/Services/Inference/InferenceServiceEventHandler.cs @@ -0,0 +1,43 @@ +using Azaion.Common.Database; +using Azaion.Common.DTO; +using Azaion.Common.Events; +using MediatR; +using MessagePack; +using Microsoft.Extensions.Logging; + +namespace Azaion.Common.Services.Inference; + +public class InferenceServiceEventHandler(IInferenceService inferenceService, + IAnnotationService annotationService, + IMediator mediator, + ILogger logger) : + INotificationHandler, + INotificationHandler +{ + public async Task Handle(InferenceDataEvent e, CancellationToken ct) + { + try + { + if (e.Command.Message == "DONE") + { + await inferenceService.InferenceCancelTokenSource.CancelAsync(); + return; + } + + var annImage = MessagePackSerializer.Deserialize(e.Command.Data, cancellationToken: ct); + var annotation = await annotationService.SaveAnnotation(annImage, ct); + await mediator.Publish(new AnnotationAddedEvent(annotation), ct); + } + catch (Exception ex) + { + logger.LogError(ex, ex.Message); + } + } + + public async Task Handle(AIAvailabilityStatusEvent e, CancellationToken ct) + { + + e.Status = AIAvailabilityEnum.Enabled; + + } +} \ No newline at end of file diff --git a/Azaion.Common/Services/Inference/InferenceServiceEvents.cs b/Azaion.Common/Services/Inference/InferenceServiceEvents.cs new file mode 100644 index 0000000..94aff5c --- /dev/null +++ b/Azaion.Common/Services/Inference/InferenceServiceEvents.cs @@ -0,0 +1,9 @@ +using Azaion.Common.DTO; +using MediatR; + +namespace Azaion.Common.Services.Inference; + +public class InferenceDataEvent(RemoteCommand command) : INotification +{ + public RemoteCommand Command { get; set; } = command; +} diff --git a/Azaion.Common/Services/InferenceService.cs b/Azaion.Common/Services/InferenceService.cs deleted file mode 100644 index 8ae9949..0000000 --- a/Azaion.Common/Services/InferenceService.cs +++ /dev/null @@ -1,82 +0,0 @@ -using Azaion.Common.Database; -using Azaion.Common.DTO; -using Azaion.Common.DTO.Config; -using Azaion.Common.Events; -using Azaion.Common.Extensions; -using MediatR; -using MessagePack; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; - -namespace Azaion.Common.Services; - -public interface IInferenceService -{ - Task RunInference(List mediaPaths, CancellationToken ct = default); - void StopInference(); -} - -public class InferenceService : IInferenceService -{ - private readonly IInferenceClient _client; - private readonly IAzaionApi _azaionApi; - private readonly IOptions _aiConfigOptions; - private readonly IAnnotationService _annotationService; - private readonly IMediator _mediator; - private CancellationTokenSource _inferenceCancelTokenSource = new(); - - public InferenceService( - ILogger logger, - IInferenceClient client, - IAzaionApi azaionApi, - IOptions aiConfigOptions, - IAnnotationService annotationService, - IMediator mediator) - { - _client = client; - _azaionApi = azaionApi; - _aiConfigOptions = aiConfigOptions; - _annotationService = annotationService; - _mediator = mediator; - - client.InferenceDataReceived += async (sender, command) => - { - try - { - if (command.Message == "DONE") - { - _inferenceCancelTokenSource?.Cancel(); - return; - } - - var annImage = MessagePackSerializer.Deserialize(command.Data); - await ProcessDetection(annImage); - } - catch (Exception e) - { - logger.LogError(e, e.Message); - } - }; - } - - private async Task ProcessDetection(AnnotationImage annotationImage, CancellationToken ct = default) - { - var annotation = await _annotationService.SaveAnnotation(annotationImage, ct); - await _mediator.Publish(new AnnotationAddedEvent(annotation), ct); - } - - public async Task RunInference(List mediaPaths, CancellationToken ct = default) - { - _inferenceCancelTokenSource = new CancellationTokenSource(); - _client.Send(RemoteCommand.Create(CommandType.Login, _azaionApi.Credentials)); - - var aiConfig = _aiConfigOptions.Value; - aiConfig.Paths = mediaPaths; - _client.Send(RemoteCommand.Create(CommandType.Inference, aiConfig)); - - using var combinedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(ct, _inferenceCancelTokenSource.Token); - await combinedTokenSource.Token.AsTask(); - } - - public void StopInference() => _client.Stop(); -} \ No newline at end of file diff --git a/Azaion.Inference/ai_availability_status.pxd b/Azaion.Inference/ai_availability_status.pxd new file mode 100644 index 0000000..5501113 --- /dev/null +++ b/Azaion.Inference/ai_availability_status.pxd @@ -0,0 +1,14 @@ +cdef enum AIAvailabilityEnum: + NONE = 0 + DOWNLOADING = 10 + CONVERTING = 20 + UPLOADING = 30 + ENABLED = 200 + ERROR = 500 + +cdef class AIAvailabilityStatus: + cdef AIAvailabilityEnum status + cdef str error_message + + cdef bytes serialize(self) + cdef set_status(self, AIAvailabilityEnum status, str error_message=*) \ No newline at end of file diff --git a/Azaion.Inference/ai_availability_status.pyx b/Azaion.Inference/ai_availability_status.pyx new file mode 100644 index 0000000..d467682 --- /dev/null +++ b/Azaion.Inference/ai_availability_status.pyx @@ -0,0 +1,36 @@ +cimport constants_inf +import msgpack + +AIStatus2Text = { + AIAvailabilityEnum.NONE: "None", + AIAvailabilityEnum.DOWNLOADING: "Downloading", + AIAvailabilityEnum.CONVERTING: "Converting", + AIAvailabilityEnum.UPLOADING: "Uploading", + AIAvailabilityEnum.ENABLED: "Enabled", + AIAvailabilityEnum.ERROR: "Error", +} + +cdef class AIAvailabilityStatus: + def __init__(self): + self.status = AIAvailabilityEnum.NONE + self.error_message = None + + def __str__(self): + status_text = AIStatus2Text.get(self.status, "Unknown") + error_text = self.error_message if self.error_message else "" + return f"{status_text} {error_text}" + + cdef bytes serialize(self): + return msgpack.packb({ + "s": self.status, + "m": self.error_message + }) + + cdef set_status(self, AIAvailabilityEnum status, str error_message=None): + self.status = status + self.error_message = error_message + if error_message is not None: + constants_inf.logerror(error_message) + else: + constants_inf.log(str(self)) + diff --git a/Azaion.Inference/build_inference.cmd b/Azaion.Inference/build_inference.cmd index a880c9b..dc99d3e 100644 --- a/Azaion.Inference/build_inference.cmd +++ b/Azaion.Inference/build_inference.cmd @@ -35,6 +35,7 @@ venv\Scripts\pyinstaller --name=azaion-inference ^ --collect-all jwt ^ --collect-all loguru ^ --hidden-import constants_inf ^ +--hidden-import ai_availability_status ^ --hidden-import file_data ^ --hidden-import remote_command_inf ^ --hidden-import remote_command_handler_inf ^ @@ -49,8 +50,9 @@ start.py robocopy "dist\azaion-inference\_internal" "..\dist-azaion\_internal" "ai_config.cp312-win_amd64.pyd" "annotation.cp312-win_amd64.pyd" robocopy "dist\azaion-inference\_internal" "..\dist-azaion\_internal" "constants_inf.cp312-win_amd64.pyd" "file_data.cp312-win_amd64.pyd" +robocopy "dist\azaion-inference\_internal" "..\dist-azaion\_internal" "ai_availability_status.pyd" robocopy "dist\azaion-inference\_internal" "..\dist-azaion\_internal" "remote_command_inf.cp312-win_amd64.pyd" "remote_command_handler_inf.cp312-win_amd64.pyd" -robocopy "dist\azaion-inference\_internal" "..\dist-azaion\_internal" "inference.cp312-win_amd64.pyd" "inference_engine.cp312-win_amd64.pyd" +robocopy "dist\azaion-inference\_internal" "..\dist-azaion\_internal" "inference.cp312-win_amd64.py=d" "inference_engine.cp312-win_amd64.pyd" robocopy "dist\azaion-inference\_internal" "..\dist-azaion\_internal" "loader_client.cp312-win_amd64.pyd" "tensorrt_engine.cp312-win_amd64.pyd" robocopy "dist\azaion-inference\_internal" "..\dist-azaion\_internal" "onnx_engine.cp312-win_amd64.pyd" "main_inference.cp312-win_amd64.pyd" diff --git a/Azaion.Inference/inference.pxd b/Azaion.Inference/inference.pxd index 5799b22..c540595 100644 --- a/Azaion.Inference/inference.pxd +++ b/Azaion.Inference/inference.pxd @@ -1,3 +1,4 @@ +from ai_availability_status cimport AIAvailabilityStatus from remote_command_inf cimport RemoteCommand from annotation cimport Annotation, Detection from ai_config cimport AIRecognitionConfig @@ -12,6 +13,7 @@ cdef class Inference: cdef dict[str, list(Detection)] _tile_detections cdef AIRecognitionConfig ai_config cdef bint stop_signal + cdef AIAvailabilityStatus ai_availability_status cdef str model_input cdef int model_width @@ -19,7 +21,7 @@ cdef class Inference: cdef int tile_width cdef int tile_height - cdef build_tensor_engine(self, object updater_callback) + cdef bytes get_onnx_engine_bytes(self) cdef init_ai(self) cdef bint is_building_engine cdef bint is_video(self, str filepath) diff --git a/Azaion.Inference/inference.pyx b/Azaion.Inference/inference.pyx index 467227b..1bdd3f5 100644 --- a/Azaion.Inference/inference.pyx +++ b/Azaion.Inference/inference.pyx @@ -5,6 +5,8 @@ from pathlib import Path import cv2 import numpy as np cimport constants_inf + +from ai_availability_status cimport AIAvailabilityEnum, AIAvailabilityStatus from remote_command_inf cimport RemoteCommand from annotation cimport Detection, Annotation from ai_config cimport AIRecognitionConfig @@ -60,67 +62,59 @@ cdef class Inference: self.tile_height = 0 self.engine = None self.is_building_engine = False + self.ai_availability_status = AIAvailabilityStatus() + self.init_ai() - cdef build_tensor_engine(self, object updater_callback): - if tensor_gpu_index == -1: - return - - try: - engine_filename = TensorRTEngine.get_engine_filename(0) - models_dir = constants_inf.MODELS_FOLDER - - self.is_building_engine = True - updater_callback('downloading') - - res = self.loader_client.load_big_small_resource(engine_filename, models_dir) - if res.err is None: - constants_inf.log('tensor rt engine is here, no need to build') - self.is_building_engine = False - updater_callback('enabled') - return - - constants_inf.logerror(res.err) - # time.sleep(8) # prevent simultaneously loading dll and models - updater_callback('converting') - constants_inf.log('try to load onnx') - res = self.loader_client.load_big_small_resource(constants_inf.AI_ONNX_MODEL_FILE, models_dir) - if res.err is not None: - updater_callback(f'Error. {res.err}') - model_bytes = TensorRTEngine.convert_from_onnx(res.data) - updater_callback('uploading') - res = self.loader_client.upload_big_small_resource(model_bytes, engine_filename, models_dir) - if res.err is not None: - updater_callback(f'Error. {res.err}') - constants_inf.log(f'uploaded {engine_filename} to CDN and API') - self.is_building_engine = False - updater_callback('enabled') - except Exception as e: - updater_callback(f'Error. {str(e)}') + cdef bytes get_onnx_engine_bytes(self): + models_dir = constants_inf.MODELS_FOLDER + self.ai_availability_status.set_status(AIAvailabilityEnum.DOWNLOADING) + res = self.loader_client.load_big_small_resource(constants_inf.AI_ONNX_MODEL_FILE, models_dir) + if res.err is not None: + raise Exception(res.err) + return res.data cdef init_ai(self): - if self.engine is not None: - return - - models_dir = constants_inf.MODELS_FOLDER - if tensor_gpu_index > -1: + constants_inf.log( 'init AI...') + try: while self.is_building_engine: time.sleep(1) - engine_filename = TensorRTEngine.get_engine_filename(0) + if self.engine is not None: + return + + self.is_building_engine = True + models_dir = constants_inf.MODELS_FOLDER + if tensor_gpu_index > -1: + try: + engine_filename = TensorRTEngine.get_engine_filename(0) + self.ai_availability_status.set_status(AIAvailabilityEnum.DOWNLOADING) + res = self.loader_client.load_big_small_resource(engine_filename, models_dir) + if res.err is not None: + raise Exception(res.err) + self.engine = TensorRTEngine(res.data) + self.ai_availability_status.set_status(AIAvailabilityEnum.ENABLED) + except Exception as e: + self.ai_availability_status.set_status(AIAvailabilityEnum.ERROR, str(e)) + onnx_engine_bytes = self.get_onnx_engine_bytes() + self.ai_availability_status.set_status(AIAvailabilityEnum.CONVERTING) + model_bytes = TensorRTEngine.convert_from_onnx(res.data) + self.ai_availability_status.set_status(AIAvailabilityEnum.UPLOADING) + res = self.loader_client.upload_big_small_resource(model_bytes, engine_filename, models_dir) + if res.err is not None: + self.ai_availability_status.set_status(AIAvailabilityEnum.ERROR, res.err) + self.ai_availability_status.set_status(AIAvailabilityEnum.ENABLED) + else: + self.engine = OnnxEngine(self.get_onnx_engine_bytes()) + self.is_building_engine = False + + self.model_height, self.model_width = self.engine.get_input_shape() + #todo: temporarily, send it from the client + self.tile_width = 550 + self.tile_height = 550 + except Exception as e: + self.ai_availability_status.set_status(AIAvailabilityEnum.ERROR, str(e)) + self.is_building_engine = False - res = self.loader_client.load_big_small_resource(engine_filename, models_dir) - if res.err is not None: - raise Exception(res.err) - self.engine = TensorRTEngine(res.data) - else: - res = self.loader_client.load_big_small_resource(constants_inf.AI_ONNX_MODEL_FILE, models_dir) - if res.err is not None: - raise Exception(res.err) - self.engine = OnnxEngine(res.data) - self.model_height, self.model_width = self.engine.get_input_shape() - #todo: temporarily, send it from the client - self.tile_width = 550 - self.tile_height = 550 cdef preprocess(self, frames): blobs = [cv2.dnn.blobFromImage(frame, diff --git a/Azaion.Inference/main_inference.pyx b/Azaion.Inference/main_inference.pyx index 6978fd8..6d27280 100644 --- a/Azaion.Inference/main_inference.pyx +++ b/Azaion.Inference/main_inference.pyx @@ -44,8 +44,8 @@ cdef class CommandProcessor: if command.command_type == CommandType.INFERENCE: self.inference_queue.put(command) elif command.command_type == CommandType.AI_AVAILABILITY_CHECK: - self.inference.build_tensor_engine(lambda status: self.remote_handler.send(command.client_id, - RemoteCommand(CommandType.AI_AVAILABILITY_RESULT, None, status).serialize())) + status = self.inference.ai_availability_status.serialize() + self.remote_handler.send(command.client_id, RemoteCommand(CommandType.AI_AVAILABILITY_RESULT, status).serialize()) elif command.command_type == CommandType.STOP_INFERENCE: self.inference.stop() elif command.command_type == CommandType.EXIT: diff --git a/Azaion.Inference/setup.py b/Azaion.Inference/setup.py index 54901f1..b7aebbf 100644 --- a/Azaion.Inference/setup.py +++ b/Azaion.Inference/setup.py @@ -14,6 +14,7 @@ trace_line = False extensions = [ Extension('constants_inf', ['constants_inf.pyx'], **debug_args), + Extension('ai_availability_status', ['ai_availability_status.pyx'], **debug_args), Extension('file_data', ['file_data.pyx'], **debug_args), Extension('remote_command_inf', ['remote_command_inf.pyx'], **debug_args), Extension('remote_command_handler_inf', ['remote_command_handler_inf.pyx'], **debug_args), diff --git a/Azaion.Suite/App.xaml.cs b/Azaion.Suite/App.xaml.cs index 0f7b671..865d5c9 100644 --- a/Azaion.Suite/App.xaml.cs +++ b/Azaion.Suite/App.xaml.cs @@ -11,6 +11,7 @@ using Azaion.Common.DTO.Config; using Azaion.Common.Events; using Azaion.Common.Extensions; using Azaion.Common.Services; +using Azaion.Common.Services.Inference; using Azaion.Dataset; using CommandLine; using LibVLCSharp.Shared; diff --git a/Azaion.Suite/MainSuite.xaml.cs b/Azaion.Suite/MainSuite.xaml.cs index 912c66d..d0272d5 100644 --- a/Azaion.Suite/MainSuite.xaml.cs +++ b/Azaion.Suite/MainSuite.xaml.cs @@ -6,8 +6,8 @@ using System.Windows.Media; using Azaion.Common.Database; using Azaion.Common.DTO; using Azaion.Common.DTO.Config; -using Azaion.Common.Extensions; using Azaion.Common.Services; +using Azaion.Common.Services.Inference; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; using SharpVectors.Converters; diff --git a/Azaion.Suite/config.json b/Azaion.Suite/config.json index 748dbb9..711fdb7 100644 --- a/Azaion.Suite/config.json +++ b/Azaion.Suite/config.json @@ -1,12 +1,12 @@ { "LoaderClientConfig": { "ZeroMqHost": "127.0.0.1", - "ZeroMqPort": 5024, + "ZeroMqPort": 5025, "ApiUrl": "https://api.azaion.com" }, "InferenceClientConfig": { "ZeroMqHost": "127.0.0.1", - "ZeroMqPort": 5126, + "ZeroMqPort": 5127, "ApiUrl": "https://api.azaion.com" }, "GpsDeniedClientConfig": {