diff --git a/.gitignore b/.gitignore index 972b8d6..528133f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ .idea bin obj +*.dll +*.exe +*.log .vs *.DotSettings* *.user diff --git a/Azaion.Annotator/Annotator.xaml b/Azaion.Annotator/Annotator.xaml index 53bd029..f77d1fa 100644 --- a/Azaion.Annotator/Annotator.xaml +++ b/Azaion.Annotator/Annotator.xaml @@ -500,7 +500,7 @@ Padding="2" Width="25" Height="25" ToolTip="Розпізнати за допомогою AI. Клавіша: [R]" Background="Black" BorderBrush="Black" - Click="AutoDetect"> + Click="AIDetectBtn_OnClick"> + { + { "disabled", aiDisabledText }, + { "downloading", "Будь ласка зачекайте, йде завантаження AI для Вашої відеокарти" }, + { "converting", "Будь ласка зачекайте, йде налаштування AI під Ваше залізо. (5-12 хвилин в залежності від моделі відеокарти, до 50 хв на старих GTX1650)" }, + { "uploading", "Будь ласка зачекайте, йде зберігання" }, + { "enabled", "AI готовий для розпізнавання" } + }; + StatusHelp.Text = messagesDict!.GetValueOrDefault(command.Message, aiDisabledText); + if (aiEnabled) + StatusHelp.Foreground = aiEnabled ? Brushes.White : Brushes.Red; + }); + }; + _inferenceClient.Send(RemoteCommand.Create(CommandType.AIAvailabilityCheck)); Editor.GetTimeFunc = () => TimeSpan.FromMilliseconds(_mediaPlayer.Time); MapMatcherComponent.Init(_appConfig, _gpsMatcherService); @@ -126,9 +149,6 @@ public partial class Annotator TbFolder.Text = _appConfig.DirectoriesConfig.VideosDirectory; LvClasses.Init(_appConfig.AnnotationConfig.DetectionClasses); - - if (LvFiles.Items.IsEmpty) - BlinkHelp(HelpTexts.HelpTextsDict[HelpTextEnum.Initial]); } public void BlinkHelp(string helpText, int times = 2) @@ -175,8 +195,6 @@ public partial class Annotator LvFiles.MouseDoubleClick += async (_, _) => { - if (IsInferenceNow) - FollowAI = false; await _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.Play)); }; @@ -238,8 +256,6 @@ public partial class Annotator public void OpenAnnotationResult(AnnotationResult res) { - if (IsInferenceNow) - FollowAI = false; _mediaPlayer.SetPause(true); Editor.RemoveAllAnns(); _mediaPlayer.Time = (long)res.Annotation.Time.TotalMilliseconds; @@ -325,6 +341,10 @@ public partial class Annotator //Add manually public void AddAnnotation(Annotation annotation) { + var mediaInfo = (MediaFileInfo)LvFiles.SelectedItem; + if ((mediaInfo?.FName ?? "") != annotation.OriginalMediaName) + return; + var time = annotation.Time; var previousAnnotations = TimedAnnotations.Query(time); TimedAnnotations.Remove(previousAnnotations); @@ -342,10 +362,8 @@ public partial class Annotator _logger.LogError(e, e.Message); throw; } - } - var dict = _formState.AnnotationResults .Select((x, i) => new { x.Annotation.Time, Index = i }) .ToDictionary(x => x.Time, x => x.Index); @@ -399,11 +417,8 @@ public partial class Annotator mediaFile.HasAnnotations = labelsDict.ContainsKey(mediaFile.FName); AllMediaFiles = new ObservableCollection(allFiles); + MediaFilesDict = AllMediaFiles.ToDictionary(x => x.FName); LvFiles.ItemsSource = AllMediaFiles; - - BlinkHelp(AllMediaFiles.Count == 0 - ? HelpTexts.HelpTextsDict[HelpTextEnum.Initial] - : HelpTexts.HelpTextsDict[HelpTextEnum.PlayVideo]); DataContext = this; } @@ -463,14 +478,13 @@ public partial class Annotator private void TbFilter_OnTextChanged(object sender, TextChangedEventArgs e) { FilteredMediaFiles = new ObservableCollection(AllMediaFiles.Where(x => x.Name.ToLower().Contains(TbFilter.Text.ToLower())).ToList()); + MediaFilesDict = FilteredMediaFiles.ToDictionary(x => x.FName); LvFiles.ItemsSource = FilteredMediaFiles; LvFiles.ItemsSource = FilteredMediaFiles; } private void PlayClick(object sender, RoutedEventArgs e) { - if (IsInferenceNow) - FollowAI = false; _mediator.Publish(new AnnotatorControlEvent(_mediaPlayer.CanPause ? PlaybackControlEnum.Pause : PlaybackControlEnum.Play)); } @@ -501,13 +515,22 @@ public partial class Annotator LvFilesContextMenu.DataContext = listItem!.DataContext; } - public void AutoDetect(object sender, RoutedEventArgs e) + private async void AIDetectBtn_OnClick(object sender, RoutedEventArgs e) + { + try + { + await AutoDetect(); + } + catch (Exception ex) + { + _logger.LogError(ex, ex.Message); + } + } + + public async Task AutoDetect() { if (IsInferenceNow) - { - FollowAI = true; return; - } if (LvFiles.Items.IsEmpty) return; @@ -517,96 +540,22 @@ public partial class Annotator Dispatcher.Invoke(() => Editor.ResetBackground()); IsInferenceNow = true; - FollowAI = true; + AIDetectBtn.IsEnabled = false; + DetectionCancellationSource = new CancellationTokenSource(); - var detectToken = DetectionCancellationSource.Token; - _ = Task.Run(async () => - { - while (!detectToken.IsCancellationRequested) - { - var files = new List(); - await Dispatcher.Invoke(async () => - { - //Take all medias - files = (LvFiles.ItemsSource as IEnumerable)?.Skip(LvFiles.SelectedIndex) - //.Where(x => !x.HasAnnotations) - .Take(Constants.DETECTION_BATCH_SIZE) - .Select(x => x.Path) - .ToList() ?? []; - if (files.Count != 0) - { - await _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.Play), detectToken); - await ReloadAnnotations(); - } - }); - if (files.Count == 0) - break; - await _inferenceService.RunInference(files, async annotationImage => await ProcessDetection(annotationImage, detectToken), detectToken); + var files = (FilteredMediaFiles.Count == 0 ? AllMediaFiles : FilteredMediaFiles) + .Skip(LvFiles.SelectedIndex) + .Select(x => x.Path) + .ToList(); + if (files.Count == 0) + return; - Dispatcher.Invoke(() => - { - if (LvFiles.SelectedIndex + files.Count >= LvFiles.Items.Count) - DetectionCancellationSource.Cancel(); - LvFiles.SelectedIndex += files.Count; - }); - } - Dispatcher.Invoke(() => - { - LvFiles.Items.Refresh(); - IsInferenceNow = false; - FollowAI = false; - }); - }); - } + await _inferenceService.RunInference(files, DetectionCancellationSource.Token); - private async Task ProcessDetection(AnnotationImage annotationImage, CancellationToken ct = default) - { - await Dispatcher.Invoke(async () => - { - try - { - var annotation = await _annotationService.SaveAnnotation(annotationImage, ct); - if (annotation.OriginalMediaName != _formState.CurrentMedia?.FName) - { - var nextFile = (LvFiles.ItemsSource as IEnumerable)? - .Select((info, i) => new - { - MediaInfo = info, - Index = i - }) - .FirstOrDefault(x => x.MediaInfo.FName == annotation.OriginalMediaName); - if (nextFile != null) - { - LvFiles.SelectedIndex = nextFile.Index; - await _mediator.Publish(new AnnotatorControlEvent(PlaybackControlEnum.Play), ct); - } - } - - AddAnnotation(annotation); - - if (FollowAI) - SeekTo(annotationImage.Milliseconds, false); - - var log = string.Join(Environment.NewLine, annotation.Detections.Select(det => - $"{_appConfig.AnnotationConfig.DetectionClassesDict[det.ClassNumber].Name}: " + - $"xy=({det.CenterX:F2},{det.CenterY:F2}), " + - $"size=({det.Width:F2}, {det.Height:F2}), " + - $"conf: {det.Confidence*100:F0}%")); - - Dispatcher.Invoke(() => - { - if (_formState.CurrentMedia != null) - _formState.CurrentMedia.HasAnnotations = true; - LvFiles.Items.Refresh(); - StatusHelp.Text = log; - }); - } - catch (Exception e) - { - _logger.LogError(e, e.Message); - } - }); + LvFiles.Items.Refresh(); + IsInferenceNow = false; + AIDetectBtn.IsEnabled = true; } private void SwitchGpsPanel(object sender, RoutedEventArgs e) diff --git a/Azaion.Annotator/AnnotatorEventHandler.cs b/Azaion.Annotator/AnnotatorEventHandler.cs index e0961e8..56c3f4c 100644 --- a/Azaion.Annotator/AnnotatorEventHandler.cs +++ b/Azaion.Annotator/AnnotatorEventHandler.cs @@ -4,6 +4,7 @@ using System.Windows.Input; using Azaion.Annotator.DTO; using Azaion.Common; using Azaion.Common.DTO; +using Azaion.Common.DTO.Config; using Azaion.Common.Events; using Azaion.Common.Extensions; using Azaion.Common.Services; @@ -21,16 +22,18 @@ public class AnnotatorEventHandler( MediaPlayer mediaPlayer, Annotator mainWindow, FormState formState, - AnnotationService annotationService, + IAnnotationService annotationService, ILogger logger, IOptions dirConfig, + IOptions annotationConfig, IInferenceService inferenceService) : INotificationHandler, INotificationHandler, INotificationHandler, INotificationHandler, - INotificationHandler + INotificationHandler, + INotificationHandler { private const int STEP = 20; private const int LARGE_STEP = 5000; @@ -81,7 +84,7 @@ public class AnnotatorEventHandler( await ControlPlayback(value, cancellationToken); if (key == Key.R) - mainWindow.AutoDetect(null!, null!); + await mainWindow.AutoDetect(); #region Volume switch (key) @@ -128,10 +131,6 @@ public class AnnotatorEventHandler( break; case PlaybackControlEnum.Pause: mediaPlayer.Pause(); - if (mainWindow.IsInferenceNow) - mainWindow.FollowAI = false; - if (!mediaPlayer.IsPlaying) - mainWindow.BlinkHelp(HelpTexts.HelpTextsDict[HelpTextEnum.AnnotationHelp]); if (formState.BackgroundTime.HasValue) { @@ -225,7 +224,6 @@ public class AnnotatorEventHandler( await Task.Delay(100, ct); mediaPlayer.Stop(); mainWindow.Title = $"Azaion Annotator - {mediaInfo.Name}"; - mainWindow.BlinkHelp(HelpTexts.HelpTextsDict[HelpTextEnum.PauseForAnnotations]); mediaPlayer.Play(new Media(libVLC, mediaInfo.Path)); if (formState.CurrentMedia.MediaType == MediaTypes.Image) mediaPlayer.SetPause(true); @@ -301,4 +299,28 @@ public class AnnotatorEventHandler( } await Task.CompletedTask; } + + public Task Handle(AnnotationAddedEvent e, CancellationToken cancellationToken) + { + mainWindow.Dispatcher.Invoke(() => + { + mainWindow.AddAnnotation(e.Annotation); + + var log = string.Join(Environment.NewLine, e.Annotation.Detections.Select(det => + $"Розпізнавання: {annotationConfig.Value.DetectionClassesDict[det.ClassNumber].ShortName}: " + + $"xy=({det.CenterX:F2},{det.CenterY:F2}), " + + $"розмір=({det.Width:F2}, {det.Height:F2}), " + + $"conf: {det.Confidence*100:F0}%")); + + mainWindow.LvFiles.Items.Refresh(); + + var media = mainWindow.MediaFilesDict.GetValueOrDefault(e.Annotation.OriginalMediaName); + if (media != null) + media.HasAnnotations = true; + + mainWindow.LvFiles.Items.Refresh(); + mainWindow.StatusHelp.Text = log; + }); + return Task.CompletedTask; + } } diff --git a/Azaion.Common/Constants.cs b/Azaion.Common/Constants.cs index 556017b..84a9927 100644 --- a/Azaion.Common/Constants.cs +++ b/Azaion.Common/Constants.cs @@ -80,7 +80,6 @@ public class Constants public const double TRACKING_INTERSECTION_THRESHOLD = 0.8; public const int DEFAULT_FRAME_PERIOD_RECOGNITION = 4; - public const int DETECTION_BATCH_SIZE = 4; # endregion AIRecognitionConfig #region Thumbnails diff --git a/Azaion.Common/Controls/DetectionClasses.xaml b/Azaion.Common/Controls/DetectionClasses.xaml index 47951a8..a7b9925 100644 --- a/Azaion.Common/Controls/DetectionClasses.xaml +++ b/Azaion.Common/Controls/DetectionClasses.xaml @@ -52,6 +52,7 @@ CanUserResizeColumns="False" SelectionChanged="DetectionDataGrid_SelectionChanged" x:FieldModifier="public" + PreviewKeyDown="OnKeyBanActivity" > diff --git a/Azaion.Common/Controls/DetectionClasses.xaml.cs b/Azaion.Common/Controls/DetectionClasses.xaml.cs index d704fa5..7b2500a 100644 --- a/Azaion.Common/Controls/DetectionClasses.xaml.cs +++ b/Azaion.Common/Controls/DetectionClasses.xaml.cs @@ -1,8 +1,10 @@ using System.Collections.ObjectModel; using System.Windows; using System.Windows.Controls; +using System.Windows.Input; using Azaion.Common.DTO; using Azaion.Common.Extensions; +using Azaion.CommonSecurity.DTO; namespace Azaion.Common.Controls; @@ -86,4 +88,11 @@ public partial class DetectionClasses { DetectionDataGrid.SelectedIndex = keyNumber; } + + private void OnKeyBanActivity(object sender, KeyEventArgs e) + { + if (e.Key.In(Key.Enter, Key.Down, Key.Up, Key.PageDown, Key.PageUp)) + e.Handled = true; + } + } \ No newline at end of file diff --git a/Azaion.Common/Events/AnnotationsDeletedEvent.cs b/Azaion.Common/Events/AnnotationsDeletedEvent.cs index 1e815f2..0b93a76 100644 --- a/Azaion.Common/Events/AnnotationsDeletedEvent.cs +++ b/Azaion.Common/Events/AnnotationsDeletedEvent.cs @@ -6,4 +6,9 @@ namespace Azaion.Common.Events; public class AnnotationsDeletedEvent(List annotations) : INotification { public List Annotations { get; set; } = annotations; +} + +public class AnnotationAddedEvent(Annotation annotation) : INotification +{ + public Annotation Annotation { get; set; } = annotation; } \ No newline at end of file diff --git a/Azaion.Common/Events/LoadErrorEvent.cs b/Azaion.Common/Events/LoadErrorEvent.cs new file mode 100644 index 0000000..051827c --- /dev/null +++ b/Azaion.Common/Events/LoadErrorEvent.cs @@ -0,0 +1,8 @@ +using MediatR; + +namespace Azaion.Common.Events; + +public class LoadErrorEvent(string error) : INotification +{ + public string Error { get; set; } = error; +} \ No newline at end of file diff --git a/Azaion.Common/Extensions/CancellationTokenExtensions.cs b/Azaion.Common/Extensions/CancellationTokenExtensions.cs new file mode 100644 index 0000000..fbde775 --- /dev/null +++ b/Azaion.Common/Extensions/CancellationTokenExtensions.cs @@ -0,0 +1,30 @@ +namespace Azaion.Common.Extensions; + +public static class CancellationTokenExtensions +{ + public static void WaitForCancel(this CancellationToken token, TimeSpan timeout) + { + try + { + Task.Delay(timeout, token).Wait(token); + } + catch (OperationCanceledException) + { + //Don't need to catch exception, need only return from the waiting + } + } + + public static Task AsTask(this CancellationToken cancellationToken) + { + if (!cancellationToken.CanBeCanceled) + return new TaskCompletionSource().Task; + + if (cancellationToken.IsCancellationRequested) + return Task.FromCanceled(cancellationToken); + + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var registration = cancellationToken.Register(() => tcs.TrySetResult(true)); + tcs.Task.ContinueWith(_ => registration.Dispose(), TaskScheduler.Default); + return tcs.Task; + } +} \ No newline at end of file diff --git a/Azaion.Common/Extensions/ThrottleExtensions.cs b/Azaion.Common/Extensions/ThrottleExtensions.cs index 219374f..b0c46c2 100644 --- a/Azaion.Common/Extensions/ThrottleExtensions.cs +++ b/Azaion.Common/Extensions/ThrottleExtensions.cs @@ -54,7 +54,6 @@ public static class ThrottleExt finally { await Task.Delay(interval); - lock (state.StateLock) { if (state.CallScheduledDuringCooldown) diff --git a/Azaion.Common/Services/AnnotationService.cs b/Azaion.Common/Services/AnnotationService.cs index af769fe..4447584 100644 --- a/Azaion.Common/Services/AnnotationService.cs +++ b/Azaion.Common/Services/AnnotationService.cs @@ -20,7 +20,7 @@ using RabbitMQ.Stream.Client.Reliable; namespace Azaion.Common.Services; -public class AnnotationService : INotificationHandler +public class AnnotationService : IAnnotationService, INotificationHandler { private readonly IDbFactory _dbFactory; private readonly FailsafeAnnotationsProducer _producer; @@ -124,11 +124,17 @@ public class AnnotationService : INotificationHandler var fName = originalMediaName.ToTimeName(time); var annotation = await _dbFactory.Run(async db => { - var ann = await db.Annotations.FirstOrDefaultAsync(x => x.Name == fName, token: token); + var ann = await db.Annotations + .LoadWith(x => x.Detections) + .FirstOrDefaultAsync(x => x.Name == fName, token: token); + status = userRole.IsValidator() && source == SourceEnum.Manual ? AnnotationStatus.Validated : AnnotationStatus.Created; + if (fromQueue && ann is { AnnotationStatus: AnnotationStatus.Validated }) + return ann; + await db.Detections.DeleteAsync(x => x.AnnotationName == fName, token: token); if (ann != null) @@ -164,6 +170,9 @@ public class AnnotationService : INotificationHandler return ann; }); + if (fromQueue && annotation is { AnnotationStatus: AnnotationStatus.Validated }) + return annotation; + if (stream != null) { var img = System.Drawing.Image.FromStream(stream); @@ -219,4 +228,12 @@ public class AnnotationService : INotificationHandler File.Delete(annotation.ThumbPath); } } +} + +public interface IAnnotationService +{ + Task SaveAnnotation(AnnotationImage a, CancellationToken ct = default); + Task SaveAnnotation(string originalMediaName, TimeSpan time, List detections, Stream? stream = null, CancellationToken token = default); + Task ValidateAnnotations(List annotations, CancellationToken token = default); + } \ No newline at end of file diff --git a/Azaion.Common/Services/GPSMatcherService.cs b/Azaion.Common/Services/GPSMatcherService.cs index 9883c5a..f633caf 100644 --- a/Azaion.Common/Services/GPSMatcherService.cs +++ b/Azaion.Common/Services/GPSMatcherService.cs @@ -17,7 +17,7 @@ public interface IGpsMatcherService public class GpsMatcherService(IGpsMatcherClient gpsMatcherClient, ISatelliteDownloader satelliteTileDownloader, IOptions dirConfig) : IGpsMatcherService { private const int ZOOM_LEVEL = 18; - private const int POINTS_COUNT = 5; + private const int POINTS_COUNT = 10; private const int DISTANCE_BETWEEN_POINTS_M = 100; private const double SATELLITE_RADIUS_M = DISTANCE_BETWEEN_POINTS_M * (POINTS_COUNT + 1); @@ -41,7 +41,7 @@ public class GpsMatcherService(IGpsMatcherClient gpsMatcherClient, ISatelliteDow var indexOffset = 0; while (routeFiles.Any()) { - await satelliteTileDownloader.GetTiles(currentLat, currentLon, SATELLITE_RADIUS_M, ZOOM_LEVEL, detectToken); + //await satelliteTileDownloader.GetTiles(currentLat, currentLon, SATELLITE_RADIUS_M, ZOOM_LEVEL, detectToken); gpsMatcherClient.StartMatching(new StartMatchingEvent { ImagesCount = POINTS_COUNT, diff --git a/Azaion.Common/Services/GpsMatcherClient.cs b/Azaion.Common/Services/GpsMatcherClient.cs index cb8e225..12db99d 100644 --- a/Azaion.Common/Services/GpsMatcherClient.cs +++ b/Azaion.Common/Services/GpsMatcherClient.cs @@ -23,13 +23,12 @@ public class StartMatchingEvent public int ImagesCount { get; set; } public double Latitude { get; set; } public double Longitude { get; set; } - public string ProcessingType { get; set; } = "cuda"; public int Altitude { get; set; } = 400; public double CameraSensorWidth { get; set; } = 23.5; public double CameraFocalLength { get; set; } = 24; public override string ToString() => - $"{RouteDir},{SatelliteImagesDir},{ImagesCount},{Latitude},{Longitude},{ProcessingType},{Altitude},{CameraSensorWidth},{CameraFocalLength}"; + $"{RouteDir},{SatelliteImagesDir},{ImagesCount},{Latitude},{Longitude},{Altitude},{CameraSensorWidth},{CameraFocalLength}"; } public class GpsMatcherClient : IGpsMatcherClient diff --git a/Azaion.Common/Services/InferenceClient.cs b/Azaion.Common/Services/InferenceClient.cs new file mode 100644 index 0000000..6341d1a --- /dev/null +++ b/Azaion.Common/Services/InferenceClient.cs @@ -0,0 +1,150 @@ +using System.Diagnostics; +using System.IO; +using System.Text; +using Azaion.Common.Database; +using Azaion.Common.Extensions; +using Azaion.CommonSecurity; +using Azaion.CommonSecurity.DTO; +using Azaion.CommonSecurity.DTO.Commands; +using Azaion.CommonSecurity.Exceptions; +using Azaion.CommonSecurity.Services; +using MessagePack; +using Microsoft.Extensions.Options; +using NetMQ; +using NetMQ.Sockets; + +namespace Azaion.Common.Services; + +public interface IInferenceClient : IDisposable +{ + event EventHandler BytesReceived; + event EventHandler? InferenceDataReceived; + event EventHandler? AIAvailabilityReceived; + void Send(RemoteCommand create); + void Stop(); +} + +public class InferenceClient : IInferenceClient, IResourceLoader +{ + private CancellationTokenSource _waitFileCancelSource = new(); + + public event EventHandler? BytesReceived; + public event EventHandler? InferenceDataReceived; + public event EventHandler? AIAvailabilityReceived; + + private readonly DealerSocket _dealer = new(); + private readonly NetMQPoller _poller = new(); + private readonly Guid _clientId = Guid.NewGuid(); + private readonly InferenceClientConfig _inferenceClientConfig; + + public InferenceClient(IOptions config, CancellationToken ct) + { + _inferenceClientConfig = config.Value; + Start(ct); + } + + private void Start(CancellationToken ct = default) + { + try + { + using var process = new Process(); + process.StartInfo = new ProcessStartInfo + { + FileName = SecurityConstants.EXTERNAL_INFERENCE_PATH, + Arguments = $"--port {_inferenceClientConfig.ZeroMqPort} --api {_inferenceClientConfig.ApiUrl}", + //RedirectStandardOutput = true, + //RedirectStandardError = true, + //CreateNoWindow = true + }; + + process.OutputDataReceived += (_, e) => { if (e.Data != null) Console.WriteLine(e.Data); }; + process.ErrorDataReceived += (_, e) => { if (e.Data != null) Console.WriteLine(e.Data); }; + //process.Start(); + } + catch (Exception e) + { + Console.WriteLine(e); + //throw; + } + + _dealer.Options.Identity = Encoding.UTF8.GetBytes(_clientId.ToString("N")); + _dealer.Connect($"tcp://{_inferenceClientConfig.ZeroMqHost}:{_inferenceClientConfig.ZeroMqPort}"); + + _dealer.ReceiveReady += (_, e) => ProcessClientCommand(e.Socket, ct); + _poller.Add(_dealer); + _ = Task.Run(() => _poller.RunAsync(), ct); + } + + private void ProcessClientCommand(NetMQSocket socket, CancellationToken ct = default) + { + while (socket.TryReceiveFrameBytes(TimeSpan.Zero, out var bytes)) + { + if (bytes?.Length == 0) + continue; + + var remoteCommand = MessagePackSerializer.Deserialize(bytes, cancellationToken: ct); + switch (remoteCommand.CommandType) + { + case CommandType.DataBytes: + BytesReceived?.Invoke(this, remoteCommand); + break; + case CommandType.InferenceData: + InferenceDataReceived?.Invoke(this, remoteCommand); + break; + case CommandType.AIAvailabilityResult: + AIAvailabilityReceived?.Invoke(this, remoteCommand); + break; + } + + } + } + + public void Stop() + { + + } + + public void Send(RemoteCommand command) + { + _dealer.SendFrame(MessagePackSerializer.Serialize(command)); + } + + public MemoryStream LoadFile(string fileName, string? folder = null, TimeSpan? timeout = null) + { + //TODO: Bad solution, look for better implementation + byte[] bytes = []; + Exception? exception = null; + _waitFileCancelSource = new CancellationTokenSource(); + Send(RemoteCommand.Create(CommandType.Load, new LoadFileData(fileName, folder))); + BytesReceived += OnBytesReceived; + + void OnBytesReceived(object? sender, RemoteCommand command) + { + if (command.Data is null) + { + exception = new BusinessException(command.Message ?? "File is empty"); + _waitFileCancelSource.Cancel(); + } + + bytes = command.Data; + _waitFileCancelSource.Cancel(); + } + _waitFileCancelSource.Token.WaitForCancel(timeout ?? TimeSpan.FromSeconds(15)); + BytesReceived -= OnBytesReceived; + if (exception != null) + throw exception; + return new MemoryStream(bytes); + } + + public void Dispose() + { + _waitFileCancelSource.Dispose(); + _poller.Stop(); + _poller.Dispose(); + + _dealer.SendFrame(MessagePackSerializer.Serialize(new RemoteCommand(CommandType.Exit))); + _dealer.Disconnect($"tcp://{_inferenceClientConfig.ZeroMqHost}:{_inferenceClientConfig.ZeroMqPort}"); + _dealer.Close(); + _dealer.Dispose(); + } +} \ No newline at end of file diff --git a/Azaion.Common/Services/InferenceService.cs b/Azaion.Common/Services/InferenceService.cs index 3ce4ef6..119353e 100644 --- a/Azaion.Common/Services/InferenceService.cs +++ b/Azaion.Common/Services/InferenceService.cs @@ -1,11 +1,11 @@ -using System.Text; -using Azaion.Common.Database; +using Azaion.Common.Database; using Azaion.Common.DTO.Config; -using Azaion.CommonSecurity; +using Azaion.Common.Events; +using Azaion.Common.Extensions; using Azaion.CommonSecurity.DTO.Commands; using Azaion.CommonSecurity.Services; +using MediatR; using MessagePack; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -13,45 +13,74 @@ namespace Azaion.Common.Services; public interface IInferenceService { - Task RunInference(List mediaPaths, Func processAnnotation, CancellationToken detectToken = default); + Task RunInference(List mediaPaths, CancellationToken ct = default); void StopInference(); } -public class InferenceService(ILogger logger, IInferenceClient client, IAzaionApi azaionApi, IOptions aiConfigOptions) : IInferenceService +public class InferenceService : IInferenceService { - public async Task RunInference(List mediaPaths, Func processAnnotation, CancellationToken detectToken = default) + private readonly IInferenceClient _client; + private readonly IAzaionApi _azaionApi; + private readonly IOptions _aiConfigOptions; + private readonly IAnnotationService _annotationService; + private readonly IMediator _mediator; + private CancellationTokenSource _inferenceCancelTokenSource = new(); + + public InferenceService( + ILogger logger, + IInferenceClient client, + IAzaionApi azaionApi, + IOptions aiConfigOptions, + IAnnotationService annotationService, + IMediator mediator) { - client.Send(RemoteCommand.Create(CommandType.Login, azaionApi.Credentials)); - var aiConfig = aiConfigOptions.Value; + _client = client; + _azaionApi = azaionApi; + _aiConfigOptions = aiConfigOptions; + _annotationService = annotationService; + _mediator = mediator; - aiConfig.Paths = mediaPaths; - client.Send(RemoteCommand.Create(CommandType.Inference, aiConfig)); - - while (!detectToken.IsCancellationRequested) + client.InferenceDataReceived += async (sender, command) => { try { - var bytes = client.GetBytes(ct: detectToken); - if (bytes == null) - throw new Exception("Can't get bytes from inference client"); - - if (bytes.Length == 4 && Encoding.UTF8.GetString(bytes) == "DONE") + if (command.Message == "DONE") + { + _inferenceCancelTokenSource?.Cancel(); return; + } - var annotationImage = MessagePackSerializer.Deserialize(bytes, cancellationToken: detectToken); - - await processAnnotation(annotationImage); + var annImage = MessagePackSerializer.Deserialize(command.Data); + await ProcessDetection(annImage); } catch (Exception e) { logger.LogError(e, e.Message); - break; } - } + }; + } + + private async Task ProcessDetection(AnnotationImage annotationImage, CancellationToken ct = default) + { + var annotation = await _annotationService.SaveAnnotation(annotationImage, ct); + await _mediator.Publish(new AnnotationAddedEvent(annotation), ct); + } + + public async Task RunInference(List mediaPaths, CancellationToken ct = default) + { + _inferenceCancelTokenSource = new CancellationTokenSource(); + _client.Send(RemoteCommand.Create(CommandType.Login, _azaionApi.Credentials)); + + var aiConfig = _aiConfigOptions.Value; + aiConfig.Paths = mediaPaths; + _client.Send(RemoteCommand.Create(CommandType.Inference, aiConfig)); + + using var combinedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(ct, _inferenceCancelTokenSource.Token); + await combinedTokenSource.Token.AsTask(); } public void StopInference() { - client.Send(RemoteCommand.Create(CommandType.StopInference)); + _client.Send(RemoteCommand.Create(CommandType.StopInference)); } } \ No newline at end of file diff --git a/Azaion.CommonSecurity/Azaion.CommonSecurity.csproj b/Azaion.CommonSecurity/Azaion.CommonSecurity.csproj index 88fbb9f..4bd3564 100644 --- a/Azaion.CommonSecurity/Azaion.CommonSecurity.csproj +++ b/Azaion.CommonSecurity/Azaion.CommonSecurity.csproj @@ -8,6 +8,7 @@ + diff --git a/Azaion.CommonSecurity/DTO/Commands/RemoteCommand.cs b/Azaion.CommonSecurity/DTO/Commands/RemoteCommand.cs index 9fd9f7e..fbeecf6 100644 --- a/Azaion.CommonSecurity/DTO/Commands/RemoteCommand.cs +++ b/Azaion.CommonSecurity/DTO/Commands/RemoteCommand.cs @@ -3,7 +3,7 @@ namespace Azaion.CommonSecurity.DTO.Commands; [MessagePackObject] -public class RemoteCommand(CommandType commandType, byte[]? data = null) +public class RemoteCommand(CommandType commandType, byte[]? data = null, string? message = null) { [Key("CommandType")] public CommandType CommandType { get; set; } = commandType; @@ -11,11 +11,14 @@ public class RemoteCommand(CommandType commandType, byte[]? data = null) [Key("Data")] public byte[]? Data { get; set; } = data; + [Key("Message")] + public string? Message { get; set; } = message; + public static RemoteCommand Create(CommandType commandType) => new(commandType); - public static RemoteCommand Create(CommandType commandType, T data) where T : class => - new(commandType, MessagePackSerializer.Serialize(data)); + public static RemoteCommand Create(CommandType commandType, T data, string? message = null) where T : class => + new(commandType, MessagePackSerializer.Serialize(data), message); } [MessagePackObject] @@ -34,7 +37,12 @@ public enum CommandType None = 0, Login = 10, Load = 20, + DataBytes = 25, Inference = 30, + InferenceData = 35, StopInference = 40, - Exit = 100 + AIAvailabilityCheck = 80, + AIAvailabilityResult = 85, + Error = 90, + Exit = 100, } \ No newline at end of file diff --git a/Azaion.CommonSecurity/DTO/ExternalClientsConfig.cs b/Azaion.CommonSecurity/DTO/ExternalClientsConfig.cs index 33d66fb..25b1276 100644 --- a/Azaion.CommonSecurity/DTO/ExternalClientsConfig.cs +++ b/Azaion.CommonSecurity/DTO/ExternalClientsConfig.cs @@ -10,7 +10,7 @@ public abstract class ExternalClientConfig public class InferenceClientConfig : ExternalClientConfig { - public string ApiUrl { get; set; } + public string ApiUrl { get; set; } = null!; } public class GpsDeniedClientConfig : ExternalClientConfig diff --git a/Azaion.CommonSecurity/DTO/IEnumerableExtensions.cs b/Azaion.CommonSecurity/DTO/IEnumerableExtensions.cs index df3cf07..32157b9 100644 --- a/Azaion.CommonSecurity/DTO/IEnumerableExtensions.cs +++ b/Azaion.CommonSecurity/DTO/IEnumerableExtensions.cs @@ -1,4 +1,4 @@ -namespace Azaion.Common.Extensions; +namespace Azaion.CommonSecurity.DTO; public static class EnumerableExtensions { diff --git a/Azaion.CommonSecurity/DTO/RoleEnum.cs b/Azaion.CommonSecurity/DTO/RoleEnum.cs index 548d5e0..5f65539 100644 --- a/Azaion.CommonSecurity/DTO/RoleEnum.cs +++ b/Azaion.CommonSecurity/DTO/RoleEnum.cs @@ -1,6 +1,4 @@ -using Azaion.Common.Extensions; - -namespace Azaion.CommonSecurity.DTO; +namespace Azaion.CommonSecurity.DTO; public enum RoleEnum { diff --git a/Azaion.CommonSecurity/Exceptions/BusinessException.cs b/Azaion.CommonSecurity/Exceptions/BusinessException.cs new file mode 100644 index 0000000..51a5a9a --- /dev/null +++ b/Azaion.CommonSecurity/Exceptions/BusinessException.cs @@ -0,0 +1,3 @@ +namespace Azaion.CommonSecurity.Exceptions; + +public class BusinessException(string message) : Exception(message); \ No newline at end of file diff --git a/Azaion.CommonSecurity/Services/InferenceClient.cs b/Azaion.CommonSecurity/Services/InferenceClient.cs deleted file mode 100644 index 30a8119..0000000 --- a/Azaion.CommonSecurity/Services/InferenceClient.cs +++ /dev/null @@ -1,104 +0,0 @@ -using System.Diagnostics; -using System.Text; -using Azaion.CommonSecurity.DTO; -using Azaion.CommonSecurity.DTO.Commands; -using MessagePack; -using Microsoft.Extensions.Options; -using NetMQ; -using NetMQ.Sockets; - -namespace Azaion.CommonSecurity.Services; - -public interface IInferenceClient -{ - void Send(RemoteCommand create); - T? Get(int retries = 24, int tryTimeoutSeconds = 5, CancellationToken ct = default) where T : class; - byte[]? GetBytes(int retries = 24, int tryTimeoutSeconds = 5, CancellationToken ct = default); - void Stop(); -} - -public class InferenceClient : IInferenceClient -{ - private readonly DealerSocket _dealer = new(); - private readonly Guid _clientId = Guid.NewGuid(); - private readonly InferenceClientConfig _inferenceClientConfig; - - public InferenceClient(IOptions config) - { - _inferenceClientConfig = config.Value; - Start(); - _ = Task.Run(ProcessClientCommands); - } - - private void Start() - { - try - { - using var process = new Process(); - process.StartInfo = new ProcessStartInfo - { - FileName = SecurityConstants.EXTERNAL_INFERENCE_PATH, - Arguments = $"--port {_inferenceClientConfig.ZeroMqPort} --api {_inferenceClientConfig.ApiUrl}", - //RedirectStandardOutput = true, - //RedirectStandardError = true, - //CreateNoWindow = true - }; - - process.OutputDataReceived += (_, e) => { if (e.Data != null) Console.WriteLine(e.Data); }; - process.ErrorDataReceived += (_, e) => { if (e.Data != null) Console.WriteLine(e.Data); }; - process.Start(); - } - catch (Exception e) - { - Console.WriteLine(e); - //throw; - } - - _dealer.Options.Identity = Encoding.UTF8.GetBytes(_clientId.ToString("N")); - _dealer.Connect($"tcp://{_inferenceClientConfig.ZeroMqHost}:{_inferenceClientConfig.ZeroMqPort}"); - } - private async Task ProcessClientCommands() - { - //TODO: implement always on ready to client's requests. Utilize RemoteCommand - - await Task.CompletedTask; - } - - public void Stop() - { - if (!_dealer.IsDisposed) - { - _dealer.SendFrame(MessagePackSerializer.Serialize(new RemoteCommand(CommandType.Exit))); - _dealer.Close(); - } - } - - public void Send(RemoteCommand command) - { - _dealer.SendFrame(MessagePackSerializer.Serialize(command)); - } - - public T? Get(int retries = 24, int tryTimeoutSeconds = 5, CancellationToken ct = default) where T : class - { - var bytes = GetBytes(retries, tryTimeoutSeconds, ct); - return bytes != null ? MessagePackSerializer.Deserialize(bytes, cancellationToken: ct) : null; - } - - public byte[]? GetBytes(int retries = 24, int tryTimeoutSeconds = 5, CancellationToken ct = default) - { - var tryNum = 0; - while (!ct.IsCancellationRequested && tryNum < retries) - { - tryNum++; - if (!_dealer.TryReceiveFrameBytes(TimeSpan.FromSeconds(tryTimeoutSeconds), out var bytes)) - continue; - - return bytes; - } - - if (!ct.IsCancellationRequested) - throw new Exception($"Unable to get bytes after {tryNum - 1} retries, {tryTimeoutSeconds} seconds each"); - - return null; - } -} diff --git a/Azaion.CommonSecurity/Services/ResourceLoader.cs b/Azaion.CommonSecurity/Services/ResourceLoader.cs index 38a4d70..773ac40 100644 --- a/Azaion.CommonSecurity/Services/ResourceLoader.cs +++ b/Azaion.CommonSecurity/Services/ResourceLoader.cs @@ -1,22 +1,8 @@ using Azaion.CommonSecurity.DTO.Commands; -using Microsoft.Extensions.DependencyInjection; namespace Azaion.CommonSecurity.Services; public interface IResourceLoader { - MemoryStream LoadFile(string fileName, string? folder = null); -} - -public class ResourceLoader([FromKeyedServices(SecurityConstants.EXTERNAL_INFERENCE_PATH)] IInferenceClient inferenceClient) : IResourceLoader -{ - public MemoryStream LoadFile(string fileName, string? folder = null) - { - inferenceClient.Send(RemoteCommand.Create(CommandType.Load, new LoadFileData(fileName, folder))); - var bytes = inferenceClient.GetBytes(2, 3); - if (bytes == null) - throw new Exception($"Unable to receive {fileName}"); - - return new MemoryStream(bytes); - } + MemoryStream LoadFile(string fileName, string? folder, TimeSpan? timeout = null); } diff --git a/Azaion.Dataset/DatasetExplorerEventHandler.cs b/Azaion.Dataset/DatasetExplorerEventHandler.cs index d2ee77e..0c222d5 100644 --- a/Azaion.Dataset/DatasetExplorerEventHandler.cs +++ b/Azaion.Dataset/DatasetExplorerEventHandler.cs @@ -13,7 +13,7 @@ namespace Azaion.Dataset; public class DatasetExplorerEventHandler( DatasetExplorer datasetExplorer, - AnnotationService annotationService) : + IAnnotationService annotationService) : INotificationHandler, INotificationHandler, INotificationHandler, @@ -26,7 +26,9 @@ public class DatasetExplorerEventHandler( { Key.X, PlaybackControlEnum.RemoveAllAnns }, { Key.Escape, PlaybackControlEnum.Close }, { Key.Down, PlaybackControlEnum.Next }, + { Key.PageDown, PlaybackControlEnum.Next }, { Key.Up, PlaybackControlEnum.Previous }, + { Key.PageUp, PlaybackControlEnum.Previous }, { Key.V, PlaybackControlEnum.ValidateAnnotations}, }; @@ -116,20 +118,23 @@ public class DatasetExplorerEventHandler( var annotation = notification.Annotation; var selectedClass = datasetExplorer.LvClasses.CurrentClassNumber; - //TODO: For editing existing need to handle updates datasetExplorer.AddAnnotationToDict(annotation); if (annotation.Classes.Contains(selectedClass) || selectedClass == -1) { + var index = 0; var annThumb = new AnnotationThumbnail(annotation); if (datasetExplorer.SelectedAnnotationDict.ContainsKey(annThumb.Annotation.Name)) { datasetExplorer.SelectedAnnotationDict.Remove(annThumb.Annotation.Name); var ann = datasetExplorer.SelectedAnnotations.FirstOrDefault(x => x.Annotation.Name == annThumb.Annotation.Name); if (ann != null) + { + index = datasetExplorer.SelectedAnnotations.IndexOf(ann); datasetExplorer.SelectedAnnotations.Remove(ann); + } } - datasetExplorer.SelectedAnnotations.Insert(0, annThumb); + datasetExplorer.SelectedAnnotations.Insert(index, annThumb); datasetExplorer.SelectedAnnotationDict.Add(annThumb.Annotation.Name, annThumb); } }); diff --git a/Azaion.Inference/api_client.pyx b/Azaion.Inference/api_client.pyx index 5de25db..a6ed66b 100644 --- a/Azaion.Inference/api_client.pyx +++ b/Azaion.Inference/api_client.pyx @@ -1,4 +1,5 @@ import json +import os from http import HTTPStatus from os import path from uuid import UUID @@ -6,6 +7,7 @@ import jwt import requests cimport constants import yaml +from requests import HTTPError from cdn_manager cimport CDNManager, CDNCredentials from hardware_service cimport HardwareService @@ -23,6 +25,9 @@ cdef class ApiClient: cdef set_credentials(self, Credentials credentials): self.credentials = credentials + if self.cdn_manager is not None: + return + yaml_bytes = self.load_bytes(constants.CDN_CONFIG, '') yaml_config = yaml.safe_load(yaml_bytes) creds = CDNCredentials(yaml_config["host"], @@ -34,11 +39,18 @@ cdef class ApiClient: self.cdn_manager = CDNManager(creds) cdef login(self): - response = requests.post(f"{self.api_url}/login", - json={"email": self.credentials.email, "password": self.credentials.password}) - response.raise_for_status() - token = response.json()["token"] - self.set_token(token) + response = None + try: + response = requests.post(f"{self.api_url}/login", + json={"email": self.credentials.email, "password": self.credentials.password}) + response.raise_for_status() + token = response.json()["token"] + self.set_token(token) + except HTTPError as e: + print(response.json()) + if response.status_code == HTTPStatus.CONFLICT: + res = response.json() + raise Exception(res['Message']) cdef set_token(self, str token): @@ -123,11 +135,21 @@ cdef class ApiClient: return data cdef load_big_small_resource(self, str resource_name, str folder, str key): - cdef str big_part = path.join(folder, f'{resource_name}.big') + cdef str big_part = f'{resource_name}.big' cdef str small_part = f'{resource_name}.small' - with open(big_part, 'rb') as binary_file: - encrypted_bytes_big = binary_file.read() + print(f'checking on existence for {folder}\\{big_part}') + if os.path.exists(os.path.join( folder, big_part)): + with open(path.join( folder, big_part), 'rb') as binary_file: + encrypted_bytes_big = binary_file.read() + print(f'local file {folder}\\{big_part} is found!') + else: + print(f'downloading file {folder}\\{big_part} from cdn...') + if self.cdn_manager.download(folder, big_part): + with open(path.join( folder, big_part), 'rb') as binary_file: + encrypted_bytes_big = binary_file.read() + else: + return None encrypted_bytes_small = self.load_bytes(small_part, folder) @@ -145,7 +167,7 @@ cdef class ApiClient: part_big = resource_encrypted[part_small_size:] - self.cdn_manager.upload(constants.MODELS_FOLDER, big_part_name, part_big) + self.cdn_manager.upload(folder, big_part_name, part_big) with open(path.join(folder, big_part_name), 'wb') as f: f.write(part_big) - self.upload_file(small_part_name, part_small, constants.MODELS_FOLDER) \ No newline at end of file + self.upload_file(small_part_name, part_small, folder) \ No newline at end of file diff --git a/Azaion.Inference/cdn_manager.pyx b/Azaion.Inference/cdn_manager.pyx index 92b4d3f..5c0fc01 100644 --- a/Azaion.Inference/cdn_manager.pyx +++ b/Azaion.Inference/cdn_manager.pyx @@ -31,10 +31,10 @@ cdef class CDNManager: print(e) return False - cdef download(self, str bucket, str filename): + cdef download(self, str folder, str filename): try: - self.download_client.download_file(bucket, filename, filename) - print(f'downloaded {filename} from the {bucket} to current folder') + self.download_client.download_file(folder, filename, f'{folder}\\{filename}') + print(f'downloaded {filename} from the {folder} to current folder') return True except Exception as e: print(e) diff --git a/Azaion.Inference/constants.pxd b/Azaion.Inference/constants.pxd index 1e678c8..6cd4339 100644 --- a/Azaion.Inference/constants.pxd +++ b/Azaion.Inference/constants.pxd @@ -13,7 +13,5 @@ cdef str MODELS_FOLDER cdef int SMALL_SIZE_KB -cdef bytes DONE_SIGNAL - cdef log(str log_message, bytes client_id=*) \ No newline at end of file diff --git a/Azaion.Inference/inference.pxd b/Azaion.Inference/inference.pxd index f0cfc39..7cb602f 100644 --- a/Azaion.Inference/inference.pxd +++ b/Azaion.Inference/inference.pxd @@ -16,7 +16,7 @@ cdef class Inference: cdef int model_width cdef int model_height - cdef build_tensor_engine(self) + cdef build_tensor_engine(self, object updater_callback) cdef init_ai(self) cdef bint is_building_engine cdef bint is_video(self, str filepath) diff --git a/Azaion.Inference/inference.pyx b/Azaion.Inference/inference.pyx index 48925de..ccae2c6 100644 --- a/Azaion.Inference/inference.pyx +++ b/Azaion.Inference/inference.pyx @@ -32,7 +32,7 @@ cdef class Inference: self.engine = None self.is_building_engine = False - cdef build_tensor_engine(self): + cdef build_tensor_engine(self, object updater_callback): is_nvidia = HardwareService.has_nvidia_gpu() if not is_nvidia: return @@ -40,18 +40,22 @@ cdef class Inference: engine_filename = TensorRTEngine.get_engine_filename(0) key = Security.get_model_encryption_key() models_dir = constants.MODELS_FOLDER - if not os.path.exists(os.path.join( models_dir, f'{engine_filename}.big')): - #TODO: Check cdn on engine exists, if there is, download - self.is_building_engine = True - time.sleep(8) # prevent simultaneously loading dll and models - onnx_model = self.api_client.load_big_small_resource(constants.AI_ONNX_MODEL_FILE, models_dir, key) - model_bytes = TensorRTEngine.convert_from_onnx(onnx_model) - self.api_client.upload_big_small_resource(model_bytes, engine_filename, models_dir, key) - print('uploaded ') - self.is_building_engine = False - else: - print('tensor rt engine is here, no need to build') + self.is_building_engine = True + updater_callback('downloading') + if self.api_client.load_big_small_resource(engine_filename, models_dir, key): + print('tensor rt engine is here, no need to build') + self.is_building_engine = False + return + + # time.sleep(8) # prevent simultaneously loading dll and models + updater_callback('converting') + onnx_model = self.api_client.load_big_small_resource(constants.AI_ONNX_MODEL_FILE, models_dir, key) + model_bytes = TensorRTEngine.convert_from_onnx(onnx_model) + updater_callback('uploading') + self.api_client.upload_big_small_resource(model_bytes, engine_filename, models_dir, key) + print(f'uploaded {engine_filename} to CDN and API') + self.is_building_engine = False cdef init_ai(self): if self.engine is not None: @@ -161,7 +165,6 @@ cdef class Inference: self.stop_signal = False self.init_ai() - print(ai_config.paths) for m in ai_config.paths: if self.is_video(m): videos.append(m) diff --git a/Azaion.Inference/main.pyx b/Azaion.Inference/main.pyx index fbb806f..7ed4306 100644 --- a/Azaion.Inference/main.pyx +++ b/Azaion.Inference/main.pyx @@ -34,7 +34,8 @@ cdef class CommandProcessor: try: command = self.inference_queue.get(timeout=0.5) self.inference.run_inference(command) - self.remote_handler.send(command.client_id, 'DONE'.encode('utf-8')) + end_inference_command = RemoteCommand(CommandType.INFERENCE_DATA, None, 'DONE') + self.remote_handler.send(command.client_id, end_inference_command.serialize()) except queue.Empty: continue except Exception as e: @@ -44,11 +45,13 @@ cdef class CommandProcessor: cdef on_command(self, RemoteCommand command): try: if command.command_type == CommandType.LOGIN: - self.login(command) + self.api_client.set_credentials(Credentials.from_msgpack(command.data)) elif command.command_type == CommandType.LOAD: self.load_file(command) elif command.command_type == CommandType.INFERENCE: self.inference_queue.put(command) + elif command.command_type == CommandType.AI_AVAILABILITY_CHECK: + self.build_tensor_engine(command.client_id) elif command.command_type == CommandType.STOP_INFERENCE: self.inference.stop() elif command.command_type == CommandType.EXIT: @@ -59,18 +62,28 @@ cdef class CommandProcessor: except Exception as e: print(f"Error handling client: {e}") - cdef login(self, RemoteCommand command): - self.api_client.set_credentials(Credentials.from_msgpack(command.data)) - Thread(target=self.inference.build_tensor_engine).start() # build AI engine in non-blocking thread + cdef build_tensor_engine(self, client_id): + self.inference.build_tensor_engine(lambda status: self.build_tensor_status_updater(client_id, status)) + self.remote_handler.send(client_id, RemoteCommand(CommandType.AI_AVAILABILITY_RESULT, None, 'enabled').serialize()) + + cdef build_tensor_status_updater(self, bytes client_id, str status): + self.remote_handler.send(client_id, RemoteCommand(CommandType.AI_AVAILABILITY_RESULT, None, status).serialize()) cdef load_file(self, RemoteCommand command): - cdef FileData file_data = FileData.from_msgpack(command.data) - response = self.api_client.load_bytes(file_data.filename, file_data.folder) - self.remote_handler.send(command.client_id, response) + cdef RemoteCommand response + cdef FileData file_data + cdef bytes file_bytes + try: + file_data = FileData.from_msgpack(command.data) + file_bytes = self.api_client.load_bytes(file_data.filename, file_data.folder) + response = RemoteCommand(CommandType.DATA_BYTES, file_bytes) + except Exception as e: + response = RemoteCommand(CommandType.DATA_BYTES, None, str(e)) + self.remote_handler.send(command.client_id, response.serialize()) cdef on_annotation(self, RemoteCommand cmd, Annotation annotation): - data = annotation.serialize() - self.remote_handler.send(cmd.client_id, data) + cdef RemoteCommand response = RemoteCommand(CommandType.INFERENCE_DATA, annotation.serialize()) + self.remote_handler.send(cmd.client_id, response.serialize()) def stop(self): self.inference.stop() diff --git a/Azaion.Inference/remote_command.pxd b/Azaion.Inference/remote_command.pxd index 014ece0..2ead3a3 100644 --- a/Azaion.Inference/remote_command.pxd +++ b/Azaion.Inference/remote_command.pxd @@ -1,13 +1,19 @@ cdef enum CommandType: LOGIN = 10 LOAD = 20 + DATA_BYTES = 25 INFERENCE = 30 + INFERENCE_DATA = 35 STOP_INFERENCE = 40 + AI_AVAILABILITY_CHECK = 80, + AI_AVAILABILITY_RESULT = 85, + ERROR = 90 EXIT = 100 cdef class RemoteCommand: cdef public bytes client_id cdef CommandType command_type + cdef str message cdef bytes data @staticmethod diff --git a/Azaion.Inference/remote_command.pyx b/Azaion.Inference/remote_command.pyx index a053685..333b34a 100644 --- a/Azaion.Inference/remote_command.pyx +++ b/Azaion.Inference/remote_command.pyx @@ -1,16 +1,22 @@ import msgpack cdef class RemoteCommand: - def __init__(self, CommandType command_type, bytes data): + def __init__(self, CommandType command_type, bytes data, str message=None): self.command_type = command_type self.data = data + self.message = message def __str__(self): command_type_names = { 10: "LOGIN", 20: "LOAD", + 25: "DATA_BYTES", 30: "INFERENCE", + 35: "INFERENCE_DATA", 40: "STOP_INFERENCE", + 80: "AI_AVAILABILITY_CHECK", + 85: "AI_AVAILABILITY_RESULT", + 90: "ERROR", 100: "EXIT" } data_str = f'{len(self.data)} bytes' if self.data else '' @@ -19,10 +25,11 @@ cdef class RemoteCommand: @staticmethod cdef from_msgpack(bytes data): unpacked = msgpack.unpackb(data, strict_map_key=False) - return RemoteCommand(unpacked.get("CommandType"), unpacked.get("Data")) + return RemoteCommand(unpacked.get("CommandType"), unpacked.get("Data"), unpacked.get("Message")) cdef bytes serialize(self): return msgpack.packb({ "CommandType": self.command_type, - "Data": self.data + "Data": self.data, + "Message": self.message }) diff --git a/Azaion.Inference/remote_command_handler.pyx b/Azaion.Inference/remote_command_handler.pyx index da34d5d..86877ed 100644 --- a/Azaion.Inference/remote_command_handler.pyx +++ b/Azaion.Inference/remote_command_handler.pyx @@ -70,10 +70,12 @@ cdef class RemoteCommandHandler: worker_socket.close() cdef send(self, bytes client_id, bytes data): - with self._context.socket(zmq.DEALER) as socket: - socket.connect("inproc://backend") - socket.send_multipart([client_id, data]) - # constants.log(f'Sent {len(data)} bytes.', client_id) + self._router.send_multipart([client_id, data]) + + # with self._context.socket(zmq.DEALER) as socket: + # socket.connect("inproc://backend") + # socket.send_multipart([client_id, data]) + # # constants.log(f'Sent {len(data)} bytes.', client_id) cdef stop(self): self._shutdown_event.set() diff --git a/Azaion.Suite/App.xaml.cs b/Azaion.Suite/App.xaml.cs index bd8a258..ecb2d8d 100644 --- a/Azaion.Suite/App.xaml.cs +++ b/Azaion.Suite/App.xaml.cs @@ -2,7 +2,6 @@ using System.Net.Http; using System.Reflection; using System.Text; -using System.Text.Unicode; using System.Windows; using System.Windows.Threading; using Azaion.Annotator; @@ -18,7 +17,6 @@ using Azaion.CommonSecurity.DTO; using Azaion.CommonSecurity.DTO.Commands; using Azaion.CommonSecurity.Services; using Azaion.Dataset; -using LazyCache; using LibVLCSharp.Shared; using MediatR; using Microsoft.Extensions.Configuration; @@ -40,13 +38,15 @@ public partial class App private FormState _formState = null!; private IInferenceClient _inferenceClient = null!; - private IResourceLoader _resourceLoader = null!; private Stream _securedConfig = null!; private Stream _systemConfig = null!; private static readonly Guid KeyPressTaskId = Guid.NewGuid(); + private string _loadErrors = ""; + private readonly ICache _cache = new MemoryCache(); private IAzaionApi _azaionApi = null!; + private CancellationTokenSource _mainCancelTokenSource = new(); private void OnDispatcherUnhandledException(object sender, DispatcherUnhandledExceptionEventArgs e) { @@ -89,10 +89,10 @@ public partial class App new ConfigUpdater().CheckConfig(); var secureAppConfig = ReadSecureAppConfig(); var apiDir = secureAppConfig.DirectoriesConfig.ApiResourcesDirectory; - _inferenceClient = new InferenceClient(new OptionsWrapper(secureAppConfig.InferenceClientConfig)); - _resourceLoader = new ResourceLoader(_inferenceClient); + _inferenceClient = new InferenceClient(new OptionsWrapper(secureAppConfig.InferenceClientConfig), _mainCancelTokenSource.Token); var login = new Login(); + var loader = (IResourceLoader)_inferenceClient; login.CredentialsEntered += (_, credentials) => { _inferenceClient.Send(RemoteCommand.Create(CommandType.Login, credentials)); @@ -100,8 +100,8 @@ public partial class App try { - _securedConfig = _resourceLoader.LoadFile("config.secured.json", apiDir); - _systemConfig = _resourceLoader.LoadFile("config.system.json", apiDir); + _securedConfig = loader.LoadFile("config.secured.json", apiDir); + _systemConfig = loader.LoadFile("config.system.json", apiDir); } catch (Exception e) { @@ -123,12 +123,13 @@ public partial class App { try { - var stream = _resourceLoader.LoadFile($"{assemblyName}.dll", apiDir); + var stream = loader.LoadFile($"{assemblyName}.dll", apiDir); return Assembly.Load(stream.ToArray()); } catch (Exception e) { - Console.WriteLine(e); + Log.Logger.Error(e, $"Failed to load assembly {assemblyName}"); + _loadErrors += $"{e.Message}{Environment.NewLine}{Environment.NewLine}"; var currentLocation = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!; var dllPath = Path.Combine(currentLocation, SecurityConstants.DUMMY_DIR, $"{assemblyName}.dll"); return Assembly.LoadFile(dllPath); @@ -143,13 +144,13 @@ public partial class App StartMain(); _host.Start(); - EventManager.RegisterClassHandler(typeof(UIElement), UIElement.KeyDownEvent, new RoutedEventHandler(GlobalClick)); + EventManager.RegisterClassHandler(typeof(UIElement), UIElement.PreviewKeyDownEvent, new RoutedEventHandler(GlobalKeyHandler)); _host.Services.GetRequiredService().Show(); }; login.Closed += (sender, args) => { if (!login.MainSuiteOpened) - _inferenceClient.Stop(); + _inferenceClient.Dispose(); }; login.ShowDialog(); } @@ -218,7 +219,7 @@ public partial class App services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); @@ -231,16 +232,20 @@ public partial class App Annotation.InitializeDirs(_host.Services.GetRequiredService>().Value); _mediator = _host.Services.GetRequiredService(); + if (!string.IsNullOrEmpty(_loadErrors)) + _mediator.Publish(new LoadErrorEvent(_loadErrors)); + _logger = _host.Services.GetRequiredService>(); _formState = _host.Services.GetRequiredService(); DispatcherUnhandledException += OnDispatcherUnhandledException; } - private void GlobalClick(object sender, RoutedEventArgs e) + private void GlobalKeyHandler(object sender, RoutedEventArgs e) { var args = (KeyEventArgs)e; var keyEvent = new KeyEvent(sender, args, _formState.ActiveWindow); ThrottleExt.Throttle(() => _mediator.Publish(keyEvent), KeyPressTaskId, TimeSpan.FromMilliseconds(50)); + //e.Handled = true; } protected override async void OnExit(ExitEventArgs e) @@ -248,4 +253,4 @@ public partial class App base.OnExit(e); await _host.StopAsync(); } -} +} \ No newline at end of file diff --git a/Azaion.Suite/Azaion.Suite.csproj b/Azaion.Suite/Azaion.Suite.csproj index cde203f..0a1e040 100644 --- a/Azaion.Suite/Azaion.Suite.csproj +++ b/Azaion.Suite/Azaion.Suite.csproj @@ -33,8 +33,8 @@ - - + + @@ -62,8 +62,8 @@ - - + + diff --git a/Azaion.Suite/MainSuite.xaml.cs b/Azaion.Suite/MainSuite.xaml.cs index 4b16c7d..9a96a9a 100644 --- a/Azaion.Suite/MainSuite.xaml.cs +++ b/Azaion.Suite/MainSuite.xaml.cs @@ -143,7 +143,7 @@ public partial class MainSuite foreach (var window in _openedWindows) window.Value.Close(); - _inferenceClient.Stop(); + _inferenceClient.Dispose(); _gpsMatcherClient.Stop(); Application.Current.Shutdown(); } diff --git a/Azaion.Test/ThrottleTest.cs b/Azaion.Test/ThrottleTest.cs new file mode 100644 index 0000000..6a5b746 --- /dev/null +++ b/Azaion.Test/ThrottleTest.cs @@ -0,0 +1,57 @@ +using Azaion.Common.Extensions; +using FluentAssertions; +using Xunit; + +namespace Azaion.Annotator.Test; + +public class ThrottleTest +{ + private readonly Guid _testTaskId = Guid.NewGuid(); + + [Fact] + public async Task TestScheduleAfterCooldown() + { + var calls = new List(); + + Console.WriteLine($"Start time: {DateTime.Now}"); + for (int i = 0; i < 10; i++) + { + ThrottleExt.Throttle(() => + { + calls.Add(DateTime.Now); + return Task.CompletedTask; + }, _testTaskId, TimeSpan.FromSeconds(1), scheduleCallAfterCooldown: true); + } + + await Task.Delay(TimeSpan.FromSeconds(2)); + Console.WriteLine(string.Join(',', calls)); + + calls.Count.Should().Be(2); + } + + [Fact] + public async Task TestScheduleAfterCooldown2() + { + var calls = new List(); + + Console.WriteLine($"Start time: {DateTime.Now}"); + ThrottleExt.Throttle(() => + { + calls.Add(DateTime.Now); + return Task.CompletedTask; + }, _testTaskId, TimeSpan.FromSeconds(1), scheduleCallAfterCooldown: true); + await Task.Delay(TimeSpan.FromSeconds(2)); + + ThrottleExt.Throttle(() => + { + calls.Add(DateTime.Now); + return Task.CompletedTask; + }, _testTaskId, TimeSpan.FromSeconds(1), scheduleCallAfterCooldown: true); + + + await Task.Delay(TimeSpan.FromSeconds(2)); + Console.WriteLine(string.Join(',', calls)); + + calls.Count.Should().Be(2); + } +} \ No newline at end of file diff --git a/Dummy/Azaion.Annotator/Annotator.xaml b/Dummy/Azaion.Annotator/Annotator.xaml index dae811d..f139f98 100644 --- a/Dummy/Azaion.Annotator/Annotator.xaml +++ b/Dummy/Azaion.Annotator/Annotator.xaml @@ -7,13 +7,39 @@ Title="Azaion Annotator" Height="800" Width="1100" WindowState="Maximized" > - - + + + + + + + + Під час запуску виникла помилка! + + Error happened during the launch! + + + Будь ласка перевірте правильність email чи паролю! Також зауважте, що запуск можливий лише з одного конкретного компьютера, копіювання заборонене! Для подальшого вирішення проблеми ви можете зв'язатися з нами: hi@azaion.com @@ -21,5 +47,6 @@ Please check your email or password! The program is restricted to start only from particular hardware, copying is forbidden! For the further guidance, please feel free to contact us: hi@azaion.com - + + diff --git a/Dummy/Azaion.Annotator/AnnotatorEventHandler.cs b/Dummy/Azaion.Annotator/AnnotatorEventHandler.cs index d27500f..f216f8a 100644 --- a/Dummy/Azaion.Annotator/AnnotatorEventHandler.cs +++ b/Dummy/Azaion.Annotator/AnnotatorEventHandler.cs @@ -1,3 +1,13 @@ -namespace Azaion.Annotator; +using Azaion.Common.Events; +using MediatR; -public class AnnotatorEventHandler; +namespace Azaion.Annotator; + +public class AnnotatorEventHandler(Annotator annotator) : INotificationHandler +{ + public Task Handle(LoadErrorEvent notification, CancellationToken cancellationToken) + { + annotator.Dispatcher.Invoke(() => annotator.TbError.Text = notification.Error); + return Task.CompletedTask; + } +} diff --git a/build/cdn_manager.py b/build/cdn_manager.py index 04fce3b..e3ed75b 100644 --- a/build/cdn_manager.py +++ b/build/cdn_manager.py @@ -34,24 +34,25 @@ class CDNManager: print(e) return False - def download(self, bucket: str, filename: str): + def download(self, folder: str, filename: str): try: if filename is not None: - self.download_client.download_file(bucket, filename, os.path.join(bucket, filename)) - print(f'downloaded {filename} from the {bucket} to current folder') + self.download_client.download_file(folder, filename, f'{folder}\\{filename}') + print(f'downloaded {filename} from the {folder} to current folder') + return True else: - response = self.download_client.list_objects_v2(Bucket=bucket) + response = self.download_client.list_objects_v2(Bucket=folder) if 'Contents' in response: for obj in response['Contents']: object_key = obj['Key'] - local_filepath = os.path.join(bucket, object_key) + local_filepath = os.path.join(folder, object_key) local_dir = os.path.dirname(local_filepath) if local_dir: os.makedirs(local_dir, exist_ok=True) if not object_key.endswith('/'): try: - self.download_client.download_file(bucket, object_key, local_filepath) + self.download_client.download_file(folder, object_key, local_filepath) except Exception as e_file: all_successful = False # Mark as failed if any file fails return True diff --git a/build/installer.iterative.iss b/build/installer.iterative.iss index 316036e..6285322 100644 --- a/build/installer.iterative.iss +++ b/build/installer.iterative.iss @@ -12,7 +12,6 @@ UninstallDisplayName=Azaion Suite UninstallDisplayIcon={app}\Azaion.Suite.exe Compression=lzma2/fast SolidCompression=yes -DiskSpanning=yes [Languages] Name: "english"; MessagesFile: "compiler:Default.isl"