From d93da155280c5ce5b8e61af7cc858ecc90b493b8 Mon Sep 17 00:00:00 2001 From: Alex Bezdieniezhnykh Date: Sun, 2 Mar 2025 21:32:31 +0200 Subject: [PATCH] fix switcher between modes in DatasetExplorer.xaml --- Azaion.Annotator/Annotator.xaml.cs | 4 +- Azaion.Common/Controls/DetectionClasses.xaml | 96 +++++++++++++++++-- .../Controls/DetectionClasses.xaml.cs | 1 + Azaion.Common/Controls/DetectionControl.cs | 2 +- Azaion.Common/DTO/DetectionClass.cs | 2 +- .../Services/PythonResourceLoader.cs | 2 +- Azaion.Dataset/DatasetExplorer.xaml | 2 +- Azaion.Dataset/DatasetExplorer.xaml.cs | 14 +-- Azaion.Dataset/DatasetExplorerEventHandler.cs | 9 +- Azaion.Inference/api_client.pyx | 1 + Azaion.Inference/inference.pxd | 8 +- Azaion.Inference/inference.pyx | 44 +++++---- Azaion.Inference/start.spec | 2 - Azaion.Suite/App.xaml.cs | 4 +- build/publish.cmd | 5 +- 15 files changed, 141 insertions(+), 55 deletions(-) diff --git a/Azaion.Annotator/Annotator.xaml.cs b/Azaion.Annotator/Annotator.xaml.cs index 19be86f..e52d578 100644 --- a/Azaion.Annotator/Annotator.xaml.cs +++ b/Azaion.Annotator/Annotator.xaml.cs @@ -49,8 +49,8 @@ public partial class Annotator public bool FollowAI = false; public bool IsInferenceNow = false; - private readonly TimeSpan _thresholdBefore = TimeSpan.FromMilliseconds(100); - private readonly TimeSpan _thresholdAfter = TimeSpan.FromMilliseconds(300); + private readonly TimeSpan _thresholdBefore = TimeSpan.FromMilliseconds(50); + private readonly TimeSpan _thresholdAfter = TimeSpan.FromMilliseconds(150); private static readonly Guid SaveConfigTaskId = Guid.NewGuid(); public ObservableCollection AllMediaFiles { get; set; } = new(); diff --git a/Azaion.Common/Controls/DetectionClasses.xaml b/Azaion.Common/Controls/DetectionClasses.xaml index 4c0b046..ec8ffef 100644 --- a/Azaion.Common/Controls/DetectionClasses.xaml +++ b/Azaion.Common/Controls/DetectionClasses.xaml @@ -82,26 +82,102 @@ Grid.Row="1" Orientation="Horizontal" HorizontalAlignment="Center" - Margin="0,5,0,5"> + Margin="0,2,0,2"> + IsChecked="{Binding IsChecked}" + Style="{StaticResource ButtonRadioButtonStyle}"> + + + + + + + + + + + + + + + + + + + + + + + + Checked="ModeRadioButton_Checked" Margin="3,0,0,0" + Style="{StaticResource ButtonRadioButtonStyle}"> + + + + + + + + + + + + + + + + Checked="ModeRadioButton_Checked" Margin="3,0,0,0" + Style="{StaticResource ButtonRadioButtonStyle}" + > + + + + + + + + + + + + + + + + diff --git a/Azaion.Common/Controls/DetectionClasses.xaml.cs b/Azaion.Common/Controls/DetectionClasses.xaml.cs index bea8c5d..0a2e3a6 100644 --- a/Azaion.Common/Controls/DetectionClasses.xaml.cs +++ b/Azaion.Common/Controls/DetectionClasses.xaml.cs @@ -13,6 +13,7 @@ public class DetectionClassChangedEventArgs(DetectionClass detectionClass, int c public partial class DetectionClasses { public event EventHandler? DetectionClassChanged; + public bool IsChecked = true; public DetectionClasses() { diff --git a/Azaion.Common/Controls/DetectionControl.cs b/Azaion.Common/Controls/DetectionControl.cs index 1cbfa97..a579df7 100644 --- a/Azaion.Common/Controls/DetectionControl.cs +++ b/Azaion.Common/Controls/DetectionControl.cs @@ -26,7 +26,7 @@ public class DetectionControl : Border { _grid.Background = value.ColorBrush; _probabilityLabel.Background = value.ColorBrush; - _classNameLabel.Text = value.Name; + _classNameLabel.Text = value.UIName; _detectionClass = value; } } diff --git a/Azaion.Common/DTO/DetectionClass.cs b/Azaion.Common/DTO/DetectionClass.cs index 00f7cf6..d050b94 100644 --- a/Azaion.Common/DTO/DetectionClass.cs +++ b/Azaion.Common/DTO/DetectionClass.cs @@ -37,7 +37,7 @@ public class DetectionClass public int ClassNumber => Id + 1; [JsonIgnore] - public int YoloId => (int)PhotoMode + Id; + public int YoloId => Id == -1 ? Id : (int)PhotoMode + Id; [JsonIgnore] public SolidColorBrush ColorBrush => new(Color); diff --git a/Azaion.CommonSecurity/Services/PythonResourceLoader.cs b/Azaion.CommonSecurity/Services/PythonResourceLoader.cs index cd7646c..be07bcd 100644 --- a/Azaion.CommonSecurity/Services/PythonResourceLoader.cs +++ b/Azaion.CommonSecurity/Services/PythonResourceLoader.cs @@ -28,7 +28,7 @@ public class PythonResourceLoader : IResourceLoader, IAuthProvider public PythonResourceLoader(PythonConfig config) { - //StartPython(); + StartPython(); _dealer.Options.Identity = Encoding.UTF8.GetBytes(_clientId.ToString("N")); _dealer.Connect($"tcp://{config.ZeroMqHost}:{config.ZeroMqPort}"); } diff --git a/Azaion.Dataset/DatasetExplorer.xaml b/Azaion.Dataset/DatasetExplorer.xaml index 73bf9e8..73eeb63 100644 --- a/Azaion.Dataset/DatasetExplorer.xaml +++ b/Azaion.Dataset/DatasetExplorer.xaml @@ -57,7 +57,7 @@ - + diff --git a/Azaion.Dataset/DatasetExplorer.xaml.cs b/Azaion.Dataset/DatasetExplorer.xaml.cs index c5a502b..9e6689f 100644 --- a/Azaion.Dataset/DatasetExplorer.xaml.cs +++ b/Azaion.Dataset/DatasetExplorer.xaml.cs @@ -22,7 +22,7 @@ public partial class DatasetExplorer private readonly AnnotationConfig _annotationConfig; private readonly DirectoriesConfig _directoriesConfig; - private readonly Dictionary> _annotationsDict; + private readonly Dictionary> _annotationsDict; private readonly CancellationTokenSource _cts = new(); public List AllDetectionClasses { get; set; } @@ -61,7 +61,7 @@ public partial class DatasetExplorer var photoModes = Enum.GetValues(typeof(PhotoMode)).Cast().ToList(); _annotationsDict = _annotationConfig.DetectionClasses.SelectMany(cls => photoModes.Select(mode => (int)mode + cls.Id)) - .ToDictionary(x => x, _ => new List()); + .ToDictionary(x => x, _ => new Dictionary()); _annotationsDict.Add(-1, []); AnnotationsClasses = annotationConfig.Value.DetectionClasses; @@ -141,8 +141,8 @@ public partial class DatasetExplorer public void AddAnnotationToDict(Annotation annotation) { foreach (var c in annotation.Classes) - _annotationsDict[c].Add(annotation); - _annotationsDict[-1].Add(annotation); + _annotationsDict[c][annotation.Name] = annotation; + _annotationsDict[-1][annotation.Name] = annotation; } private async Task LoadClassDistribution() @@ -277,9 +277,11 @@ public partial class DatasetExplorer private async Task ReloadThumbnails() { SelectedAnnotations.Clear(); - foreach (var ann in _annotationsDict[ExplorerEditor.CurrentAnnClass.Id]) + SelectedAnnotationDict.Clear(); + var annotations = _annotationsDict[ExplorerEditor.CurrentAnnClass.YoloId]; + foreach (var ann in annotations.OrderByDescending(x => x.Value.CreatedDate)) { - var annThumb = new AnnotationThumbnail(ann); + var annThumb = new AnnotationThumbnail(ann.Value); SelectedAnnotations.Add(annThumb); SelectedAnnotationDict.Add(annThumb.Annotation.Name, annThumb); } diff --git a/Azaion.Dataset/DatasetExplorerEventHandler.cs b/Azaion.Dataset/DatasetExplorerEventHandler.cs index 3a7926b..62c1540 100644 --- a/Azaion.Dataset/DatasetExplorerEventHandler.cs +++ b/Azaion.Dataset/DatasetExplorerEventHandler.cs @@ -92,11 +92,14 @@ public class DatasetExplorerEventHandler( //TODO: For editing existing need to handle updates datasetExplorer.AddAnnotationToDict(annotation); - if (annotation.Classes.Contains(selectedClass)) + if (annotation.Classes.Contains(selectedClass) || selectedClass == -1) { var annThumb = new AnnotationThumbnail(annotation); - datasetExplorer.SelectedAnnotations.Add(annThumb); - datasetExplorer.SelectedAnnotationDict.Add(annThumb.Annotation.Name, annThumb); + if (!datasetExplorer.SelectedAnnotationDict.ContainsKey(annThumb.Annotation.Name)) + { + datasetExplorer.SelectedAnnotations.Insert(0, annThumb); + datasetExplorer.SelectedAnnotationDict.Add(annThumb.Annotation.Name, annThumb); + } } await Task.CompletedTask; } diff --git a/Azaion.Inference/api_client.pyx b/Azaion.Inference/api_client.pyx index 7063d41..5601e2f 100644 --- a/Azaion.Inference/api_client.pyx +++ b/Azaion.Inference/api_client.pyx @@ -119,3 +119,4 @@ cdef class ApiClient: key = Security.get_model_encryption_key() model_bytes = Security.decrypt_to(encrypted_model_bytes, key) + return model_bytes diff --git a/Azaion.Inference/inference.pxd b/Azaion.Inference/inference.pxd index 3b9979d..6715804 100644 --- a/Azaion.Inference/inference.pxd +++ b/Azaion.Inference/inference.pxd @@ -18,13 +18,13 @@ cdef class Inference: cdef bint is_video(self, str filepath) cdef run_inference(self, RemoteCommand cmd) - cdef _process_video(self, RemoteCommand cmd, str video_name) - cdef _process_images(self, RemoteCommand cmd, list[str] image_paths) + cdef _process_video(self, RemoteCommand cmd, AIRecognitionConfig ai_config, str video_name) + cdef _process_images(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list[str] image_paths) cdef stop(self) cdef preprocess(self, frames) cdef remove_overlapping_detections(self, list[Detection] detections) - cdef postprocess(self, output) + cdef postprocess(self, output, ai_config) cdef split_list_extend(self, lst, chunk_size) - cdef bint is_valid_annotation(self, Annotation annotation) + cdef bint is_valid_annotation(self, Annotation annotation, AIRecognitionConfig ai_config) diff --git a/Azaion.Inference/inference.pyx b/Azaion.Inference/inference.pyx index d59e5bd..ff5eba7 100644 --- a/Azaion.Inference/inference.pyx +++ b/Azaion.Inference/inference.pyx @@ -20,7 +20,6 @@ cdef class Inference: self.model_width = 0 self.model_height = 0 self.class_names = None - self.ai_config = AIRecognitionConfig(4, 2, 0.25, 0.15, 15, 0.8, b'', []) def init_ai(self): model_bytes = self.api_client.load_ai_model() @@ -47,7 +46,7 @@ cdef class Inference: for frame in frames] return np.vstack(blobs) - cdef postprocess(self, output): + cdef postprocess(self, output, ai_config): cdef list[Detection] detections = [] cdef int ann_index cdef float x1, y1, x2, y2, conf, cx, cy, w, h @@ -70,7 +69,8 @@ cdef class Inference: y = (y1 + y2) / 2 w = x2 - x1 h = y2 - y1 - detections.append(Detection(x, y, w, h, class_id, conf)) + if conf >= ai_config.probability_threshold: + detections.append(Detection(x, y, w, h, class_id, conf)) filtered_detections = self.remove_overlapping_detections(detections) results.append(filtered_detections) return results @@ -116,30 +116,32 @@ cdef class Inference: cdef run_inference(self, RemoteCommand cmd): cdef list[str] videos = [] cdef list[str] images = [] + cdef AIRecognitionConfig ai_config = AIRecognitionConfig.from_msgpack(cmd.data) + if ai_config is None: + raise Exception('ai recognition config is empty') - self.ai_config = AIRecognitionConfig.from_msgpack(cmd.data) self.stop_signal = False if self.session is None: self.init_ai() - for m in self.ai_config.paths: + print(ai_config.paths) + for m in ai_config.paths: if self.is_video(m): videos.append(m) else: images.append(m) - # images first, it's faster if len(images) > 0: for chunk in self.split_list_extend(images, constants.MODEL_BATCH_SIZE): print(f'run inference on {" ".join(chunk)}...') - self._process_images(cmd, chunk) + self._process_images(cmd, ai_config, chunk) if len(videos) > 0: for v in videos: print(f'run inference on {v}...') - self._process_video(cmd, v) + self._process_video(cmd, ai_config, v) - cdef _process_video(self, RemoteCommand cmd, str video_name): + cdef _process_video(self, RemoteCommand cmd, AIRecognitionConfig ai_config, str video_name): cdef int frame_count = 0 cdef list batch_frames = [] cdef list[int] batch_timestamps = [] @@ -152,30 +154,32 @@ cdef class Inference: break frame_count += 1 - if frame_count % self.ai_config.frame_period_recognition == 0: + if frame_count % ai_config.frame_period_recognition == 0: batch_frames.append(frame) batch_timestamps.append(int(v_input.get(cv2.CAP_PROP_POS_MSEC))) if len(batch_frames) == constants.MODEL_BATCH_SIZE: input_blob = self.preprocess(batch_frames) outputs = self.session.run(None, {self.model_input: input_blob}) - list_detections = self.postprocess(outputs) + list_detections = self.postprocess(outputs, ai_config) for i in range(len(list_detections)): detections = list_detections[i] annotation = Annotation(video_name, batch_timestamps[i], detections) - if self.is_valid_annotation(annotation): - _, image = cv2.imencode('.jpg', frame) + if self.is_valid_annotation(annotation, ai_config): + _, image = cv2.imencode('.jpg', batch_frames[i]) annotation.image = image.tobytes() + self._previous_annotation = annotation + print(annotation.to_str(self.class_names)) self.on_annotation(cmd, annotation) - self._previous_annotation = annotation + batch_frames.clear() batch_timestamps.clear() v_input.release() - cdef _process_images(self, RemoteCommand cmd, list[str] image_paths): + cdef _process_images(self, RemoteCommand cmd, AIRecognitionConfig ai_config, list[str] image_paths): cdef list frames = [] cdef list timestamps = [] self._previous_annotation = None @@ -186,7 +190,7 @@ cdef class Inference: input_blob = self.preprocess(frames) outputs = self.session.run(None, {self.model_input: input_blob}) - list_detections = self.postprocess(outputs) + list_detections = self.postprocess(outputs, ai_config) for i in range(len(list_detections)): detections = list_detections[i] annotation = Annotation(image_paths[i], timestamps[i], detections) @@ -198,7 +202,7 @@ cdef class Inference: cdef stop(self): self.stop_signal = True - cdef bint is_valid_annotation(self, Annotation annotation): + cdef bint is_valid_annotation(self, Annotation annotation, AIRecognitionConfig ai_config): # No detections, invalid if not annotation.detections: return False @@ -208,7 +212,7 @@ cdef class Inference: return True # Enough time has passed since last annotation - if annotation.time >= self._previous_annotation.time + (self.ai_config.frame_recognition_seconds * 1000): + if annotation.time >= self._previous_annotation.time + (ai_config.frame_recognition_seconds * 1000): return True # More objects detected than before @@ -236,11 +240,11 @@ cdef class Inference: closest_det = prev_det # Check if beyond tracking distance - if min_distance_sq > self.ai_config.tracking_distance_confidence: + if min_distance_sq > ai_config.tracking_distance_confidence: return True # Check probability increase - if current_det.confidence >= closest_det.confidence + self.ai_config.tracking_probability_increase: + if current_det.confidence >= closest_det.confidence + ai_config.tracking_probability_increase: return True return False diff --git a/Azaion.Inference/start.spec b/Azaion.Inference/start.spec index 7ce92bd..7d997c0 100644 --- a/Azaion.Inference/start.spec +++ b/Azaion.Inference/start.spec @@ -4,8 +4,6 @@ from PyInstaller.utils.hooks import collect_all datas = [] binaries = [] hiddenimports = ['constants', 'annotation', 'credentials', 'file_data', 'user', 'security', 'secure_model', 'api_client', 'hardware_service', 'remote_command', 'ai_config', 'inference', 'remote_command_handler'] -tmp_ret = collect_all('pyyaml') -datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] tmp_ret = collect_all('jwt') datas += tmp_ret[0]; binaries += tmp_ret[1]; hiddenimports += tmp_ret[2] tmp_ret = collect_all('requests') diff --git a/Azaion.Suite/App.xaml.cs b/Azaion.Suite/App.xaml.cs index 332fa60..9dfc9d7 100644 --- a/Azaion.Suite/App.xaml.cs +++ b/Azaion.Suite/App.xaml.cs @@ -51,8 +51,8 @@ public partial class App private readonly List _encryptedResources = [ - // "Azaion.Annotator", - // "Azaion.Dataset" + "Azaion.Annotator", + "Azaion.Dataset" ]; private static PythonConfig ReadPythonConfig() diff --git a/build/publish.cmd b/build/publish.cmd index b66df79..1679cd8 100644 --- a/build/publish.cmd +++ b/build/publish.cmd @@ -1,5 +1,6 @@ @echo off +cd %~dp0.. echo Build .net app dotnet build -c Release @@ -46,8 +47,8 @@ move dist\start.exe ..\dist\azaion-inference.exe copy config.yaml ..\dist echo Download onnx model -cd build -call onnx_download.exe +cd ..\build +call onnx_downloader.exe move azaion.onnx.big ..\dist\ cd ..