From e9a44e368d1a38364d5cba9f5eae8ae8fb581823 Mon Sep 17 00:00:00 2001 From: Alex Bezdieniezhnykh Date: Thu, 24 Apr 2025 16:30:21 +0300 Subject: [PATCH] autoconvert tensor rt engine from onnx to specific CUDA gpu --- Azaion.Annotator/Controls/CircleVisual.cs | 6 +++--- Azaion.Common/Constants.cs | 2 +- Azaion.CommonSecurity/DTO/ApiCredentials.cs | 3 --- Azaion.CommonSecurity/Services/AuthProvider.cs | 2 +- Azaion.Inference/annotation.pxd | 2 -- Azaion.Inference/annotation.pyx | 4 ++-- Azaion.Inference/api_client.pyx | 2 +- Azaion.Inference/inference.pxd | 1 - Azaion.Inference/inference.pyx | 14 +++++--------- Azaion.Inference/inference_engine.pxd | 3 --- Azaion.Inference/inference_engine.pyx | 18 ++++++------------ Azaion.Inference/remote_command_handler.pyx | 2 +- Azaion.Suite/App.xaml.cs | 2 +- Azaion.Suite/MainSuite.xaml.cs | 8 ++++---- 14 files changed, 25 insertions(+), 44 deletions(-) diff --git a/Azaion.Annotator/Controls/CircleVisual.cs b/Azaion.Annotator/Controls/CircleVisual.cs index b7ff86e..82239a6 100644 --- a/Azaion.Annotator/Controls/CircleVisual.cs +++ b/Azaion.Annotator/Controls/CircleVisual.cs @@ -80,7 +80,7 @@ namespace Azaion.Annotator.Controls FontWeights.Bold, FontStretches.Normal); - FormattedText _fText; + FormattedText _fText = null!; private Brush _background = Brushes.Blue; @@ -187,7 +187,7 @@ namespace Azaion.Annotator.Controls IsChanged = true; } - string _text; + string _text = null!; public string Text { @@ -205,7 +205,7 @@ namespace Azaion.Annotator.Controls } } - Visual _child; + Visual _child = null!; public virtual Visual Child { diff --git a/Azaion.Common/Constants.cs b/Azaion.Common/Constants.cs index 917f3b1..fe55ccd 100644 --- a/Azaion.Common/Constants.cs +++ b/Azaion.Common/Constants.cs @@ -33,7 +33,7 @@ public class Constants AnnotationsDbFile = DEFAULT_ANNOTATIONS_DB_FILE }; - public static readonly List DefaultAnnotationClasses = + private static readonly List DefaultAnnotationClasses = [ new() { Id = 0, Name = "ArmorVehicle", ShortName = "Броня", Color = "#FF0000".ToColor() }, new() { Id = 1, Name = "Truck", ShortName = "Вантаж.", Color = "#00FF00".ToColor() }, diff --git a/Azaion.CommonSecurity/DTO/ApiCredentials.cs b/Azaion.CommonSecurity/DTO/ApiCredentials.cs index d16dd52..77fca61 100644 --- a/Azaion.CommonSecurity/DTO/ApiCredentials.cs +++ b/Azaion.CommonSecurity/DTO/ApiCredentials.cs @@ -10,7 +10,4 @@ public class ApiCredentials(string email, string password) : EventArgs [Key(nameof(Password))] public string Password { get; set; } = password; - - [Key(nameof(Folder))] - public string Folder { get; set; } = null!; } \ No newline at end of file diff --git a/Azaion.CommonSecurity/Services/AuthProvider.cs b/Azaion.CommonSecurity/Services/AuthProvider.cs index c345927..a729a6f 100644 --- a/Azaion.CommonSecurity/Services/AuthProvider.cs +++ b/Azaion.CommonSecurity/Services/AuthProvider.cs @@ -70,7 +70,7 @@ public class AzaionApi(HttpClient client, ICache cache, ApiCredentials credentia if (response.StatusCode == HttpStatusCode.Conflict) { var result = JsonConvert.DeserializeObject(content); - throw new Exception($"Failed: {response.StatusCode}! Error Code: {result.ErrorCode}. Message: {result.Message}"); + throw new Exception($"Failed: {response.StatusCode}! Error Code: {result?.ErrorCode}. Message: {result?.Message}"); } throw new Exception($"Failed: {response.StatusCode}! Result: {content}"); } diff --git a/Azaion.Inference/annotation.pxd b/Azaion.Inference/annotation.pxd index e8af677..b8b1b34 100644 --- a/Azaion.Inference/annotation.pxd +++ b/Azaion.Inference/annotation.pxd @@ -14,5 +14,3 @@ cdef class Annotation: cdef format_time(self, ms) cdef bytes serialize(self) - cdef to_str(self, class_names) - diff --git a/Azaion.Inference/annotation.pyx b/Azaion.Inference/annotation.pyx index 4614156..1d4f481 100644 --- a/Azaion.Inference/annotation.pyx +++ b/Azaion.Inference/annotation.pyx @@ -32,12 +32,12 @@ cdef class Annotation: d.annotation_name = self.name self.image = b'' - cdef to_str(self, class_names): + def __str__(self): if not self.detections: return f"{self.name}: No detections" detections_str = ", ".join( - f"{class_names[d.cls]} {d.confidence * 100:.1f}% ({d.x:.2f}, {d.y:.2f})" + f"class: {d.cls} {d.confidence * 100:.1f}% ({d.x:.2f}, {d.y:.2f})" for d in self.detections ) return f"{self.name}: {detections_str}" diff --git a/Azaion.Inference/api_client.pyx b/Azaion.Inference/api_client.pyx index c933ae5..f0d533c 100644 --- a/Azaion.Inference/api_client.pyx +++ b/Azaion.Inference/api_client.pyx @@ -79,7 +79,7 @@ cdef class ApiClient: try: r = requests.post(url, headers=headers, files=files, allow_redirects=True) r.raise_for_status() - print(f"Upload success: {r.status_code}") + print(f"Uploaded {filename} to {constants.API_URL}/{folder} successfully: {r.status_code}.") except Exception as e: print(f"Upload fail: {e}") diff --git a/Azaion.Inference/inference.pxd b/Azaion.Inference/inference.pxd index ec2f2e0..f0cfc39 100644 --- a/Azaion.Inference/inference.pxd +++ b/Azaion.Inference/inference.pxd @@ -10,7 +10,6 @@ cdef class Inference: cdef object on_annotation cdef Annotation _previous_annotation cdef AIRecognitionConfig ai_config - cdef object class_names cdef bint stop_signal cdef str model_input diff --git a/Azaion.Inference/inference.pyx b/Azaion.Inference/inference.pyx index c32249a..2a0c6e6 100644 --- a/Azaion.Inference/inference.pyx +++ b/Azaion.Inference/inference.pyx @@ -24,7 +24,6 @@ cdef class Inference: self.model_width = 0 self.model_height = 0 self.engine = None - self.class_names = None self.is_building_engine = False cdef build_tensor_engine(self): @@ -37,9 +36,11 @@ cdef class Inference: models_dir = constants.MODELS_FOLDER if not os.path.exists(os.path.join( models_dir, f'{engine_filename}.big')): self.is_building_engine = True + time.sleep(5) # prevent simultaneously loading dll and models onnx_model = self.api_client.load_big_small_resource(constants.AI_ONNX_MODEL_FILE, models_dir, key) model_bytes = TensorRTEngine.convert_from_onnx(onnx_model) self.api_client.upload_big_small_resource(model_bytes, engine_filename, models_dir, key) + print('uploaded ') self.is_building_engine = False @@ -62,7 +63,6 @@ cdef class Inference: self.engine = OnnxEngine(model_bytes) self.model_height, self.model_width = self.engine.get_input_shape() - self.class_names = self.engine.get_class_names() cdef preprocess(self, frames): blobs = [cv2.dnn.blobFromImage(frame, @@ -75,13 +75,11 @@ cdef class Inference: return np.vstack(blobs) cdef postprocess(self, output, ai_config): - print('enter postprocess') cdef list[Detection] detections = [] cdef int ann_index cdef float x1, y1, x2, y2, conf, cx, cy, w, h cdef int class_id cdef list[list[Detection]] results = [] - print('start try: code') try: for ann_index in range(len(output[0])): detections.clear() @@ -163,7 +161,7 @@ cdef class Inference: images.append(m) # images first, it's faster if len(images) > 0: - for chunk in self.split_list_extend(images, self.engine.get_input_shape()): + for chunk in self.split_list_extend(images, self.engine.get_batch_size()): print(f'run inference on {" ".join(chunk)}...') self._process_images(cmd, ai_config, chunk) if len(videos) > 0: @@ -189,7 +187,7 @@ cdef class Inference: batch_frames.append(frame) batch_timestamps.append(int(v_input.get(cv2.CAP_PROP_POS_MSEC))) - if len(batch_frames) == self.engine.get_input_shape(): + if len(batch_frames) == self.engine.get_batch_size(): input_blob = self.preprocess(batch_frames) outputs = self.engine.run(input_blob) @@ -203,10 +201,9 @@ cdef class Inference: annotation.image = image.tobytes() self._previous_annotation = annotation - print(annotation.to_str(self.class_names)) + print(annotation) self.on_annotation(cmd, annotation) - batch_frames.clear() batch_timestamps.clear() v_input.release() @@ -231,7 +228,6 @@ cdef class Inference: annotation = Annotation(image_paths[i], timestamps[i], detections) _, image = cv2.imencode('.jpg', frames[i]) annotation.image = image.tobytes() - print(annotation.to_str(self.class_names)) self.on_annotation(cmd, annotation) diff --git a/Azaion.Inference/inference_engine.pxd b/Azaion.Inference/inference_engine.pxd index 0e74a46..15799a0 100644 --- a/Azaion.Inference/inference_engine.pxd +++ b/Azaion.Inference/inference_engine.pxd @@ -6,7 +6,6 @@ cdef class InferenceEngine: cdef public int batch_size cdef tuple get_input_shape(self) cdef int get_batch_size(self) - cdef get_class_names(self) cpdef run(self, input_data) cdef class OnnxEngine(InferenceEngine): @@ -14,7 +13,6 @@ cdef class OnnxEngine(InferenceEngine): cdef list model_inputs cdef str input_name cdef object input_shape - cdef object class_names cdef class TensorRTEngine(InferenceEngine): cdef object stream @@ -26,7 +24,6 @@ cdef class TensorRTEngine(InferenceEngine): cdef object input_shape cdef object output_shape cdef object h_output - cdef object class_names @staticmethod cdef bytes convert_from_onnx(bytes onnx_model) diff --git a/Azaion.Inference/inference_engine.pyx b/Azaion.Inference/inference_engine.pyx index 6c61138..6617279 100644 --- a/Azaion.Inference/inference_engine.pyx +++ b/Azaion.Inference/inference_engine.pyx @@ -22,9 +22,6 @@ cdef class InferenceEngine: cpdef run(self, input_data): raise NotImplementedError("Subclass must implement run") - cdef get_class_names(self): - raise NotImplementedError("Subclass must implement get_class_names") - cdef class OnnxEngine(InferenceEngine): def __init__(self, model_bytes: bytes, batch_size: int = 1, **kwargs): @@ -39,7 +36,6 @@ cdef class OnnxEngine(InferenceEngine): print(f'AI detection model input: {self.model_inputs} {self.input_shape}') model_meta = self.session.get_modelmeta() print("Metadata:", model_meta.custom_metadata_map) - self.class_names = eval(model_meta.custom_metadata_map["names"]) cdef tuple get_input_shape(self): shape = self.input_shape @@ -48,9 +44,6 @@ cdef class OnnxEngine(InferenceEngine): cdef int get_batch_size(self): return self.batch_size - cdef get_class_names(self): - return self.class_names - cpdef run(self, input_data): return self.session.run(None, {self.input_name: input_data}) @@ -69,10 +62,14 @@ cdef class TensorRTEngine(InferenceEngine): raise RuntimeError(f"Failed to load TensorRT engine from bytes") self.context = engine.create_execution_context() + # input self.input_name = engine.get_tensor_name(0) engine_input_shape = engine.get_tensor_shape(self.input_name) - self.batch_size = self.input_shape[0] if self.input_shape[0] != -1 else batch_size + if engine_input_shape[0] != -1: + self.batch_size = engine_input_shape[0] + else: + self.batch_size = batch_size self.input_shape = [ self.batch_size, @@ -154,7 +151,7 @@ cdef class TensorRTEngine(InferenceEngine): if plan is None: print('Conversion failed.') return None - + print('conversion done!') return bytes(plan) cdef tuple get_input_shape(self): @@ -163,9 +160,6 @@ cdef class TensorRTEngine(InferenceEngine): cdef int get_batch_size(self): return self.batch_size - cdef get_class_names(self): - return self.class_names - cpdef run(self, input_data): try: cuda.memcpy_htod_async(self.d_input, input_data, self.stream) diff --git a/Azaion.Inference/remote_command_handler.pyx b/Azaion.Inference/remote_command_handler.pyx index 42c64ce..3cefff3 100644 --- a/Azaion.Inference/remote_command_handler.pyx +++ b/Azaion.Inference/remote_command_handler.pyx @@ -76,7 +76,7 @@ cdef class RemoteCommandHandler: with self._context.socket(zmq.DEALER) as socket: socket.connect("inproc://backend") socket.send_multipart([client_id, data]) - constants.log(f'Sent {len(data)} bytes.', client_id) + # constants.log(f'Sent {len(data)} bytes.', client_id) cdef stop(self): self._shutdown_event.set() diff --git a/Azaion.Suite/App.xaml.cs b/Azaion.Suite/App.xaml.cs index cf16aba..88d463c 100644 --- a/Azaion.Suite/App.xaml.cs +++ b/Azaion.Suite/App.xaml.cs @@ -94,7 +94,7 @@ public partial class App _resourceLoader = new ResourceLoader(_inferenceClient); var login = new Login(); - login.CredentialsEntered += async (_, credentials) => + login.CredentialsEntered += (_, credentials) => { _inferenceClient.Send(RemoteCommand.Create(CommandType.Login, credentials)); _azaionApi = new AzaionApi(new HttpClient { BaseAddress = new Uri(SecurityConstants.API_URL) }, _cache, credentials, _hardwareService); diff --git a/Azaion.Suite/MainSuite.xaml.cs b/Azaion.Suite/MainSuite.xaml.cs index c377f00..761dcf4 100644 --- a/Azaion.Suite/MainSuite.xaml.cs +++ b/Azaion.Suite/MainSuite.xaml.cs @@ -50,9 +50,9 @@ public partial class MainSuite Loaded += OnLoaded; Closed += OnFormClosed; - SizeChanged += async (_, _) => await SaveUserSettings(); - LocationChanged += async (_, _) => await SaveUserSettings(); - StateChanged += async (_, _) => await SaveUserSettings(); + SizeChanged += (_, _) => SaveUserSettings(); + LocationChanged += (_, _) => SaveUserSettings(); + StateChanged += (_, _) => SaveUserSettings(); Left = (SystemParameters.WorkArea.Width - Width) / 2; } @@ -130,7 +130,7 @@ public partial class MainSuite } - private async Task SaveUserSettings() + private void SaveUserSettings() { ThrottleExt.Throttle(() => {