autoconvert tensor rt engine from onnx to specific CUDA gpu

Azaion.Inference/inference.pyx

@@ -24,7 +24,6 @@ cdef class Inference:
         self.model_width = 0
         self.model_height = 0
         self.engine = None
-        self.class_names = None
         self.is_building_engine = False
 
     cdef build_tensor_engine(self):
@@ -37,9 +36,11 @@ cdef class Inference:
         models_dir = constants.MODELS_FOLDER
         if not os.path.exists(os.path.join(<str> models_dir, f'{engine_filename}.big')):
             self.is_building_engine = True
+            time.sleep(5) # prevent simultaneously loading dll and models
             onnx_model = self.api_client.load_big_small_resource(constants.AI_ONNX_MODEL_FILE, models_dir, key)
             model_bytes = TensorRTEngine.convert_from_onnx(onnx_model)
             self.api_client.upload_big_small_resource(model_bytes, <str> engine_filename, models_dir, key)
+            print('uploaded ')
             self.is_building_engine = False
 
 
@@ -62,7 +63,6 @@ cdef class Inference:
             self.engine = OnnxEngine(model_bytes)
 
         self.model_height, self.model_width = self.engine.get_input_shape()
-        self.class_names = self.engine.get_class_names()
 
     cdef preprocess(self, frames):
         blobs = [cv2.dnn.blobFromImage(frame,
@@ -75,13 +75,11 @@ cdef class Inference:
         return np.vstack(blobs)
 
     cdef postprocess(self, output, ai_config):
-        print('enter postprocess')
         cdef list[Detection] detections = []
         cdef int ann_index
         cdef float x1, y1, x2, y2, conf, cx, cy, w, h
         cdef int class_id
         cdef list[list[Detection]] results = []
-        print('start try: code')
         try:
             for ann_index in range(len(output[0])):
                 detections.clear()
@@ -163,7 +161,7 @@ cdef class Inference:
                 images.append(m)
         # images first, it's faster
         if len(images) > 0:
-            for chunk in self.split_list_extend(images, self.engine.get_input_shape()):
+            for chunk in self.split_list_extend(images, self.engine.get_batch_size()):
                 print(f'run inference on {" ".join(chunk)}...')
                 self._process_images(cmd, ai_config, chunk)
         if len(videos) > 0:
@@ -189,7 +187,7 @@ cdef class Inference:
                 batch_frames.append(frame)
                 batch_timestamps.append(int(v_input.get(cv2.CAP_PROP_POS_MSEC)))
 
-            if len(batch_frames) == self.engine.get_input_shape():
+            if len(batch_frames) == self.engine.get_batch_size():
                 input_blob = self.preprocess(batch_frames)
 
                 outputs = self.engine.run(input_blob)
@@ -203,10 +201,9 @@ cdef class Inference:
                         annotation.image = image.tobytes()
                         self._previous_annotation = annotation
 
-                        print(annotation.to_str(self.class_names))
+                        print(annotation)
                         self.on_annotation(cmd, annotation)
 
-
                 batch_frames.clear()
                 batch_timestamps.clear()
         v_input.release()
@@ -231,7 +228,6 @@ cdef class Inference:
             annotation = Annotation(image_paths[i], timestamps[i], detections)
             _, image = cv2.imencode('.jpg', frames[i])
             annotation.image = image.tobytes()
-            print(annotation.to_str(self.class_names))
             self.on_annotation(cmd, annotation)