Push model to docker registry

src/engines/tensorrt_engine.pxd

@@ -4,6 +4,8 @@ from engines.inference_engine cimport InferenceEngine
 cdef class TensorRTEngine(InferenceEngine):
 
     cdef public object context
+    cdef object cuda_context
+    cdef object cuda_lock
 
     cdef public object d_input
     cdef public object d_output

src/engines/tensorrt_engine.pyx

@@ -1,10 +1,10 @@
 from engines.inference_engine cimport InferenceEngine
 import tensorrt as trt  # pyright: ignore[reportMissingImports]
 import pycuda.driver as cuda  # pyright: ignore[reportMissingImports]
-import pycuda.autoinit  # pyright: ignore[reportMissingImports]
 import pynvml
 import numpy as np
 import os
+import threading
 cimport constants_inf
 
 GPU_MEMORY_FRACTION = 0.8
@@ -32,6 +32,11 @@ class _CacheCalibrator(trt.IInt8EntropyCalibrator2):
 cdef class TensorRTEngine(InferenceEngine):
     def __init__(self, model_bytes: bytes, max_batch_size: int = 8, **kwargs):
         InferenceEngine.__init__(self, model_bytes, max_batch_size, engine_name="tensorrt")
+        self.cuda_context = TensorRTEngine.create_cuda_context()
+        self.cuda_lock = threading.Lock()
+        try:
+            with self.cuda_lock:
+                self.cuda_context.push()
                 try:
                     logger = trt.Logger(trt.Logger.WARNING)
                     runtime = trt.Runtime(logger)
@@ -70,10 +75,21 @@ cdef class TensorRTEngine(InferenceEngine):
                     self.d_output = cuda.mem_alloc(self.h_output.nbytes)
 
                     self.stream = cuda.Stream()
-
+                finally:
+                    try:
+                        self.cuda_context.pop()
+                    except Exception:
+                        pass
         except Exception as e:
             raise RuntimeError(f"Failed to initialize TensorRT engine: {str(e)}")
 
+    def __dealloc__(self):
+        try:
+            if self.cuda_context is not None:
+                self.cuda_context.detach()
+        except Exception:
+            pass
+
     @staticmethod
     def calculate_max_batch_size(gpu_memory_bytes, int input_h, int input_w):
         frame_input_bytes = 3 * input_h * input_w * 4
@@ -99,9 +115,18 @@ cdef class TensorRTEngine(InferenceEngine):
                 pass
         return 2 * 1024 * 1024 * 1024 if total_memory is None else total_memory
 
+    @staticmethod
+    def create_cuda_context():
+        cuda.init()
+        from engines import tensor_gpu_index
+        ctx = cuda.Device(max(tensor_gpu_index, 0)).make_context()
+        ctx.pop()
+        return ctx
+
     @staticmethod
     def get_engine_filename(str precision="fp16"):
         try:
+            cuda.init()
             from engines import tensor_gpu_index
             device = cuda.Device(max(tensor_gpu_index, 0))
             sm_count = device.multiprocessor_count
@@ -115,6 +140,8 @@ cdef class TensorRTEngine(InferenceEngine):
 
     @staticmethod
     def convert_from_source(bytes onnx_model, str calib_cache_path=None, bint force_static_input=False):
+        cuda_context = TensorRTEngine.create_cuda_context()
+        cuda_context.push()
         gpu_mem = TensorRTEngine.get_gpu_memory_bytes(0)
         workspace_bytes = int(gpu_mem * 0.9)
 
@@ -129,6 +156,7 @@ cdef class TensorRTEngine(InferenceEngine):
             except Exception as e:
                 constants_inf.logerror(<str>f'ONNX TensorRT compatibility preparation failed: {str(e)}')
 
+        try:
             with trt.Builder(trt_logger) as builder, \
                     builder.create_network(explicit_batch_flag) as network, \
                     trt.OnnxParser(network, trt_logger) as parser, \
@@ -180,11 +208,23 @@ cdef class TensorRTEngine(InferenceEngine):
                     return None
                 constants_inf.log('conversion done!')
                 return bytes(plan)
+        finally:
+            try:
+                cuda_context.pop()
+            except Exception:
+                pass
+            try:
+                cuda_context.detach()
+            except Exception:
+                pass
 
     cdef tuple get_input_shape(self):
         return <tuple>(self.input_shape[2], self.input_shape[3])
 
     cdef run(self, input_data):
+        try:
+            with self.cuda_lock:
+                self.cuda_context.push()
                 try:
                     actual_batch = input_data.shape[0]
                     if actual_batch != self.input_shape[0]:
@@ -202,6 +242,11 @@ cdef class TensorRTEngine(InferenceEngine):
                     output_shape = [actual_batch, self.output_shape[1], self.output_shape[2]]
                     output = self.h_output[:actual_batch].reshape(output_shape)
                     return [output]
+                finally:
+                    try:
+                        self.cuda_context.pop()
+                    except Exception:
+                        pass
 
         except Exception as e:
             raise RuntimeError(f"Failed to run TensorRT inference: {str(e)}")