Feature/run jetson e2e tests (#4)

* Run tests

* Run tests

* Run tests

* Run tests

* Added rebuild

* Added files for e2e tests

* Added rebuild

* Added rebuild

* Added biuld TensorRT flag

* Changed to use NumPy 1.x for Jetson

* Make universal invocation

* Make Cython constans

* Changed to prepare onnx

* Changed smoke-test to wait AI conversion

* Added step for model conversion

* Changed to not run step in parallel

* Push model to docker registry

* Push model to docker registry

* Push model to docker registry

e2e/tests/test_health_engine.py

@@ -1,4 +1,5 @@
 import json
+import os
 import threading
 import time
 import uuid
@@ -7,6 +8,13 @@ import pytest
 import sseclient
 
 _DETECT_TIMEOUT = 60
+_ENGINE_WAIT_TIMEOUT = int(os.environ.get("E2E_ENGINE_WAIT_TIMEOUT", "900"))
+_WAIT_FOR_ENGINE_ENABLED = os.environ.get("E2E_WAIT_FOR_ENGINE_ENABLED", "").lower() in (
+    "1",
+    "true",
+    "yes",
+    "on",
+)
 
 
 def _get_health(http_client):
@@ -20,6 +28,24 @@ def _assert_active_ai(data):
     assert data["aiAvailability"] not in ("None", "Downloading")
 
 
+def _wait_for_engine_enabled(http_client):
+    deadline = time.time() + _ENGINE_WAIT_TIMEOUT
+    last = None
+    while time.time() < deadline:
+        last = _get_health(http_client)
+        availability = last.get("aiAvailability")
+        if availability == "Enabled":
+            assert last.get("errorMessage") is None
+            return last
+        if availability == "Error":
+            pytest.fail(f"engine conversion failed: {last.get('errorMessage')}")
+        time.sleep(3)
+    pytest.fail(
+        f"engine did not become Enabled within {_ENGINE_WAIT_TIMEOUT}s "
+        f"(last health: {last})"
+    )
+
+
 @pytest.mark.cpu
 class TestHealthEngineStep01PreInit:
     def test_ft_p_01_pre_init_health(self, http_client):
@@ -92,8 +118,12 @@ class TestHealthEngineStep03Warmed:
     def _warm(self, warm_engine):
         pass
 
+    @pytest.mark.timeout(_ENGINE_WAIT_TIMEOUT + 30)
     def test_ft_p_02_post_init_health(self, http_client):
-        data = _get_health(http_client)
+        if _WAIT_FOR_ENGINE_ENABLED:
+            data = _wait_for_engine_enabled(http_client)
+        else:
+            data = _get_health(http_client)
         _assert_active_ai(data)
         assert data.get("errorMessage") is None