Feature/run jetson e2e tests (#4)

* Run tests

* Run tests

* Run tests

* Run tests

* Added rebuild

* Added files for e2e tests

* Added rebuild

* Added rebuild

* Added biuld TensorRT flag

* Changed to use NumPy 1.x for Jetson

* Make universal invocation

* Make Cython constans

* Changed to prepare onnx

* Changed smoke-test to wait AI conversion

* Added step for model conversion

* Changed to not run step in parallel

* Push model to docker registry

* Push model to docker registry

* Push model to docker registry

e2e/docker-compose.test.yml

@@ -2,9 +2,9 @@ name: detections-e2e
 
 services:
   mock-loader:
-    build: ./mocks/loader
-    volumes:
-      - ./fixtures:/models
+    build:
+      context: .
+      dockerfile: mocks/loader/Dockerfile
     networks:
       - e2e-net
 
@@ -74,9 +74,7 @@ services:
       JWT_SECRET: test-secret-e2e-only
       CLASSES_JSON_PATH: /app/classes.json
     volumes:
-      - ./fixtures/classes.json:/app/classes.json:ro
       - ./fixtures:/media:ro
-      - ./logs:/app/Logs
     shm_size: 512m
     networks:
       e2e-net:
@@ -94,6 +92,7 @@ services:
       - mock-annotations
     environment:
       JWT_SECRET: test-secret-e2e-only
+      MEDIA_DIR: /app/fixtures
     volumes:
       - ./fixtures:/media
       - ./results:/results

e2e/engine-artifact.Dockerfile

@@ -0,0 +1,5 @@
+FROM alpine:3.20
+
+COPY . /models/
+
+CMD ["sh"]

e2e/mocks/loader/Dockerfile

@@ -1,6 +1,7 @@
 FROM python:3.11-slim
 WORKDIR /app
 RUN pip install --no-cache-dir flask gunicorn
-COPY app.py .
+COPY mocks/loader/app.py .
+COPY fixtures /models
 EXPOSE 8080
 CMD ["gunicorn", "-b", "0.0.0.0:8080", "-w", "1", "--timeout", "120", "app:app"]

e2e/mocks/loader/app.py

@@ -31,6 +31,16 @@ def _resolve_disk_path(filename: str, folder: str | None) -> Path | None:
     return None
 
 
+def _write_disk_path(filename: str, folder: str | None, data: bytes) -> Path:
+    root = _models_root()
+    safe_filename = Path(filename).name
+    target_dir = root / folder if folder else root
+    target_dir.mkdir(parents=True, exist_ok=True)
+    target = target_dir / safe_filename
+    target.write_bytes(data)
+    return target
+
+
 def _should_fail_load() -> bool:
     global _first_fail_remaining
     if _mode == "error":
@@ -73,7 +83,9 @@ def upload(filename):
     f = request.files.get("data")
     if not f:
         return "", 400
-    _uploads[(folder, filename)] = f.read()
+    data = f.read()
+    _uploads[(folder, filename)] = data
+    _write_disk_path(filename, folder, data)
     _upload_count += 1
     return "", 200
 

e2e/requirements.txt

@@ -1,6 +1,7 @@
 pytest
 pytest-csv
 requests==2.32.4
+PyJWT==2.12.1
 sseclient-py
 pytest-timeout
 flask

e2e/run_test.sh

@@ -19,6 +19,14 @@ case "$PROFILE" in
 esac
 
 COMPOSE="docker compose -f docker-compose.test.yml --profile $PROFILE"
+LOG_TAIL="${E2E_LOG_TAIL:-100}"
+RUNNER_ENV_ARGS=(-e E2E_PROFILE="$PROFILE")
+if [[ "$PROFILE" == "jetson" ]]; then
+  RUNNER_ENV_ARGS+=(
+    -e E2E_WAIT_FOR_ENGINE_ENABLED="${E2E_WAIT_FOR_ENGINE_ENABLED:-0}"
+    -e E2E_ENGINE_WAIT_TIMEOUT="${E2E_ENGINE_WAIT_TIMEOUT:-900}"
+  )
+fi
 
 usage() {
   echo "Usage: $0 <test_path> [pytest_args...]"
@@ -46,7 +54,7 @@ for i in $(seq 1 60); do
   fi
   if [[ "$i" == "60" ]]; then
     echo "ERROR: detections service did not become healthy"
-    $COMPOSE logs "$DETECTIONS_SERVICE" --tail 100
+    $COMPOSE logs "$DETECTIONS_SERVICE" --tail "$LOG_TAIL"
     exit 1
   fi
   sleep 2
@@ -54,11 +62,11 @@ done
 
 echo "--- Running: pytest $* -v -x -s --csv=/results/report.csv"
 set +e
-$COMPOSE run --rm --no-deps e2e-runner pytest "$@" -v -x -s --csv=/results/report.csv
+$COMPOSE run --rm --build --no-deps "${RUNNER_ENV_ARGS[@]}" e2e-runner pytest "$@" -v -x -s --csv=/results/report.csv
 EXIT_CODE=$?
 set -e
 
 echo "--- Test finished with exit code $EXIT_CODE"
-echo "--- Detections logs (last 100 lines):"
-$COMPOSE logs "$DETECTIONS_SERVICE" --tail 100
+echo "--- Detections logs (last $LOG_TAIL lines):"
+$COMPOSE logs "$DETECTIONS_SERVICE" --tail "$LOG_TAIL"
 exit $EXIT_CODE

e2e/scripts/publish_jetson_engine.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+COMPOSE="${COMPOSE:-docker compose -f docker-compose.test.yml --profile jetson}"
+REGISTRY_HOST="${REGISTRY_HOST:?REGISTRY_HOST is required}"
+ENGINE_REPOSITORY="${JETSON_ENGINE_REPOSITORY:-$REGISTRY_HOST/azaion/detections-jetson-engine}"
+BRANCH="${CI_COMMIT_BRANCH:-local}"
+ENGINE_TAG="${JETSON_ENGINE_TAG:-$(printf '%s' "$BRANCH" | tr -c 'A-Za-z0-9_.-' '-')}"
+OUT_DIR="${JETSON_ENGINE_OUT_DIR:-results/jetson-engine}"
+
+mkdir -p "$OUT_DIR/models"
+
+loader_id="$($COMPOSE ps -q mock-loader)"
+if [[ -z "$loader_id" ]]; then
+  echo "ERROR: mock-loader container is not running"
+  exit 1
+fi
+
+docker cp "$loader_id:/models/models/." "$OUT_DIR/models/"
+find "$OUT_DIR/models" -maxdepth 1 -type f ! -name 'azaion*.engine' -delete
+
+engine_count="$(find "$OUT_DIR/models" -maxdepth 1 -type f -name 'azaion*.engine' | wc -l | tr -d ' ')"
+if [[ "$engine_count" == "0" ]]; then
+  echo "ERROR: no converted TensorRT engine found in mock-loader /models/models"
+  find "$OUT_DIR/models" -maxdepth 2 -type f -print
+  exit 1
+fi
+
+echo "--- Converted TensorRT engine files:"
+find "$OUT_DIR/models" -maxdepth 1 -type f -name 'azaion*.engine' -print -exec ls -lh {} \;
+
+image="$ENGINE_REPOSITORY:$ENGINE_TAG"
+echo "--- Building Jetson engine artifact image: $image"
+docker build -f engine-artifact.Dockerfile -t "$image" "$OUT_DIR/models"
+docker push "$image"
+
+if [[ -n "${CI_COMMIT_SHA:-}" ]]; then
+  sha_tag="$(printf '%s' "$CI_COMMIT_SHA" | cut -c1-12)"
+  docker tag "$image" "$ENGINE_REPOSITORY:$sha_tag"
+  docker push "$ENGINE_REPOSITORY:$sha_tag"
+fi
+
+echo "--- Published Jetson engine artifact image: $image"

e2e/scripts/pull_jetson_engine.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ -z "${REGISTRY_HOST:-}" ]]; then
+  echo "--- REGISTRY_HOST is not set; skipping Jetson engine artifact pull"
+  exit 0
+fi
+
+ENGINE_REPOSITORY="${JETSON_ENGINE_REPOSITORY:-$REGISTRY_HOST/azaion/detections-jetson-engine}"
+BRANCH="${CI_COMMIT_BRANCH:-local}"
+ENGINE_TAG="${JETSON_ENGINE_TAG:-$(printf '%s' "$BRANCH" | tr -c 'A-Za-z0-9_.-' '-')}"
+TARGET_DIR="${JETSON_ENGINE_TARGET_DIR:-fixtures/models}"
+image="$ENGINE_REPOSITORY:$ENGINE_TAG"
+
+echo "--- Pulling Jetson engine artifact image: $image"
+if ! docker pull "$image"; then
+  echo "--- Jetson engine artifact image not found; smoke will use ONNX fallback"
+  exit 0
+fi
+
+cid="$(docker create "$image")"
+trap 'docker rm -f "$cid" >/dev/null 2>&1 || true' EXIT
+
+mkdir -p "$TARGET_DIR"
+docker cp "$cid:/models/." "$TARGET_DIR/"
+
+echo "--- Installed Jetson engine files:"
+find "$TARGET_DIR" -maxdepth 1 -type f -name 'azaion*.engine' -print -exec ls -lh {} \;

e2e/tests/test_health_engine.py

@@ -1,4 +1,5 @@
 import json
+import os
 import threading
 import time
 import uuid
@@ -7,6 +8,13 @@ import pytest
 import sseclient
 
 _DETECT_TIMEOUT = 60
+_ENGINE_WAIT_TIMEOUT = int(os.environ.get("E2E_ENGINE_WAIT_TIMEOUT", "900"))
+_WAIT_FOR_ENGINE_ENABLED = os.environ.get("E2E_WAIT_FOR_ENGINE_ENABLED", "").lower() in (
+    "1",
+    "true",
+    "yes",
+    "on",
+)
 
 
 def _get_health(http_client):
@@ -20,6 +28,24 @@ def _assert_active_ai(data):
     assert data["aiAvailability"] not in ("None", "Downloading")
 
 
+def _wait_for_engine_enabled(http_client):
+    deadline = time.time() + _ENGINE_WAIT_TIMEOUT
+    last = None
+    while time.time() < deadline:
+        last = _get_health(http_client)
+        availability = last.get("aiAvailability")
+        if availability == "Enabled":
+            assert last.get("errorMessage") is None
+            return last
+        if availability == "Error":
+            pytest.fail(f"engine conversion failed: {last.get('errorMessage')}")
+        time.sleep(3)
+    pytest.fail(
+        f"engine did not become Enabled within {_ENGINE_WAIT_TIMEOUT}s "
+        f"(last health: {last})"
+    )
+
+
 @pytest.mark.cpu
 class TestHealthEngineStep01PreInit:
     def test_ft_p_01_pre_init_health(self, http_client):
@@ -92,8 +118,12 @@ class TestHealthEngineStep03Warmed:
     def _warm(self, warm_engine):
         pass
 
+    @pytest.mark.timeout(_ENGINE_WAIT_TIMEOUT + 30)
     def test_ft_p_02_post_init_health(self, http_client):
-        data = _get_health(http_client)
+        if _WAIT_FOR_ENGINE_ENABLED:
+            data = _wait_for_engine_enabled(http_client)
+        else:
+            data = _get_health(http_client)
         _assert_active_ai(data)
         assert data.get("errorMessage") is None