[AZ-143] [AZ-145] [AZ-148] Implement video, resilience, and resource limit integration tests

Made-with: Cursor

e2e/tests/test_resilience.py

@@ -1 +1,194 @@
-"""Loader and annotations outage modes, retries, and degraded behavior."""
+import json
+import threading
+import time
+import uuid
+
+import pytest
+import requests
+
+_DETECT_TIMEOUT = 60
+
+
+def _ai_config_video(mock_loader_url: str) -> dict:
+    base = mock_loader_url.rstrip("/")
+    return {
+        "probability_threshold": 0.25,
+        "tracking_intersection_threshold": 0.6,
+        "altitude": 400,
+        "focal_length": 24,
+        "sensor_width": 23.5,
+        "paths": [f"{base}/load/video_short01.mp4"],
+        "frame_period_recognition": 4,
+        "frame_recognition_seconds": 2,
+    }
+
+
+def test_ft_n_06_loader_unreachable_during_init_health(
+    http_client, mock_loader_url, image_small
+):
+    h0 = http_client.get("/health")
+    h0.raise_for_status()
+    if h0.json().get("aiAvailability") != "None":
+        pytest.skip("engine already warm")
+    requests.post(
+        f"{mock_loader_url}/mock/config", json={"mode": "error"}, timeout=10
+    ).raise_for_status()
+    files = {"file": ("n06.jpg", image_small, "image/jpeg")}
+    r = http_client.post("/detect", files=files, timeout=_DETECT_TIMEOUT)
+    assert r.status_code != 500
+    h = http_client.get("/health")
+    assert h.status_code == 200
+    d = h.json()
+    assert d["status"] == "healthy"
+    assert d.get("errorMessage") is None
+
+
+@pytest.mark.slow
+@pytest.mark.timeout(120)
+def test_ft_n_07_annotations_unreachable_detection_continues(
+    warm_engine,
+    http_client,
+    jwt_token,
+    mock_loader_url,
+    mock_annotations_url,
+    sse_client_factory,
+):
+    requests.post(
+        f"{mock_annotations_url}/mock/config", json={"mode": "error"}, timeout=10
+    ).raise_for_status()
+    media_id = f"res-n07-{uuid.uuid4().hex}"
+    body = _ai_config_video(mock_loader_url)
+    headers = {"Authorization": f"Bearer {jwt_token}"}
+    collected = []
+    thread_exc = []
+    done = threading.Event()
+
+    def _listen():
+        try:
+            with sse_client_factory() as sse:
+                time.sleep(0.3)
+                for event in sse.events():
+                    if not event.data or not str(event.data).strip():
+                        continue
+                    data = json.loads(event.data)
+                    if data.get("mediaId") != media_id:
+                        continue
+                    collected.append(data)
+                    if (
+                        data.get("mediaStatus") == "AIProcessed"
+                        and data.get("mediaPercent") == 100
+                    ):
+                        break
+        except BaseException as e:
+            thread_exc.append(e)
+        finally:
+            done.set()
+
+    th = threading.Thread(target=_listen, daemon=True)
+    th.start()
+    time.sleep(0.5)
+    pr = http_client.post(f"/detect/{media_id}", json=body, headers=headers)
+    assert pr.status_code == 200
+    ok = done.wait(timeout=120)
+    assert ok
+    th.join(timeout=5)
+    assert not thread_exc
+    assert any(
+        e.get("mediaStatus") == "AIProcessed" and e.get("mediaPercent") == 100
+        for e in collected
+    )
+
+
+def test_nft_res_01_loader_outage_after_init(
+    warm_engine, http_client, mock_loader_url, image_small
+):
+    requests.post(
+        f"{mock_loader_url}/mock/config", json={"mode": "error"}, timeout=10
+    ).raise_for_status()
+    files = {"file": ("r1.jpg", image_small, "image/jpeg")}
+    r = http_client.post("/detect", files=files, timeout=_DETECT_TIMEOUT)
+    assert r.status_code == 200
+    assert isinstance(r.json(), list)
+    h = http_client.get("/health")
+    assert h.status_code == 200
+    hd = h.json()
+    assert hd["status"] == "healthy"
+    assert hd.get("errorMessage") is None
+
+
+@pytest.mark.slow
+@pytest.mark.timeout(120)
+def test_nft_res_02_annotations_outage_during_async_detection(
+    warm_engine,
+    http_client,
+    jwt_token,
+    mock_loader_url,
+    mock_annotations_url,
+    sse_client_factory,
+):
+    media_id = f"res-n02-{uuid.uuid4().hex}"
+    body = _ai_config_video(mock_loader_url)
+    headers = {"Authorization": f"Bearer {jwt_token}"}
+    collected = []
+    thread_exc = []
+    done = threading.Event()
+
+    def _listen():
+        try:
+            with sse_client_factory() as sse:
+                time.sleep(0.3)
+                for event in sse.events():
+                    if not event.data or not str(event.data).strip():
+                        continue
+                    data = json.loads(event.data)
+                    if data.get("mediaId") != media_id:
+                        continue
+                    collected.append(data)
+                    if (
+                        data.get("mediaStatus") == "AIProcessed"
+                        and data.get("mediaPercent") == 100
+                    ):
+                        break
+        except BaseException as e:
+            thread_exc.append(e)
+        finally:
+            done.set()
+
+    th = threading.Thread(target=_listen, daemon=True)
+    th.start()
+    time.sleep(0.5)
+    pr = http_client.post(f"/detect/{media_id}", json=body, headers=headers)
+    assert pr.status_code == 200
+    requests.post(
+        f"{mock_annotations_url}/mock/config", json={"mode": "error"}, timeout=10
+    ).raise_for_status()
+    ok = done.wait(timeout=120)
+    assert ok
+    th.join(timeout=5)
+    assert not thread_exc
+    assert any(
+        e.get("mediaStatus") == "AIProcessed" and e.get("mediaPercent") == 100
+        for e in collected
+    )
+
+
+def test_nft_res_03_transient_loader_first_fail(
+    mock_loader_url, http_client, image_small
+):
+    requests.post(
+        f"{mock_loader_url}/mock/config", json={"mode": "first_fail"}, timeout=10
+    ).raise_for_status()
+    files = {"file": ("r3a.jpg", image_small, "image/jpeg")}
+    r1 = http_client.post("/detect", files=files, timeout=_DETECT_TIMEOUT)
+    files2 = {"file": ("r3b.jpg", image_small, "image/jpeg")}
+    r2 = http_client.post("/detect", files=files2, timeout=_DETECT_TIMEOUT)
+    assert r2.status_code == 200
+    if r1.status_code != 200:
+        assert r1.status_code != 500
+
+
+@pytest.mark.skip(
+    reason="Requires docker compose restart capability not available in e2e-runner"
+)
+def test_nft_res_04_service_restart():
+    pass