[AZ-421] Batch 82: FT-P-15 + FT-P-16 + FT-P-18 cache / offline / no-raw-retention

FT-P-15: parse FDR `cache-self-check` records; assert every tile-manifest entry has CRS, tile_matrix, dimension, m_per_px, capture_date, source, compression; m_per_px >= 0.5 (or rejected by FDR `tile-load-rejected`). FT-P-16: read `docker network inspect e2e-net` + `docker inspect <sut>` snapshots; assert `Internal == true` AND SUT attached only to e2e-net. The 0-egress semantic of AC-8.3 is enforced structurally. FT-P-18: walk FDR + tile-cache, probe JPEG dimensions via stdlib SOF parser, reject any file matching nav-camera raw pattern (5472x3648 or 880x720). Extrapolate thumbnail-log size to 8h; assert < 1 GB. Adds runner.helpers.tile_cache_inspector with five evaluators (manifest schema, offline mode, raw-frame detection, thumbnail budget, JPEG dimension probe) + walk_files helper. Pure-logic coverage: 43 new unit tests; full e2e/_unit_tests/ suite 793 passing (was 746). Scenarios skip locally when SITL replay fixture or docker-inspect env vars are missing; production hooks (cache-self-check FDR record, tile-load-rejected events, docker-inspect snapshots) are tracked outside this task. See _docs/03_implementation/batch_82_report.md + reviews/batch_82_review.md. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-06-22 14:01:12 +00:00 · 2026-05-17 15:09:58 +03:00
parent b0296da911
commit 7d1288e4ba
9 changed files with 1693 additions and 3 deletions
@@ -0,0 +1,491 @@
+"""Unit tests for ``runner.helpers.tile_cache_inspector`` (AZ-421).
+
+Pure-logic AC-8.1 / AC-8.3 / AC-8.5 coverage for FT-P-15 / FT-P-16 /
+FT-P-18. The full e2e scenarios in ``e2e/tests/positive/test_ft_p_1[568]_*.py``
+exercise the same helpers end-to-end when ``E2E_SITL_REPLAY_DIR`` is
+prepared; this file covers the helpers in isolation so AC verification
+does not depend on the SITL fixture or a live docker daemon.
+"""
+
+from __future__ import annotations
+
+import struct
+from pathlib import Path
+
+import pytest
+
+from runner.helpers import tile_cache_inspector as tci
+
+
+# ─────────────────────── evaluate_manifest_schema ───────────────────────
+
+
+def _full_entry(**overrides: object) -> dict:
+    """Construct a manifest entry that has every required field by default."""
+    # Arrange — return a complete dict the caller can selectively break
+    base: dict[str, object] = {
+        "id": "tile_001",
+        "crs": "EPSG:3857",
+        "tile_matrix": "WGS84_Quad/16",
+        "dimension": 256,
+        "m_per_px": 0.5,
+        "capture_date": "2025-04-12",
+        "source": "internal_drone_2024_capture",
+        "compression": "JPEG-Q85",
+    }
+    base.update(overrides)
+    return base
+
+
+def test_evaluate_manifest_schema_all_fields_present_floor_met_passes() -> None:
+    # Arrange
+    entries = [_full_entry(id=f"t_{i}", m_per_px=0.5 + i * 0.1) for i in range(3)]
+    # Act
+    report = tci.evaluate_manifest_schema(entries)
+    # Assert
+    assert report.passes
+    assert report.total_entries == 3
+    assert report.entries_with_missing_fields == ()
+    assert report.entries_below_floor == ()
+
+
+def test_evaluate_manifest_schema_missing_field_fails() -> None:
+    # Arrange
+    entries = [_full_entry()]
+    del entries[0]["compression"]
+    # Act
+    report = tci.evaluate_manifest_schema(entries)
+    # Assert
+    assert not report.passes
+    assert report.entries_with_missing_fields[0].missing_fields == ("compression",)
+
+
+def test_evaluate_manifest_schema_multiple_missing_fields_listed_in_order() -> None:
+    # Arrange
+    entries = [_full_entry()]
+    del entries[0]["crs"]
+    del entries[0]["compression"]
+    # Act
+    report = tci.evaluate_manifest_schema(entries)
+    # Assert
+    assert report.entries_with_missing_fields[0].missing_fields == ("crs", "compression")
+
+
+def test_evaluate_manifest_schema_below_floor_without_rejection_fails() -> None:
+    # Arrange
+    entries = [_full_entry(id="lowres", m_per_px=0.4)]
+    # Act
+    report = tci.evaluate_manifest_schema(entries)
+    # Assert
+    assert not report.passes
+    assert report.entries_below_floor[0].entry_id == "lowres"
+
+
+def test_evaluate_manifest_schema_below_floor_with_rejection_passes() -> None:
+    # Arrange
+    entries = [
+        _full_entry(id="good", m_per_px=0.5),
+        _full_entry(id="lowres", m_per_px=0.4),
+    ]
+    # Act
+    report = tci.evaluate_manifest_schema(entries, tile_load_rejected_ids=("lowres",))
+    # Assert
+    assert report.passes
+
+
+def test_evaluate_manifest_schema_at_floor_exactly_passes() -> None:
+    # Arrange
+    entries = [_full_entry(m_per_px=0.5)]
+    # Act
+    report = tci.evaluate_manifest_schema(entries)
+    # Assert
+    assert report.passes
+
+
+def test_evaluate_manifest_schema_empty_list_fails() -> None:
+    # Act
+    report = tci.evaluate_manifest_schema([])
+    # Assert
+    assert not report.passes
+    assert report.total_entries == 0
+
+
+def test_evaluate_manifest_schema_non_numeric_m_per_px_fails() -> None:
+    # Arrange
+    entries = [_full_entry(m_per_px="0.5")]
+    # Act
+    report = tci.evaluate_manifest_schema(entries)
+    # Assert
+    assert not report.passes
+    assert report.entries[0].m_per_px is None
+
+
+def test_evaluate_manifest_schema_entry_id_falls_back_to_synthesised() -> None:
+    # Arrange
+    entry = _full_entry()
+    del entry["id"]
+    # Act
+    report = tci.evaluate_manifest_schema([entry])
+    # Assert
+    assert report.entries[0].entry_id == "tile_matrix" or report.entries[0].entry_id.startswith("entry_") or report.entries[0].entry_id == "WGS84_Quad/16"
+
+
+def test_evaluate_manifest_schema_invalid_floor_raises() -> None:
+    with pytest.raises(ValueError, match="m_per_px_floor"):
+        tci.evaluate_manifest_schema([_full_entry()], m_per_px_floor=0)
+
+
+def test_evaluate_manifest_schema_custom_required_fields() -> None:
+    # Arrange — using a minimal field set the test owns
+    entries = [{"id": "t1", "m_per_px": 1.0, "crs": "EPSG:3857"}]
+    # Act
+    report = tci.evaluate_manifest_schema(
+        entries, required_fields=("id", "crs", "m_per_px")
+    )
+    # Assert
+    assert report.passes
+
+
+def test_evaluate_manifest_schema_one_good_one_bad_fails() -> None:
+    # Arrange
+    entries = [_full_entry(id="ok"), _full_entry(id="bad", m_per_px=0.3)]
+    # Act
+    report = tci.evaluate_manifest_schema(entries)
+    # Assert
+    assert not report.passes
+    assert len(report.entries_below_floor) == 1
+    assert report.entries_below_floor[0].entry_id == "bad"
+
+
+# ─────────────────────── evaluate_offline_mode ───────────────────────
+
+
+def _network_inspect(*, name: str = "e2e-net", internal: bool = True) -> dict:
+    return {"Name": name, "Internal": internal, "Driver": "bridge"}
+
+
+def _container_inspect(*networks: str) -> dict:
+    return {
+        "Id": "deadbeef",
+        "NetworkSettings": {"Networks": {n: {"IPAddress": "172.20.0.2"} for n in networks}},
+    }
+
+
+def test_evaluate_offline_mode_internal_and_only_e2e_net_passes() -> None:
+    # Act
+    report = tci.evaluate_offline_mode(_network_inspect(), _container_inspect("e2e-net"))
+    # Assert
+    assert report.passes
+    assert report.network_internal is True
+    assert report.container_networks == ("e2e-net",)
+
+
+def test_evaluate_offline_mode_non_internal_fails() -> None:
+    # Act
+    report = tci.evaluate_offline_mode(
+        _network_inspect(internal=False), _container_inspect("e2e-net")
+    )
+    # Assert
+    assert not report.passes
+
+
+def test_evaluate_offline_mode_extra_network_fails() -> None:
+    # Act
+    report = tci.evaluate_offline_mode(
+        _network_inspect(), _container_inspect("e2e-net", "bridge")
+    )
+    # Assert
+    assert not report.passes
+
+
+def test_evaluate_offline_mode_no_networks_fails() -> None:
+    # Act
+    report = tci.evaluate_offline_mode(_network_inspect(), _container_inspect())
+    # Assert
+    assert not report.passes
+
+
+def test_evaluate_offline_mode_missing_internal_key_fails() -> None:
+    # Arrange
+    net = {"Name": "e2e-net", "Driver": "bridge"}
+    # Act
+    report = tci.evaluate_offline_mode(net, _container_inspect("e2e-net"))
+    # Assert
+    assert not report.passes
+    assert report.network_internal is None
+
+
+def test_evaluate_offline_mode_non_bool_internal_fails() -> None:
+    # Arrange
+    net = {"Name": "e2e-net", "Internal": "true"}  # string, not bool
+    # Act
+    report = tci.evaluate_offline_mode(net, _container_inspect("e2e-net"))
+    # Assert
+    assert not report.passes
+
+
+def test_evaluate_offline_mode_custom_expected_network() -> None:
+    # Act
+    report = tci.evaluate_offline_mode(
+        _network_inspect(name="custom-net"),
+        _container_inspect("custom-net"),
+        expected_network="custom-net",
+    )
+    # Assert
+    assert report.passes
+    assert report.expected_network == "custom-net"
+
+
+# ─────────────────────── detect_raw_frames ───────────────────────
+
+
+def test_detect_raw_frames_nav_camera_raw_dimension_match() -> None:
+    # Arrange
+    specs = [(Path("/data/frame.jpg"), 12345, (5472, 3648))]
+    # Act
+    report = tci.detect_raw_frames(specs)
+    # Assert
+    assert not report.passes
+    assert report.candidate_count == 1
+    assert report.candidates[0].dimensions == (5472, 3648)
+
+
+def test_detect_raw_frames_h264_decoded_dimension_match() -> None:
+    # Arrange
+    specs = [(Path("/cache/buf.jpg"), 500, (880, 720))]
+    # Act
+    report = tci.detect_raw_frames(specs)
+    # Assert
+    assert not report.passes
+
+
+def test_detect_raw_frames_dimension_order_insensitive() -> None:
+    # Arrange — (3648, 5472) is a sideways encoding of the raw nav-cam shape
+    specs = [(Path("/data/frame.jpg"), 12345, (3648, 5472))]
+    # Act
+    report = tci.detect_raw_frames(specs)
+    # Assert
+    assert report.candidate_count == 1
+
+
+def test_detect_raw_frames_thumbnail_dimensions_pass() -> None:
+    # Arrange — small thumbnail
+    specs = [(Path("/cache/thumb.jpg"), 4096, (128, 96))]
+    # Act
+    report = tci.detect_raw_frames(specs)
+    # Assert
+    assert report.passes
+
+
+def test_detect_raw_frames_no_raw_extension_pass() -> None:
+    # Arrange — .png is not in the raw-extension list
+    specs = [(Path("/cache/snap.png"), 1024, (5472, 3648))]
+    # Act
+    report = tci.detect_raw_frames(specs)
+    # Assert
+    assert report.passes
+
+
+def test_detect_raw_frames_unknown_dimensions_pass() -> None:
+    # Arrange — dimension probe failed; per docstring this is NOT a match
+    specs = [(Path("/cache/frame.jpg"), 1024, None)]
+    # Act
+    report = tci.detect_raw_frames(specs)
+    # Assert
+    assert report.passes
+
+
+def test_detect_raw_frames_empty_list_passes() -> None:
+    # Act
+    report = tci.detect_raw_frames([])
+    # Assert
+    assert report.passes
+    assert report.candidate_count == 0
+
+
+def test_detect_raw_frames_dng_extension_matches() -> None:
+    # Arrange
+    specs = [(Path("/data/img.dng"), 1024, (5472, 3648))]
+    # Act
+    report = tci.detect_raw_frames(specs)
+    # Assert
+    assert not report.passes
+
+
+def test_detect_raw_frames_custom_dimensions() -> None:
+    # Arrange
+    specs = [(Path("/data/img.jpg"), 1024, (100, 100))]
+    # Act
+    report = tci.detect_raw_frames(
+        specs, raw_dimensions=(100, 100), decoded_dimensions=(50, 50)
+    )
+    # Assert
+    assert not report.passes
+
+
+# ─────────────────────── evaluate_thumbnail_budget ───────────────────────
+
+
+def test_evaluate_thumbnail_budget_under_budget_passes() -> None:
+    # Arrange — 100 MB over 1 h extrapolates to 800 MB / 8 h (< 1 GB)
+    size = 100 * 1024**2
+    # Act
+    report = tci.evaluate_thumbnail_budget(size, observed_duration_h=1.0)
+    # Assert
+    assert report.passes
+
+
+def test_evaluate_thumbnail_budget_over_budget_fails() -> None:
+    # Arrange — 200 MB over 1 h extrapolates to 1.6 GB / 8 h (> 1 GB)
+    size = 200 * 1024**2
+    # Act
+    report = tci.evaluate_thumbnail_budget(size, observed_duration_h=1.0)
+    # Assert
+    assert not report.passes
+
+
+def test_evaluate_thumbnail_budget_extrapolation_math() -> None:
+    # Arrange — 1 MB over 2 h extrapolates to 4 MB / 8 h
+    one_mb = 1024**2
+    # Act
+    report = tci.evaluate_thumbnail_budget(one_mb, observed_duration_h=2.0)
+    # Assert
+    assert report.extrapolated_8h_size_bytes == 4 * one_mb
+
+
+def test_evaluate_thumbnail_budget_zero_duration_fails() -> None:
+    # Act
+    report = tci.evaluate_thumbnail_budget(1024, observed_duration_h=0.0)
+    # Assert
+    assert not report.passes
+
+
+def test_evaluate_thumbnail_budget_negative_size_raises() -> None:
+    with pytest.raises(ValueError, match="observed_size_bytes"):
+        tci.evaluate_thumbnail_budget(-1, observed_duration_h=1.0)
+
+
+def test_evaluate_thumbnail_budget_invalid_budget_raises() -> None:
+    with pytest.raises(ValueError, match="max_size_bytes_per_8h"):
+        tci.evaluate_thumbnail_budget(1024, observed_duration_h=1.0, max_size_bytes_per_8h=0)
+
+
+def test_evaluate_thumbnail_budget_custom_budget() -> None:
+    # Arrange — 500 MB over 1 h ≈ 4 GB / 8 h; budget = 10 GB → passes
+    size = 500 * 1024**2
+    budget = 10 * 1024**3
+    # Act
+    report = tci.evaluate_thumbnail_budget(
+        size, observed_duration_h=1.0, max_size_bytes_per_8h=budget
+    )
+    # Assert
+    assert report.passes
+
+
+# ─────────────────────── walk_files ───────────────────────
+
+
+def test_walk_files_skips_missing_roots(tmp_path: Path) -> None:
+    # Arrange
+    (tmp_path / "real").mkdir()
+    (tmp_path / "real" / "f.txt").write_text("x")
+    missing = tmp_path / "missing"
+    # Act
+    files = list(tci.walk_files(missing, tmp_path / "real"))
+    # Assert
+    assert len(files) == 1
+    assert files[0].name == "f.txt"
+
+
+def test_walk_files_recursive(tmp_path: Path) -> None:
+    # Arrange
+    (tmp_path / "a" / "b").mkdir(parents=True)
+    (tmp_path / "a" / "top.txt").write_text("x")
+    (tmp_path / "a" / "b" / "nested.txt").write_text("x")
+    # Act
+    files = sorted(tci.walk_files(tmp_path), key=lambda p: p.name)
+    # Assert
+    assert [f.name for f in files] == ["nested.txt", "top.txt"]
+
+
+def test_walk_files_no_directories_yielded(tmp_path: Path) -> None:
+    # Arrange
+    (tmp_path / "subdir").mkdir()
+    (tmp_path / "subdir" / "f.txt").write_text("x")
+    # Act
+    files = list(tci.walk_files(tmp_path))
+    # Assert — only the file, not the directory itself
+    assert all(p.is_file() for p in files)
+
+
+# ─────────────────────── probe_jpeg_dimensions ───────────────────────
+
+
+def _make_minimal_jpeg(width: int, height: int) -> bytes:
+    """Construct a minimal-but-valid JPEG with the given SOF0 dimensions.
+
+    The result starts with SOI then jumps straight to an SOF0 segment
+    that encodes the requested w/h. Nothing past the SOF needs to be
+    valid for the dimension probe to succeed.
+    """
+    # SOI marker
+    soi = b"\xff\xd8"
+    # SOF0 segment: marker (FFC0) + length (2) + precision (1) + h (2) + w (2) + nf (1) + components (3*nf)
+    # length = 8 + 3 (1 component)
+    sof0 = (
+        b"\xff\xc0"
+        + struct.pack(">H", 11)
+        + b"\x08"  # precision
+        + struct.pack(">H", height)
+        + struct.pack(">H", width)
+        + b"\x01"  # n components
+        + b"\x01\x22\x00"  # component spec
+    )
+    return soi + sof0
+
+
+def test_probe_jpeg_dimensions_returns_width_height(tmp_path: Path) -> None:
+    # Arrange
+    f = tmp_path / "img.jpg"
+    f.write_bytes(_make_minimal_jpeg(640, 480))
+    # Act
+    dims = tci.probe_jpeg_dimensions(f)
+    # Assert
+    assert dims == (640, 480)
+
+
+def test_probe_jpeg_dimensions_handles_raw_nav_camera_dims(tmp_path: Path) -> None:
+    # Arrange
+    f = tmp_path / "raw.jpg"
+    f.write_bytes(_make_minimal_jpeg(5472, 3648))
+    # Act
+    dims = tci.probe_jpeg_dimensions(f)
+    # Assert
+    assert dims == (5472, 3648)
+
+
+def test_probe_jpeg_dimensions_not_a_jpeg(tmp_path: Path) -> None:
+    # Arrange
+    f = tmp_path / "not.jpg"
+    f.write_bytes(b"PNG\x00not a jpeg")
+    # Act
+    dims = tci.probe_jpeg_dimensions(f)
+    # Assert
+    assert dims is None
+
+
+def test_probe_jpeg_dimensions_truncated(tmp_path: Path) -> None:
+    # Arrange — SOI marker only, no SOF segment
+    f = tmp_path / "trunc.jpg"
+    f.write_bytes(b"\xff\xd8")
+    # Act
+    dims = tci.probe_jpeg_dimensions(f)
+    # Assert
+    assert dims is None
+
+
+def test_probe_jpeg_dimensions_nonexistent(tmp_path: Path) -> None:
+    # Act
+    dims = tci.probe_jpeg_dimensions(tmp_path / "missing.jpg")
+    # Assert
+    assert dims is None
@@ -52,6 +52,7 @@ E2E_ROOT = Path(__file__).resolve().parents[1]
        "runner/helpers/msp_frame_observer.py",
        "runner/helpers/ap_contract_evaluator.py",
        "runner/helpers/gcs_telemetry_evaluator.py",
+        "runner/helpers/tile_cache_inspector.py",
        "runner/helpers/cold_start_evaluator.py",
        "runner/helpers/outlier_tolerance_evaluator.py",
        "runner/helpers/outage_request_evaluator.py",
@@ -109,6 +110,9 @@ E2E_ROOT = Path(__file__).resolve().parents[1]
        "tests/positive/test_ft_p_11_cold_start_init.py",
        "tests/positive/test_ft_p_12_gcs_downsample.py",
        "tests/positive/test_ft_p_13_gcs_command.py",
+        "tests/positive/test_ft_p_15_cache_schema.py",
+        "tests/positive/test_ft_p_16_offline_only.py",
+        "tests/positive/test_ft_p_18_no_raw_retention.py",
        "tests/negative/test_ft_n_01_outlier_tolerance.py",
        "tests/negative/test_ft_n_02_sharp_turn_failure.py",
        "tests/negative/test_ft_n_03_outage_reloc.py",