mirror of
https://github.com/azaion/gps-denied-onboard.git
synced 2026-06-21 17:01:13 +00:00
Oleksandr Bezdieniezhnykh
eb6dc17880
Subprocess-spawned tests in e2e/_unit_tests/reporting/ crashed with "argparse.ArgumentError: argument --csv: conflicting option string: --csv" because pytest-csv (autoloaded via entry-point) and our custom plugin both register --csv. pytest's option registry does not allow overrides. Fix: drop pytest-csv from e2e/runner/requirements.txt. It was unused, dead weight, and incompatible with pytest 9.x (uses removed hookwrapper marker). Update conftest + csv_reporter comments to match. After fix: 1229/1229 in e2e/_unit_tests pass. Bug ticket creation deferred (user skipped interactive Q this session) — payload recorded in _docs/_process_leftovers/2026-05-17_csv_reporter_*.md for replay on next /autodev. Co-authored-by: Cursor <cursoragent@cursor.com>
219 lines
8.6 KiB
Python
219 lines
8.6 KiB
Python
"""Top-level pytest conftest for the blackbox e2e harness.
|
|
|
|
Responsibilities:
|
|
1. Session-level parameterization over ``(fc_adapter, vio_strategy)``.
|
|
2. Skip-rule enforcement per the traceability matrix
|
|
(`_docs/02_document/tests/traceability-matrix.md`):
|
|
- AC-7.1, AC-7.2 → SKIP (deferred — no AI-camera fixture)
|
|
- RESTRICT-CAM-2 → SKIP (paired with AC-7.x)
|
|
- AC-NEW-5 chamber portion → SKIP unless --enable-chamber
|
|
- RESTRICT-HW-2 chamber portion → SKIP unless --enable-chamber
|
|
- Tier-2-only tests → SKIP on tier1-docker
|
|
- `vins_mono` parametrization → SKIP on production-build sessions
|
|
3. Wiring of the boundary-driving fixtures (`sitl_observer`,
|
|
`mavproxy_tlog`, `fdr_reader`, `mock_suite_sat_client`) consumed by
|
|
per-scenario tests.
|
|
|
|
The actual boundary-driving fixtures import helper modules from
|
|
``runner.helpers.*``. They are registered here but their implementations
|
|
live in the helpers package.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from collections.abc import Iterator
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Command-line options
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def pytest_addoption(parser: pytest.Parser) -> None:
|
|
"""Harness-level options (not exposed to individual tests)."""
|
|
group = parser.getgroup("e2e-runner", "Blackbox e2e harness options")
|
|
group.addoption(
|
|
"--enable-chamber",
|
|
action="store_true",
|
|
default=False,
|
|
help="Enable thermal-chamber-gated tests (AC-NEW-5 hot-soak, RESTRICT-HW-2). "
|
|
"Requires the chamber-attached Jetson runner; default off.",
|
|
)
|
|
group.addoption(
|
|
"--build-kind",
|
|
action="store",
|
|
default=os.environ.get("BUILD_KIND", "production"),
|
|
choices=("production", "research"),
|
|
help="Selects which VIO strategies are valid: production excludes vins_mono.",
|
|
)
|
|
group.addoption(
|
|
"--evidence-out",
|
|
action="store",
|
|
default=os.environ.get("EVIDENCE_OUT", "/e2e-results/evidence"),
|
|
help="Directory the evidence bundler writes per-run artifacts to.",
|
|
)
|
|
group.addoption(
|
|
"--allow-no-skip-reason",
|
|
action="store_true",
|
|
default=False,
|
|
help="Allow @pytest.mark.deferred_ac without an explicit reason= kwarg. "
|
|
"Default off — every deferred AC must cite its traceability-matrix row.",
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Parameterization matrix
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_FC_ADAPTERS = ("ardupilot", "inav")
|
|
_VIO_STRATEGIES = ("okvis2", "klt_ransac", "vins_mono")
|
|
|
|
|
|
def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
|
|
"""Parametrize tests that request the ``fc_adapter`` / ``vio_strategy`` fixtures.
|
|
|
|
Tests opt in by listing the fixture name in their signature. Tests that
|
|
explicitly do not depend on the matrix simply do not request the fixture.
|
|
"""
|
|
if "fc_adapter" in metafunc.fixturenames:
|
|
env_default = os.environ.get("FC_ADAPTER")
|
|
if env_default:
|
|
metafunc.parametrize("fc_adapter", [env_default], ids=[env_default])
|
|
else:
|
|
metafunc.parametrize("fc_adapter", _FC_ADAPTERS, ids=_FC_ADAPTERS)
|
|
if "vio_strategy" in metafunc.fixturenames:
|
|
env_default = os.environ.get("VIO_STRATEGY")
|
|
if env_default:
|
|
metafunc.parametrize("vio_strategy", [env_default], ids=[env_default])
|
|
else:
|
|
metafunc.parametrize("vio_strategy", _VIO_STRATEGIES, ids=_VIO_STRATEGIES)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Skip-rule enforcement (deterministic; runs at collection time)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def pytest_collection_modifyitems(
|
|
config: pytest.Config, items: list[pytest.Item]
|
|
) -> None:
|
|
"""Apply traceability-matrix-driven skips before any test executes.
|
|
|
|
The mapping between AC / RESTRICT IDs and the SKIP reason strings is the
|
|
one declared in `_docs/02_document/tests/traceability-matrix.md` §
|
|
Uncovered Items Analysis. Any change to that matrix MUST be mirrored
|
|
here (and vice-versa) — the unit tests in
|
|
`e2e/_unit_tests/test_traceability_skip_rules.py` catch drift.
|
|
"""
|
|
tier = os.environ.get("TIER", "tier1-docker")
|
|
chamber_enabled = config.getoption("--enable-chamber")
|
|
build_kind = config.getoption("--build-kind")
|
|
|
|
skip_tier2 = pytest.mark.skip(reason="Tier-2 only — Jetson hardware required")
|
|
skip_chamber = pytest.mark.skip(
|
|
reason="Chamber-gated — run with --enable-chamber on the chamber-attached Jetson runner"
|
|
)
|
|
skip_research = pytest.mark.skip(
|
|
reason="vins_mono is research-build-only per D-C1-1-SUB-A"
|
|
)
|
|
|
|
for item in items:
|
|
# ----- Tier-2 only -----
|
|
if "tier2_only" in item.keywords and tier != "tier2-jetson":
|
|
item.add_marker(skip_tier2)
|
|
continue
|
|
|
|
# ----- Chamber only -----
|
|
if "chamber_only" in item.keywords and not chamber_enabled:
|
|
item.add_marker(skip_chamber)
|
|
continue
|
|
|
|
# ----- Research-build vs production matrix -----
|
|
# Skip vins_mono on production-build runs (the marker is set on the
|
|
# parametrize id, not the test fn — we check the param id).
|
|
if build_kind == "production":
|
|
call_params = getattr(item, "callspec", None)
|
|
if call_params is not None and call_params.params.get("vio_strategy") == "vins_mono":
|
|
item.add_marker(skip_research)
|
|
continue
|
|
|
|
# ----- Deferred-AC traceability-matrix skips -----
|
|
deferred = item.get_closest_marker("deferred_ac")
|
|
if deferred is not None:
|
|
reason = deferred.kwargs.get("reason")
|
|
if reason is None and not config.getoption("--allow-no-skip-reason"):
|
|
# Hard failure at collection — every deferred_ac MUST cite its
|
|
# matrix row to prevent silent coverage erosion.
|
|
item.add_marker(
|
|
pytest.mark.skip(
|
|
reason=(
|
|
"deferred_ac marker without reason= kwarg; cite the "
|
|
"traceability-matrix row that justifies the deferral, "
|
|
"or run with --allow-no-skip-reason for local debugging."
|
|
)
|
|
)
|
|
)
|
|
continue
|
|
verdict = deferred.kwargs.get("verdict", "skip").lower()
|
|
if verdict == "xfail":
|
|
item.add_marker(pytest.mark.xfail(reason=reason or "deferred AC (xfail)", strict=False))
|
|
else:
|
|
item.add_marker(
|
|
pytest.mark.skip(
|
|
reason=(
|
|
reason
|
|
or "deferred AC — see _docs/02_document/tests/traceability-matrix.md"
|
|
)
|
|
)
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fixtures
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def run_id() -> str:
|
|
return os.environ.get("RUN_ID", "local")
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def tier() -> str:
|
|
return os.environ.get("TIER", "tier1-docker")
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def evidence_dir(pytestconfig: pytest.Config, run_id: str) -> Path:
|
|
base = Path(pytestconfig.getoption("--evidence-out"))
|
|
target = base if base.name == "evidence" else base / "evidence"
|
|
target.mkdir(parents=True, exist_ok=True)
|
|
return target
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def mock_suite_sat_url() -> str:
|
|
return os.environ.get("MOCK_SUITE_SAT_URL", "http://mock-suite-sat-service:8080")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Plugin registration
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# The CSV reporter plugin is a separate module so the unit tests can exercise
|
|
# it directly without going through a real pytest run. It is registered via
|
|
# `pytest_plugins` to provide the `--csv` / `--csv-columns` CLI surface used
|
|
# by docker-compose. The upstream `pytest-csv` package is intentionally NOT a
|
|
# dependency — its `--csv` flag would conflict with ours (pytest does not
|
|
# allow option overrides) and it is incompatible with pytest 9.x.
|
|
pytest_plugins = [
|
|
"runner.reporting.csv_reporter",
|
|
"runner.reporting.evidence_bundler",
|
|
"runner.reporting.nfr_recorder",
|
|
"runner.helpers.injector_fixtures",
|
|
]
|