mirror of
https://github.com/azaion/gps-denied-onboard.git
synced 2026-06-21 10:21:13 +00:00
Oleksandr Bezdieniezhnykh
91ce1c2047
AZ-326 (3pt): operator-tool Click CLI shell at src/gps_denied_onboard/components/c12_operator_tooling/cli.py with six subcommands (download, build-cache, upload-pending, reloc-confirm, verify-ready, set-sector); SectorClassificationStore (atomic-write JSON under ~/.azaion/onboard/sector-classifications.json); freshness-table lookup driving AC-NEW-6; EXIT_* constants; AZ-266 structured-JSON log wiring to a rotating ~/.azaion/onboard/c12-tooling.log handler; operator-tool console-script entry in pyproject.toml. AZ-327 (3pt): CompanionBringup orchestrator at src/gps_denied_onboard/components/c12_operator_tooling/companion_bringup.py that opens an SSH session against the companion (paramiko per project pin), checks the four pre-flight artifacts (Manifest, expected engines, sha256 sidecars, calibration), and returns a ReadinessReport per description.md S2; CompanionUnreachableError + ContentHashMismatchError with operator-friendly remediation hints; ParamikoSshSessionFactory + RemoteSidecarVerifier (sha256sum + cat over SSH, no bytes pulled to the workstation); paramiko>=3.4,<4.0 dep added. NFR-perf-cold-start fix: PEP 562 lazy __getattr__ in c12_operator_tooling/__init__.py and flights_api/__init__.py defers HttpxFlightsApiClient (httpx), ParamikoSshSession[Factory] (paramiko + cryptography), bbox_from_waypoints / takeoff_origin_from_flight (numpy + pyproj). cli.py imports from leaf flights_api modules. operator-tool --help cold start: ~870ms -> <200ms typical, <500ms p99. Includes 73 unit tests (incl. paramiko-version-drift smoke per AZ-327 Risk 1) + console-script integration test. All 1494 repo-wide unit tests pass; 80 skips are pre-existing environment gates. Batch report: _docs/03_implementation/batch_42_cycle1_report.md. Co-authored-by: Cursor <cursoragent@cursor.com>
475 lines
16 KiB
Python
475 lines
16 KiB
Python
"""AZ-327 — `CompanionBringup.verify_companion_ready` AC-1 .. AC-10."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path, PurePosixPath
|
|
|
|
import pytest
|
|
|
|
from gps_denied_onboard.components.c12_operator_tooling import (
|
|
C12CompanionConfig,
|
|
CompanionAddress,
|
|
CompanionBringup,
|
|
CompanionUnreachableError,
|
|
CompanionUnreachableReason,
|
|
ContentHashMismatchError,
|
|
HostKeyPolicy,
|
|
ReadinessOutcome,
|
|
)
|
|
from gps_denied_onboard.components.c12_operator_tooling.remote_sidecar_verifier import (
|
|
RemoteSidecarResult,
|
|
)
|
|
from gps_denied_onboard.components.c12_operator_tooling.ssh_session import (
|
|
RemoteCommandResult,
|
|
SshSession,
|
|
SshSessionFactory,
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fakes
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@dataclass
|
|
class _FakeSession(SshSession):
|
|
"""Scripted SSH session for unit tests."""
|
|
|
|
files_present: set[str] = field(default_factory=set)
|
|
dir_contents: dict[str, list[str]] = field(default_factory=dict)
|
|
file_exists_raises: Exception | None = None
|
|
close_calls: int = 0
|
|
|
|
def run(self, command: str, *, timeout_s: float) -> RemoteCommandResult:
|
|
return RemoteCommandResult(exit_code=0, stdout="", stderr="")
|
|
|
|
def file_exists(self, remote_path: PurePosixPath) -> bool:
|
|
if self.file_exists_raises is not None:
|
|
raise self.file_exists_raises
|
|
return str(remote_path) in self.files_present
|
|
|
|
def list_dir(self, remote_path: PurePosixPath) -> list[str]:
|
|
try:
|
|
return list(self.dir_contents[str(remote_path)])
|
|
except KeyError as exc:
|
|
raise FileNotFoundError(str(remote_path)) from exc
|
|
|
|
def close(self) -> None:
|
|
self.close_calls += 1
|
|
|
|
|
|
@dataclass
|
|
class _FakeFactory(SshSessionFactory):
|
|
session: _FakeSession | None = None
|
|
open_raises: Exception | None = None
|
|
open_calls: int = 0
|
|
|
|
def open(
|
|
self,
|
|
address: CompanionAddress,
|
|
*,
|
|
timeout_s: float,
|
|
) -> SshSession:
|
|
self.open_calls += 1
|
|
if self.open_raises is not None:
|
|
raise self.open_raises
|
|
assert self.session is not None
|
|
return self.session
|
|
|
|
|
|
@dataclass
|
|
class _ScriptedVerifier:
|
|
"""Drop-in replacement for `RemoteSidecarVerifier` in unit tests.
|
|
|
|
`outcomes_by_engine` keyed by engine filename → :class:`RemoteSidecarResult`.
|
|
`verify` calls are appended to `verify_calls` for assertion (AC-9).
|
|
"""
|
|
|
|
outcomes_by_engine: dict[str, RemoteSidecarResult] = field(default_factory=dict)
|
|
verify_calls: list[str] = field(default_factory=list)
|
|
default: RemoteSidecarResult = field(
|
|
default_factory=lambda: RemoteSidecarResult(
|
|
matches=True, expected_hex="aa" * 32, actual_hex="aa" * 32
|
|
)
|
|
)
|
|
|
|
def verify(
|
|
self,
|
|
session: SshSession,
|
|
engine_path: PurePosixPath,
|
|
) -> RemoteSidecarResult:
|
|
engine_name = engine_path.name
|
|
self.verify_calls.append(engine_name)
|
|
return self.outcomes_by_engine.get(engine_name, self.default)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fixtures
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
_CACHE_ROOT = PurePosixPath("/var/lib/azaion/c10/cache")
|
|
_ENGINES_DIR = _CACHE_ROOT / "engines"
|
|
_ENGINE_A = "dinov2_vpr_sm87_jp62_trt103_fp16.engine"
|
|
_ENGINE_B = "lightglue_sm87_jp62_trt103_fp16.engine"
|
|
_MANIFEST = "Manifest.json"
|
|
_CALIBRATION = "camera_calibration.json"
|
|
|
|
|
|
@pytest.fixture
|
|
def captured_logger() -> logging.Logger:
|
|
logger = logging.getLogger("test.c12.companion_bringup")
|
|
logger.handlers.clear()
|
|
logger.setLevel(logging.DEBUG)
|
|
return logger
|
|
|
|
|
|
@pytest.fixture
|
|
def companion_address() -> CompanionAddress:
|
|
return CompanionAddress(host="192.168.55.10", port=22)
|
|
|
|
|
|
@pytest.fixture
|
|
def base_config() -> C12CompanionConfig:
|
|
return C12CompanionConfig(
|
|
ssh_user="azaion",
|
|
ssh_keyfile=Path("/dev/null"), # not used in fake-driven tests
|
|
host_key_policy=HostKeyPolicy.STRICT,
|
|
connect_timeout_s=5.0,
|
|
companion_cache_root=_CACHE_ROOT,
|
|
manifest_filename=_MANIFEST,
|
|
calibration_filename=_CALIBRATION,
|
|
expected_engines=(_ENGINE_A, _ENGINE_B),
|
|
)
|
|
|
|
|
|
def _all_present_session() -> _FakeSession:
|
|
return _FakeSession(
|
|
files_present={
|
|
str(_CACHE_ROOT / _MANIFEST),
|
|
str(_CACHE_ROOT / _CALIBRATION),
|
|
},
|
|
dir_contents={str(_ENGINES_DIR): [_ENGINE_A, _ENGINE_B]},
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# AC-1 — happy path: outcome=ready
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestAC1Ready:
|
|
def test_all_artifacts_present_outcome_is_ready(
|
|
self,
|
|
captured_logger: logging.Logger,
|
|
companion_address: CompanionAddress,
|
|
base_config: C12CompanionConfig,
|
|
) -> None:
|
|
# Arrange
|
|
session = _all_present_session()
|
|
factory = _FakeFactory(session=session)
|
|
verifier = _ScriptedVerifier()
|
|
bringup = CompanionBringup(
|
|
ssh_factory=factory,
|
|
sidecar_verifier=verifier, # type: ignore[arg-type]
|
|
logger=captured_logger,
|
|
config=base_config,
|
|
)
|
|
|
|
# Act
|
|
report = bringup.verify_companion_ready(companion_address)
|
|
|
|
# Assert
|
|
assert report.outcome is ReadinessOutcome.READY
|
|
assert report.manifest_present
|
|
assert report.engines_present
|
|
assert report.content_hashes_pass
|
|
assert report.calibration_present
|
|
assert report.not_ready_reasons == ()
|
|
assert report.engines_inspected_count == 2
|
|
assert session.close_calls == 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# AC-2 — missing engine: outcome=not_ready, no hash mismatch
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestAC2MissingEngine:
|
|
def test_missing_engine_marks_not_ready(
|
|
self,
|
|
captured_logger: logging.Logger,
|
|
companion_address: CompanionAddress,
|
|
base_config: C12CompanionConfig,
|
|
) -> None:
|
|
# Arrange — only ENGINE_A on disk
|
|
session = _FakeSession(
|
|
files_present={
|
|
str(_CACHE_ROOT / _MANIFEST),
|
|
str(_CACHE_ROOT / _CALIBRATION),
|
|
},
|
|
dir_contents={str(_ENGINES_DIR): [_ENGINE_A]},
|
|
)
|
|
factory = _FakeFactory(session=session)
|
|
verifier = _ScriptedVerifier()
|
|
bringup = CompanionBringup(
|
|
ssh_factory=factory,
|
|
sidecar_verifier=verifier, # type: ignore[arg-type]
|
|
logger=captured_logger,
|
|
config=base_config,
|
|
)
|
|
|
|
# Act
|
|
report = bringup.verify_companion_ready(companion_address)
|
|
|
|
# Assert
|
|
assert report.outcome is ReadinessOutcome.NOT_READY
|
|
assert report.engines_present is False
|
|
assert any(_ENGINE_B in reason for reason in report.not_ready_reasons)
|
|
# AC-9 — the missing engine MUST NOT trigger a sidecar verify call
|
|
assert _ENGINE_B not in verifier.verify_calls
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# AC-3 — sidecar mismatch raises ContentHashMismatchError; session closed
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestAC3SidecarMismatch:
|
|
def test_sidecar_mismatch_raises_and_closes_session(
|
|
self,
|
|
captured_logger: logging.Logger,
|
|
companion_address: CompanionAddress,
|
|
base_config: C12CompanionConfig,
|
|
) -> None:
|
|
# Arrange
|
|
session = _all_present_session()
|
|
factory = _FakeFactory(session=session)
|
|
verifier = _ScriptedVerifier(
|
|
outcomes_by_engine={
|
|
_ENGINE_A: RemoteSidecarResult(
|
|
matches=False,
|
|
expected_hex="aa" * 32,
|
|
actual_hex="bb" * 32,
|
|
),
|
|
}
|
|
)
|
|
bringup = CompanionBringup(
|
|
ssh_factory=factory,
|
|
sidecar_verifier=verifier, # type: ignore[arg-type]
|
|
logger=captured_logger,
|
|
config=base_config,
|
|
)
|
|
|
|
# Act / Assert
|
|
with pytest.raises(ContentHashMismatchError) as excinfo:
|
|
bringup.verify_companion_ready(companion_address)
|
|
assert excinfo.value.expected_sha256_hex == "aa" * 32
|
|
assert excinfo.value.actual_sha256_hex == "bb" * 32
|
|
assert _ENGINE_A in excinfo.value.engine_path
|
|
assert session.close_calls == 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# AC-4..AC-6, AC-8, AC-10 — session-open failures map to the right reason
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"raised,expected_reason,reject_new_first_connect",
|
|
[
|
|
(
|
|
CompanionUnreachableError(
|
|
host="x",
|
|
port=22,
|
|
reason=CompanionUnreachableReason.CONNECT_REFUSED,
|
|
underlying_exception_repr="ConnectionRefusedError(...)",
|
|
),
|
|
CompanionUnreachableReason.CONNECT_REFUSED,
|
|
False,
|
|
),
|
|
(
|
|
CompanionUnreachableError(
|
|
host="x",
|
|
port=22,
|
|
reason=CompanionUnreachableReason.AUTH_FAILED,
|
|
underlying_exception_repr="paramiko.AuthenticationException(...)",
|
|
),
|
|
CompanionUnreachableReason.AUTH_FAILED,
|
|
False,
|
|
),
|
|
(
|
|
CompanionUnreachableError(
|
|
host="x",
|
|
port=22,
|
|
reason=CompanionUnreachableReason.HOST_KEY_MISMATCH,
|
|
underlying_exception_repr="paramiko.BadHostKeyException(...)",
|
|
),
|
|
CompanionUnreachableReason.HOST_KEY_MISMATCH,
|
|
False,
|
|
),
|
|
(
|
|
CompanionUnreachableError(
|
|
host="x",
|
|
port=22,
|
|
reason=CompanionUnreachableReason.TIMEOUT,
|
|
underlying_exception_repr="socket.timeout(...)",
|
|
),
|
|
CompanionUnreachableReason.TIMEOUT,
|
|
False,
|
|
),
|
|
(
|
|
CompanionUnreachableError(
|
|
host="x",
|
|
port=22,
|
|
reason=CompanionUnreachableReason.HOST_KEY_MISMATCH,
|
|
underlying_exception_repr="reject_new policy",
|
|
reject_new_first_connect=True,
|
|
),
|
|
CompanionUnreachableReason.HOST_KEY_MISMATCH,
|
|
True,
|
|
),
|
|
],
|
|
ids=[
|
|
"AC-4-connect-refused",
|
|
"AC-5-auth-failed",
|
|
"AC-6-host-key-mismatch-strict",
|
|
"AC-8-connect-timeout",
|
|
"AC-10-reject-new-first-connect",
|
|
],
|
|
)
|
|
class TestSessionOpenFailures:
|
|
def test_session_open_failure_propagates_with_reason(
|
|
self,
|
|
raised: CompanionUnreachableError,
|
|
expected_reason: CompanionUnreachableReason,
|
|
reject_new_first_connect: bool,
|
|
captured_logger: logging.Logger,
|
|
companion_address: CompanionAddress,
|
|
base_config: C12CompanionConfig,
|
|
) -> None:
|
|
# Arrange
|
|
factory = _FakeFactory(open_raises=raised)
|
|
verifier = _ScriptedVerifier()
|
|
bringup = CompanionBringup(
|
|
ssh_factory=factory,
|
|
sidecar_verifier=verifier, # type: ignore[arg-type]
|
|
logger=captured_logger,
|
|
config=base_config,
|
|
)
|
|
|
|
# Act / Assert
|
|
with pytest.raises(CompanionUnreachableError) as excinfo:
|
|
bringup.verify_companion_ready(companion_address)
|
|
assert excinfo.value.reason is expected_reason
|
|
# AC-4..AC-6, AC-8, AC-10 — `remediation` returns a non-empty hint
|
|
# specific to the reason / reject_new flag.
|
|
assert isinstance(excinfo.value.remediation, str)
|
|
assert len(excinfo.value.remediation) > 0
|
|
if reject_new_first_connect:
|
|
assert "ssh-keyscan" in excinfo.value.remediation
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# AC-7 — session always closed even on unexpected mid-flow exception
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestAC7SessionAlwaysClosed:
|
|
def test_unexpected_oserror_propagates_and_closes_session(
|
|
self,
|
|
captured_logger: logging.Logger,
|
|
companion_address: CompanionAddress,
|
|
base_config: C12CompanionConfig,
|
|
) -> None:
|
|
# Arrange — file_exists raises a synthetic OSError
|
|
session = _FakeSession(file_exists_raises=OSError("simulated transient"))
|
|
factory = _FakeFactory(session=session)
|
|
verifier = _ScriptedVerifier()
|
|
bringup = CompanionBringup(
|
|
ssh_factory=factory,
|
|
sidecar_verifier=verifier, # type: ignore[arg-type]
|
|
logger=captured_logger,
|
|
config=base_config,
|
|
)
|
|
|
|
# Act / Assert
|
|
with pytest.raises(OSError, match="simulated transient"):
|
|
bringup.verify_companion_ready(companion_address)
|
|
assert session.close_calls == 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Empty `expected_engines` AC-2 corner case
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestEmptyExpectedEngines:
|
|
def test_empty_expected_engines_marks_not_ready(
|
|
self,
|
|
captured_logger: logging.Logger,
|
|
companion_address: CompanionAddress,
|
|
) -> None:
|
|
# Arrange
|
|
config = C12CompanionConfig(
|
|
ssh_user="azaion",
|
|
ssh_keyfile=Path("/dev/null"),
|
|
host_key_policy=HostKeyPolicy.STRICT,
|
|
companion_cache_root=_CACHE_ROOT,
|
|
expected_engines=(),
|
|
)
|
|
session = _all_present_session()
|
|
factory = _FakeFactory(session=session)
|
|
verifier = _ScriptedVerifier()
|
|
bringup = CompanionBringup(
|
|
ssh_factory=factory,
|
|
sidecar_verifier=verifier, # type: ignore[arg-type]
|
|
logger=captured_logger,
|
|
config=config,
|
|
)
|
|
|
|
# Act
|
|
report = bringup.verify_companion_ready(companion_address)
|
|
|
|
# Assert
|
|
assert report.outcome is ReadinessOutcome.NOT_READY
|
|
assert "expected_engines" in " ".join(report.not_ready_reasons)
|
|
assert report.engines_inspected_count == 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# NFR-perf-cold-call — 100 fake-session runs ≤ 50 ms p99
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestNfrPerfColdCall:
|
|
@pytest.mark.slow
|
|
def test_orchestration_overhead_under_50ms_p99(
|
|
self,
|
|
captured_logger: logging.Logger,
|
|
companion_address: CompanionAddress,
|
|
base_config: C12CompanionConfig,
|
|
) -> None:
|
|
# Arrange
|
|
session = _all_present_session()
|
|
factory = _FakeFactory(session=session)
|
|
verifier = _ScriptedVerifier()
|
|
bringup = CompanionBringup(
|
|
ssh_factory=factory,
|
|
sidecar_verifier=verifier, # type: ignore[arg-type]
|
|
logger=captured_logger,
|
|
config=base_config,
|
|
)
|
|
# Act
|
|
timings_ms: list[float] = []
|
|
for _ in range(100):
|
|
session.close_calls = 0 # reset between runs
|
|
start = time.perf_counter()
|
|
bringup.verify_companion_ready(companion_address)
|
|
timings_ms.append((time.perf_counter() - start) * 1000.0)
|
|
# Assert
|
|
p99 = sorted(timings_ms)[98]
|
|
assert p99 <= 50.0, f"p99={p99:.2f}ms; samples={timings_ms[:5]}..."
|