mirror of
https://github.com/azaion/gps-denied-onboard.git
synced 2026-06-22 16:31:14 +00:00
[AZ-326] [AZ-327] C12 operator-tool CLI + companion SSH bringup
AZ-326 (3pt): operator-tool Click CLI shell at src/gps_denied_onboard/components/c12_operator_tooling/cli.py with six subcommands (download, build-cache, upload-pending, reloc-confirm, verify-ready, set-sector); SectorClassificationStore (atomic-write JSON under ~/.azaion/onboard/sector-classifications.json); freshness-table lookup driving AC-NEW-6; EXIT_* constants; AZ-266 structured-JSON log wiring to a rotating ~/.azaion/onboard/c12-tooling.log handler; operator-tool console-script entry in pyproject.toml. AZ-327 (3pt): CompanionBringup orchestrator at src/gps_denied_onboard/components/c12_operator_tooling/companion_bringup.py that opens an SSH session against the companion (paramiko per project pin), checks the four pre-flight artifacts (Manifest, expected engines, sha256 sidecars, calibration), and returns a ReadinessReport per description.md S2; CompanionUnreachableError + ContentHashMismatchError with operator-friendly remediation hints; ParamikoSshSessionFactory + RemoteSidecarVerifier (sha256sum + cat over SSH, no bytes pulled to the workstation); paramiko>=3.4,<4.0 dep added. NFR-perf-cold-start fix: PEP 562 lazy __getattr__ in c12_operator_tooling/__init__.py and flights_api/__init__.py defers HttpxFlightsApiClient (httpx), ParamikoSshSession[Factory] (paramiko + cryptography), bbox_from_waypoints / takeoff_origin_from_flight (numpy + pyproj). cli.py imports from leaf flights_api modules. operator-tool --help cold start: ~870ms -> <200ms typical, <500ms p99. Includes 73 unit tests (incl. paramiko-version-drift smoke per AZ-327 Risk 1) + console-script integration test. All 1494 repo-wide unit tests pass; 80 skips are pre-existing environment gates. Batch report: _docs/03_implementation/batch_42_cycle1_report.md. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Oleksandr Bezdieniezhnykh
@@ -0,0 +1,474 @@
|
||||
"""AZ-327 — `CompanionBringup.verify_companion_ready` AC-1 .. AC-10."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
import pytest
|
||||
|
||||
from gps_denied_onboard.components.c12_operator_tooling import (
|
||||
C12CompanionConfig,
|
||||
CompanionAddress,
|
||||
CompanionBringup,
|
||||
CompanionUnreachableError,
|
||||
CompanionUnreachableReason,
|
||||
ContentHashMismatchError,
|
||||
HostKeyPolicy,
|
||||
ReadinessOutcome,
|
||||
)
|
||||
from gps_denied_onboard.components.c12_operator_tooling.remote_sidecar_verifier import (
|
||||
RemoteSidecarResult,
|
||||
)
|
||||
from gps_denied_onboard.components.c12_operator_tooling.ssh_session import (
|
||||
RemoteCommandResult,
|
||||
SshSession,
|
||||
SshSessionFactory,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fakes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeSession(SshSession):
|
||||
"""Scripted SSH session for unit tests."""
|
||||
|
||||
files_present: set[str] = field(default_factory=set)
|
||||
dir_contents: dict[str, list[str]] = field(default_factory=dict)
|
||||
file_exists_raises: Exception | None = None
|
||||
close_calls: int = 0
|
||||
|
||||
def run(self, command: str, *, timeout_s: float) -> RemoteCommandResult:
|
||||
return RemoteCommandResult(exit_code=0, stdout="", stderr="")
|
||||
|
||||
def file_exists(self, remote_path: PurePosixPath) -> bool:
|
||||
if self.file_exists_raises is not None:
|
||||
raise self.file_exists_raises
|
||||
return str(remote_path) in self.files_present
|
||||
|
||||
def list_dir(self, remote_path: PurePosixPath) -> list[str]:
|
||||
try:
|
||||
return list(self.dir_contents[str(remote_path)])
|
||||
except KeyError as exc:
|
||||
raise FileNotFoundError(str(remote_path)) from exc
|
||||
|
||||
def close(self) -> None:
|
||||
self.close_calls += 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeFactory(SshSessionFactory):
|
||||
session: _FakeSession | None = None
|
||||
open_raises: Exception | None = None
|
||||
open_calls: int = 0
|
||||
|
||||
def open(
|
||||
self,
|
||||
address: CompanionAddress,
|
||||
*,
|
||||
timeout_s: float,
|
||||
) -> SshSession:
|
||||
self.open_calls += 1
|
||||
if self.open_raises is not None:
|
||||
raise self.open_raises
|
||||
assert self.session is not None
|
||||
return self.session
|
||||
|
||||
|
||||
@dataclass
|
||||
class _ScriptedVerifier:
|
||||
"""Drop-in replacement for `RemoteSidecarVerifier` in unit tests.
|
||||
|
||||
`outcomes_by_engine` keyed by engine filename → :class:`RemoteSidecarResult`.
|
||||
`verify` calls are appended to `verify_calls` for assertion (AC-9).
|
||||
"""
|
||||
|
||||
outcomes_by_engine: dict[str, RemoteSidecarResult] = field(default_factory=dict)
|
||||
verify_calls: list[str] = field(default_factory=list)
|
||||
default: RemoteSidecarResult = field(
|
||||
default_factory=lambda: RemoteSidecarResult(
|
||||
matches=True, expected_hex="aa" * 32, actual_hex="aa" * 32
|
||||
)
|
||||
)
|
||||
|
||||
def verify(
|
||||
self,
|
||||
session: SshSession,
|
||||
engine_path: PurePosixPath,
|
||||
) -> RemoteSidecarResult:
|
||||
engine_name = engine_path.name
|
||||
self.verify_calls.append(engine_name)
|
||||
return self.outcomes_by_engine.get(engine_name, self.default)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
_CACHE_ROOT = PurePosixPath("/var/lib/azaion/c10/cache")
|
||||
_ENGINES_DIR = _CACHE_ROOT / "engines"
|
||||
_ENGINE_A = "dinov2_vpr_sm87_jp62_trt103_fp16.engine"
|
||||
_ENGINE_B = "lightglue_sm87_jp62_trt103_fp16.engine"
|
||||
_MANIFEST = "Manifest.json"
|
||||
_CALIBRATION = "camera_calibration.json"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def captured_logger() -> logging.Logger:
|
||||
logger = logging.getLogger("test.c12.companion_bringup")
|
||||
logger.handlers.clear()
|
||||
logger.setLevel(logging.DEBUG)
|
||||
return logger
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def companion_address() -> CompanionAddress:
|
||||
return CompanionAddress(host="192.168.55.10", port=22)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def base_config() -> C12CompanionConfig:
|
||||
return C12CompanionConfig(
|
||||
ssh_user="azaion",
|
||||
ssh_keyfile=Path("/dev/null"), # not used in fake-driven tests
|
||||
host_key_policy=HostKeyPolicy.STRICT,
|
||||
connect_timeout_s=5.0,
|
||||
companion_cache_root=_CACHE_ROOT,
|
||||
manifest_filename=_MANIFEST,
|
||||
calibration_filename=_CALIBRATION,
|
||||
expected_engines=(_ENGINE_A, _ENGINE_B),
|
||||
)
|
||||
|
||||
|
||||
def _all_present_session() -> _FakeSession:
|
||||
return _FakeSession(
|
||||
files_present={
|
||||
str(_CACHE_ROOT / _MANIFEST),
|
||||
str(_CACHE_ROOT / _CALIBRATION),
|
||||
},
|
||||
dir_contents={str(_ENGINES_DIR): [_ENGINE_A, _ENGINE_B]},
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AC-1 — happy path: outcome=ready
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAC1Ready:
|
||||
def test_all_artifacts_present_outcome_is_ready(
|
||||
self,
|
||||
captured_logger: logging.Logger,
|
||||
companion_address: CompanionAddress,
|
||||
base_config: C12CompanionConfig,
|
||||
) -> None:
|
||||
# Arrange
|
||||
session = _all_present_session()
|
||||
factory = _FakeFactory(session=session)
|
||||
verifier = _ScriptedVerifier()
|
||||
bringup = CompanionBringup(
|
||||
ssh_factory=factory,
|
||||
sidecar_verifier=verifier, # type: ignore[arg-type]
|
||||
logger=captured_logger,
|
||||
config=base_config,
|
||||
)
|
||||
|
||||
# Act
|
||||
report = bringup.verify_companion_ready(companion_address)
|
||||
|
||||
# Assert
|
||||
assert report.outcome is ReadinessOutcome.READY
|
||||
assert report.manifest_present
|
||||
assert report.engines_present
|
||||
assert report.content_hashes_pass
|
||||
assert report.calibration_present
|
||||
assert report.not_ready_reasons == ()
|
||||
assert report.engines_inspected_count == 2
|
||||
assert session.close_calls == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AC-2 — missing engine: outcome=not_ready, no hash mismatch
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAC2MissingEngine:
|
||||
def test_missing_engine_marks_not_ready(
|
||||
self,
|
||||
captured_logger: logging.Logger,
|
||||
companion_address: CompanionAddress,
|
||||
base_config: C12CompanionConfig,
|
||||
) -> None:
|
||||
# Arrange — only ENGINE_A on disk
|
||||
session = _FakeSession(
|
||||
files_present={
|
||||
str(_CACHE_ROOT / _MANIFEST),
|
||||
str(_CACHE_ROOT / _CALIBRATION),
|
||||
},
|
||||
dir_contents={str(_ENGINES_DIR): [_ENGINE_A]},
|
||||
)
|
||||
factory = _FakeFactory(session=session)
|
||||
verifier = _ScriptedVerifier()
|
||||
bringup = CompanionBringup(
|
||||
ssh_factory=factory,
|
||||
sidecar_verifier=verifier, # type: ignore[arg-type]
|
||||
logger=captured_logger,
|
||||
config=base_config,
|
||||
)
|
||||
|
||||
# Act
|
||||
report = bringup.verify_companion_ready(companion_address)
|
||||
|
||||
# Assert
|
||||
assert report.outcome is ReadinessOutcome.NOT_READY
|
||||
assert report.engines_present is False
|
||||
assert any(_ENGINE_B in reason for reason in report.not_ready_reasons)
|
||||
# AC-9 — the missing engine MUST NOT trigger a sidecar verify call
|
||||
assert _ENGINE_B not in verifier.verify_calls
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AC-3 — sidecar mismatch raises ContentHashMismatchError; session closed
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAC3SidecarMismatch:
|
||||
def test_sidecar_mismatch_raises_and_closes_session(
|
||||
self,
|
||||
captured_logger: logging.Logger,
|
||||
companion_address: CompanionAddress,
|
||||
base_config: C12CompanionConfig,
|
||||
) -> None:
|
||||
# Arrange
|
||||
session = _all_present_session()
|
||||
factory = _FakeFactory(session=session)
|
||||
verifier = _ScriptedVerifier(
|
||||
outcomes_by_engine={
|
||||
_ENGINE_A: RemoteSidecarResult(
|
||||
matches=False,
|
||||
expected_hex="aa" * 32,
|
||||
actual_hex="bb" * 32,
|
||||
),
|
||||
}
|
||||
)
|
||||
bringup = CompanionBringup(
|
||||
ssh_factory=factory,
|
||||
sidecar_verifier=verifier, # type: ignore[arg-type]
|
||||
logger=captured_logger,
|
||||
config=base_config,
|
||||
)
|
||||
|
||||
# Act / Assert
|
||||
with pytest.raises(ContentHashMismatchError) as excinfo:
|
||||
bringup.verify_companion_ready(companion_address)
|
||||
assert excinfo.value.expected_sha256_hex == "aa" * 32
|
||||
assert excinfo.value.actual_sha256_hex == "bb" * 32
|
||||
assert _ENGINE_A in excinfo.value.engine_path
|
||||
assert session.close_calls == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AC-4..AC-6, AC-8, AC-10 — session-open failures map to the right reason
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raised,expected_reason,reject_new_first_connect",
|
||||
[
|
||||
(
|
||||
CompanionUnreachableError(
|
||||
host="x",
|
||||
port=22,
|
||||
reason=CompanionUnreachableReason.CONNECT_REFUSED,
|
||||
underlying_exception_repr="ConnectionRefusedError(...)",
|
||||
),
|
||||
CompanionUnreachableReason.CONNECT_REFUSED,
|
||||
False,
|
||||
),
|
||||
(
|
||||
CompanionUnreachableError(
|
||||
host="x",
|
||||
port=22,
|
||||
reason=CompanionUnreachableReason.AUTH_FAILED,
|
||||
underlying_exception_repr="paramiko.AuthenticationException(...)",
|
||||
),
|
||||
CompanionUnreachableReason.AUTH_FAILED,
|
||||
False,
|
||||
),
|
||||
(
|
||||
CompanionUnreachableError(
|
||||
host="x",
|
||||
port=22,
|
||||
reason=CompanionUnreachableReason.HOST_KEY_MISMATCH,
|
||||
underlying_exception_repr="paramiko.BadHostKeyException(...)",
|
||||
),
|
||||
CompanionUnreachableReason.HOST_KEY_MISMATCH,
|
||||
False,
|
||||
),
|
||||
(
|
||||
CompanionUnreachableError(
|
||||
host="x",
|
||||
port=22,
|
||||
reason=CompanionUnreachableReason.TIMEOUT,
|
||||
underlying_exception_repr="socket.timeout(...)",
|
||||
),
|
||||
CompanionUnreachableReason.TIMEOUT,
|
||||
False,
|
||||
),
|
||||
(
|
||||
CompanionUnreachableError(
|
||||
host="x",
|
||||
port=22,
|
||||
reason=CompanionUnreachableReason.HOST_KEY_MISMATCH,
|
||||
underlying_exception_repr="reject_new policy",
|
||||
reject_new_first_connect=True,
|
||||
),
|
||||
CompanionUnreachableReason.HOST_KEY_MISMATCH,
|
||||
True,
|
||||
),
|
||||
],
|
||||
ids=[
|
||||
"AC-4-connect-refused",
|
||||
"AC-5-auth-failed",
|
||||
"AC-6-host-key-mismatch-strict",
|
||||
"AC-8-connect-timeout",
|
||||
"AC-10-reject-new-first-connect",
|
||||
],
|
||||
)
|
||||
class TestSessionOpenFailures:
|
||||
def test_session_open_failure_propagates_with_reason(
|
||||
self,
|
||||
raised: CompanionUnreachableError,
|
||||
expected_reason: CompanionUnreachableReason,
|
||||
reject_new_first_connect: bool,
|
||||
captured_logger: logging.Logger,
|
||||
companion_address: CompanionAddress,
|
||||
base_config: C12CompanionConfig,
|
||||
) -> None:
|
||||
# Arrange
|
||||
factory = _FakeFactory(open_raises=raised)
|
||||
verifier = _ScriptedVerifier()
|
||||
bringup = CompanionBringup(
|
||||
ssh_factory=factory,
|
||||
sidecar_verifier=verifier, # type: ignore[arg-type]
|
||||
logger=captured_logger,
|
||||
config=base_config,
|
||||
)
|
||||
|
||||
# Act / Assert
|
||||
with pytest.raises(CompanionUnreachableError) as excinfo:
|
||||
bringup.verify_companion_ready(companion_address)
|
||||
assert excinfo.value.reason is expected_reason
|
||||
# AC-4..AC-6, AC-8, AC-10 — `remediation` returns a non-empty hint
|
||||
# specific to the reason / reject_new flag.
|
||||
assert isinstance(excinfo.value.remediation, str)
|
||||
assert len(excinfo.value.remediation) > 0
|
||||
if reject_new_first_connect:
|
||||
assert "ssh-keyscan" in excinfo.value.remediation
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AC-7 — session always closed even on unexpected mid-flow exception
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestAC7SessionAlwaysClosed:
|
||||
def test_unexpected_oserror_propagates_and_closes_session(
|
||||
self,
|
||||
captured_logger: logging.Logger,
|
||||
companion_address: CompanionAddress,
|
||||
base_config: C12CompanionConfig,
|
||||
) -> None:
|
||||
# Arrange — file_exists raises a synthetic OSError
|
||||
session = _FakeSession(file_exists_raises=OSError("simulated transient"))
|
||||
factory = _FakeFactory(session=session)
|
||||
verifier = _ScriptedVerifier()
|
||||
bringup = CompanionBringup(
|
||||
ssh_factory=factory,
|
||||
sidecar_verifier=verifier, # type: ignore[arg-type]
|
||||
logger=captured_logger,
|
||||
config=base_config,
|
||||
)
|
||||
|
||||
# Act / Assert
|
||||
with pytest.raises(OSError, match="simulated transient"):
|
||||
bringup.verify_companion_ready(companion_address)
|
||||
assert session.close_calls == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Empty `expected_engines` AC-2 corner case
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEmptyExpectedEngines:
|
||||
def test_empty_expected_engines_marks_not_ready(
|
||||
self,
|
||||
captured_logger: logging.Logger,
|
||||
companion_address: CompanionAddress,
|
||||
) -> None:
|
||||
# Arrange
|
||||
config = C12CompanionConfig(
|
||||
ssh_user="azaion",
|
||||
ssh_keyfile=Path("/dev/null"),
|
||||
host_key_policy=HostKeyPolicy.STRICT,
|
||||
companion_cache_root=_CACHE_ROOT,
|
||||
expected_engines=(),
|
||||
)
|
||||
session = _all_present_session()
|
||||
factory = _FakeFactory(session=session)
|
||||
verifier = _ScriptedVerifier()
|
||||
bringup = CompanionBringup(
|
||||
ssh_factory=factory,
|
||||
sidecar_verifier=verifier, # type: ignore[arg-type]
|
||||
logger=captured_logger,
|
||||
config=config,
|
||||
)
|
||||
|
||||
# Act
|
||||
report = bringup.verify_companion_ready(companion_address)
|
||||
|
||||
# Assert
|
||||
assert report.outcome is ReadinessOutcome.NOT_READY
|
||||
assert "expected_engines" in " ".join(report.not_ready_reasons)
|
||||
assert report.engines_inspected_count == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# NFR-perf-cold-call — 100 fake-session runs ≤ 50 ms p99
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestNfrPerfColdCall:
|
||||
@pytest.mark.slow
|
||||
def test_orchestration_overhead_under_50ms_p99(
|
||||
self,
|
||||
captured_logger: logging.Logger,
|
||||
companion_address: CompanionAddress,
|
||||
base_config: C12CompanionConfig,
|
||||
) -> None:
|
||||
# Arrange
|
||||
session = _all_present_session()
|
||||
factory = _FakeFactory(session=session)
|
||||
verifier = _ScriptedVerifier()
|
||||
bringup = CompanionBringup(
|
||||
ssh_factory=factory,
|
||||
sidecar_verifier=verifier, # type: ignore[arg-type]
|
||||
logger=captured_logger,
|
||||
config=base_config,
|
||||
)
|
||||
# Act
|
||||
timings_ms: list[float] = []
|
||||
for _ in range(100):
|
||||
session.close_calls = 0 # reset between runs
|
||||
start = time.perf_counter()
|
||||
bringup.verify_companion_ready(companion_address)
|
||||
timings_ms.append((time.perf_counter() - start) * 1000.0)
|
||||
# Assert
|
||||
p99 = sorted(timings_ms)[98]
|
||||
assert p99 <= 50.0, f"p99={p99:.2f}ms; samples={timings_ms[:5]}..."
|
||||
Reference in New Issue
Block a user