"""AZ-327 — `CompanionBringup.verify_companion_ready` AC-1 .. AC-10.""" from __future__ import annotations import logging import time from dataclasses import dataclass, field from pathlib import Path, PurePosixPath import pytest from gps_denied_onboard.components.c12_operator_tooling import ( C12CompanionConfig, CompanionAddress, CompanionBringup, CompanionUnreachableError, CompanionUnreachableReason, ContentHashMismatchError, HostKeyPolicy, ReadinessOutcome, ) from gps_denied_onboard.components.c12_operator_tooling.remote_sidecar_verifier import ( RemoteSidecarResult, ) from gps_denied_onboard.components.c12_operator_tooling.ssh_session import ( RemoteCommandResult, SshSession, SshSessionFactory, ) # --------------------------------------------------------------------------- # Fakes # --------------------------------------------------------------------------- @dataclass class _FakeSession(SshSession): """Scripted SSH session for unit tests.""" files_present: set[str] = field(default_factory=set) dir_contents: dict[str, list[str]] = field(default_factory=dict) file_exists_raises: Exception | None = None close_calls: int = 0 def run(self, command: str, *, timeout_s: float) -> RemoteCommandResult: return RemoteCommandResult(exit_code=0, stdout="", stderr="") def file_exists(self, remote_path: PurePosixPath) -> bool: if self.file_exists_raises is not None: raise self.file_exists_raises return str(remote_path) in self.files_present def list_dir(self, remote_path: PurePosixPath) -> list[str]: try: return list(self.dir_contents[str(remote_path)]) except KeyError as exc: raise FileNotFoundError(str(remote_path)) from exc def close(self) -> None: self.close_calls += 1 @dataclass class _FakeFactory(SshSessionFactory): session: _FakeSession | None = None open_raises: Exception | None = None open_calls: int = 0 def open( self, address: CompanionAddress, *, timeout_s: float, ) -> SshSession: self.open_calls += 1 if self.open_raises is not None: raise self.open_raises assert self.session is not None return self.session @dataclass class _ScriptedVerifier: """Drop-in replacement for `RemoteSidecarVerifier` in unit tests. `outcomes_by_engine` keyed by engine filename → :class:`RemoteSidecarResult`. `verify` calls are appended to `verify_calls` for assertion (AC-9). """ outcomes_by_engine: dict[str, RemoteSidecarResult] = field(default_factory=dict) verify_calls: list[str] = field(default_factory=list) default: RemoteSidecarResult = field( default_factory=lambda: RemoteSidecarResult( matches=True, expected_hex="aa" * 32, actual_hex="aa" * 32 ) ) def verify( self, session: SshSession, engine_path: PurePosixPath, ) -> RemoteSidecarResult: engine_name = engine_path.name self.verify_calls.append(engine_name) return self.outcomes_by_engine.get(engine_name, self.default) # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- _CACHE_ROOT = PurePosixPath("/var/lib/azaion/c10/cache") _ENGINES_DIR = _CACHE_ROOT / "engines" _ENGINE_A = "dinov2_vpr_sm87_jp62_trt103_fp16.engine" _ENGINE_B = "lightglue_sm87_jp62_trt103_fp16.engine" _MANIFEST = "Manifest.json" _CALIBRATION = "camera_calibration.json" @pytest.fixture def captured_logger() -> logging.Logger: logger = logging.getLogger("test.c12.companion_bringup") logger.handlers.clear() logger.setLevel(logging.DEBUG) return logger @pytest.fixture def companion_address() -> CompanionAddress: return CompanionAddress(host="192.168.55.10", port=22) @pytest.fixture def base_config() -> C12CompanionConfig: return C12CompanionConfig( ssh_user="azaion", ssh_keyfile=Path("/dev/null"), # not used in fake-driven tests host_key_policy=HostKeyPolicy.STRICT, connect_timeout_s=5.0, companion_cache_root=_CACHE_ROOT, manifest_filename=_MANIFEST, calibration_filename=_CALIBRATION, expected_engines=(_ENGINE_A, _ENGINE_B), ) def _all_present_session() -> _FakeSession: return _FakeSession( files_present={ str(_CACHE_ROOT / _MANIFEST), str(_CACHE_ROOT / _CALIBRATION), }, dir_contents={str(_ENGINES_DIR): [_ENGINE_A, _ENGINE_B]}, ) # --------------------------------------------------------------------------- # AC-1 — happy path: outcome=ready # --------------------------------------------------------------------------- class TestAC1Ready: def test_all_artifacts_present_outcome_is_ready( self, captured_logger: logging.Logger, companion_address: CompanionAddress, base_config: C12CompanionConfig, ) -> None: # Arrange session = _all_present_session() factory = _FakeFactory(session=session) verifier = _ScriptedVerifier() bringup = CompanionBringup( ssh_factory=factory, sidecar_verifier=verifier, # type: ignore[arg-type] logger=captured_logger, config=base_config, ) # Act report = bringup.verify_companion_ready(companion_address) # Assert assert report.outcome is ReadinessOutcome.READY assert report.manifest_present assert report.engines_present assert report.content_hashes_pass assert report.calibration_present assert report.not_ready_reasons == () assert report.engines_inspected_count == 2 assert session.close_calls == 1 # --------------------------------------------------------------------------- # AC-2 — missing engine: outcome=not_ready, no hash mismatch # --------------------------------------------------------------------------- class TestAC2MissingEngine: def test_missing_engine_marks_not_ready( self, captured_logger: logging.Logger, companion_address: CompanionAddress, base_config: C12CompanionConfig, ) -> None: # Arrange — only ENGINE_A on disk session = _FakeSession( files_present={ str(_CACHE_ROOT / _MANIFEST), str(_CACHE_ROOT / _CALIBRATION), }, dir_contents={str(_ENGINES_DIR): [_ENGINE_A]}, ) factory = _FakeFactory(session=session) verifier = _ScriptedVerifier() bringup = CompanionBringup( ssh_factory=factory, sidecar_verifier=verifier, # type: ignore[arg-type] logger=captured_logger, config=base_config, ) # Act report = bringup.verify_companion_ready(companion_address) # Assert assert report.outcome is ReadinessOutcome.NOT_READY assert report.engines_present is False assert any(_ENGINE_B in reason for reason in report.not_ready_reasons) # AC-9 — the missing engine MUST NOT trigger a sidecar verify call assert _ENGINE_B not in verifier.verify_calls # --------------------------------------------------------------------------- # AC-3 — sidecar mismatch raises ContentHashMismatchError; session closed # --------------------------------------------------------------------------- class TestAC3SidecarMismatch: def test_sidecar_mismatch_raises_and_closes_session( self, captured_logger: logging.Logger, companion_address: CompanionAddress, base_config: C12CompanionConfig, ) -> None: # Arrange session = _all_present_session() factory = _FakeFactory(session=session) verifier = _ScriptedVerifier( outcomes_by_engine={ _ENGINE_A: RemoteSidecarResult( matches=False, expected_hex="aa" * 32, actual_hex="bb" * 32, ), } ) bringup = CompanionBringup( ssh_factory=factory, sidecar_verifier=verifier, # type: ignore[arg-type] logger=captured_logger, config=base_config, ) # Act / Assert with pytest.raises(ContentHashMismatchError) as excinfo: bringup.verify_companion_ready(companion_address) assert excinfo.value.expected_sha256_hex == "aa" * 32 assert excinfo.value.actual_sha256_hex == "bb" * 32 assert _ENGINE_A in excinfo.value.engine_path assert session.close_calls == 1 # --------------------------------------------------------------------------- # AC-4..AC-6, AC-8, AC-10 — session-open failures map to the right reason # --------------------------------------------------------------------------- @pytest.mark.parametrize( "raised,expected_reason,reject_new_first_connect", [ ( CompanionUnreachableError( host="x", port=22, reason=CompanionUnreachableReason.CONNECT_REFUSED, underlying_exception_repr="ConnectionRefusedError(...)", ), CompanionUnreachableReason.CONNECT_REFUSED, False, ), ( CompanionUnreachableError( host="x", port=22, reason=CompanionUnreachableReason.AUTH_FAILED, underlying_exception_repr="paramiko.AuthenticationException(...)", ), CompanionUnreachableReason.AUTH_FAILED, False, ), ( CompanionUnreachableError( host="x", port=22, reason=CompanionUnreachableReason.HOST_KEY_MISMATCH, underlying_exception_repr="paramiko.BadHostKeyException(...)", ), CompanionUnreachableReason.HOST_KEY_MISMATCH, False, ), ( CompanionUnreachableError( host="x", port=22, reason=CompanionUnreachableReason.TIMEOUT, underlying_exception_repr="socket.timeout(...)", ), CompanionUnreachableReason.TIMEOUT, False, ), ( CompanionUnreachableError( host="x", port=22, reason=CompanionUnreachableReason.HOST_KEY_MISMATCH, underlying_exception_repr="reject_new policy", reject_new_first_connect=True, ), CompanionUnreachableReason.HOST_KEY_MISMATCH, True, ), ], ids=[ "AC-4-connect-refused", "AC-5-auth-failed", "AC-6-host-key-mismatch-strict", "AC-8-connect-timeout", "AC-10-reject-new-first-connect", ], ) class TestSessionOpenFailures: def test_session_open_failure_propagates_with_reason( self, raised: CompanionUnreachableError, expected_reason: CompanionUnreachableReason, reject_new_first_connect: bool, captured_logger: logging.Logger, companion_address: CompanionAddress, base_config: C12CompanionConfig, ) -> None: # Arrange factory = _FakeFactory(open_raises=raised) verifier = _ScriptedVerifier() bringup = CompanionBringup( ssh_factory=factory, sidecar_verifier=verifier, # type: ignore[arg-type] logger=captured_logger, config=base_config, ) # Act / Assert with pytest.raises(CompanionUnreachableError) as excinfo: bringup.verify_companion_ready(companion_address) assert excinfo.value.reason is expected_reason # AC-4..AC-6, AC-8, AC-10 — `remediation` returns a non-empty hint # specific to the reason / reject_new flag. assert isinstance(excinfo.value.remediation, str) assert len(excinfo.value.remediation) > 0 if reject_new_first_connect: assert "ssh-keyscan" in excinfo.value.remediation # --------------------------------------------------------------------------- # AC-7 — session always closed even on unexpected mid-flow exception # --------------------------------------------------------------------------- class TestAC7SessionAlwaysClosed: def test_unexpected_oserror_propagates_and_closes_session( self, captured_logger: logging.Logger, companion_address: CompanionAddress, base_config: C12CompanionConfig, ) -> None: # Arrange — file_exists raises a synthetic OSError session = _FakeSession(file_exists_raises=OSError("simulated transient")) factory = _FakeFactory(session=session) verifier = _ScriptedVerifier() bringup = CompanionBringup( ssh_factory=factory, sidecar_verifier=verifier, # type: ignore[arg-type] logger=captured_logger, config=base_config, ) # Act / Assert with pytest.raises(OSError, match="simulated transient"): bringup.verify_companion_ready(companion_address) assert session.close_calls == 1 # --------------------------------------------------------------------------- # Empty `expected_engines` AC-2 corner case # --------------------------------------------------------------------------- class TestEmptyExpectedEngines: def test_empty_expected_engines_marks_not_ready( self, captured_logger: logging.Logger, companion_address: CompanionAddress, ) -> None: # Arrange config = C12CompanionConfig( ssh_user="azaion", ssh_keyfile=Path("/dev/null"), host_key_policy=HostKeyPolicy.STRICT, companion_cache_root=_CACHE_ROOT, expected_engines=(), ) session = _all_present_session() factory = _FakeFactory(session=session) verifier = _ScriptedVerifier() bringup = CompanionBringup( ssh_factory=factory, sidecar_verifier=verifier, # type: ignore[arg-type] logger=captured_logger, config=config, ) # Act report = bringup.verify_companion_ready(companion_address) # Assert assert report.outcome is ReadinessOutcome.NOT_READY assert "expected_engines" in " ".join(report.not_ready_reasons) assert report.engines_inspected_count == 0 # --------------------------------------------------------------------------- # NFR-perf-cold-call — 100 fake-session runs ≤ 50 ms p99 # --------------------------------------------------------------------------- class TestNfrPerfColdCall: @pytest.mark.slow def test_orchestration_overhead_under_50ms_p99( self, captured_logger: logging.Logger, companion_address: CompanionAddress, base_config: C12CompanionConfig, ) -> None: # Arrange session = _all_present_session() factory = _FakeFactory(session=session) verifier = _ScriptedVerifier() bringup = CompanionBringup( ssh_factory=factory, sidecar_verifier=verifier, # type: ignore[arg-type] logger=captured_logger, config=base_config, ) # Act timings_ms: list[float] = [] for _ in range(100): session.close_calls = 0 # reset between runs start = time.perf_counter() bringup.verify_companion_ready(companion_address) timings_ms.append((time.perf_counter() - start) * 1000.0) # Assert p99 = sorted(timings_ms)[98] assert p99 <= 50.0, f"p99={p99:.2f}ms; samples={timings_ms[:5]}..."