mirror of
https://github.com/azaion/autopilot.git
synced 2026-06-21 18:11:10 +00:00
8a4bd00526
AZ-650 (mission_executor pre-flight Built-In Test):
- BitEvaluator trait + BitItemStatus { Pass, Degraded, Fail, Skipped }
+ BitReport + BitOverall fusion. Pluggable per-item evaluators so
the composition root decides which dependencies are wired today.
- BitController owns evaluator list + mpsc ack channel + sticky-pass
+ ack deadline. Publishes bit_ok via tokio watch — composition root
pipes it into the telemetry projection where the existing FSM
bit_ok guard already consumes it (no FSM changes needed).
- BitState { Idle, Pass, AwaitingAck { report_id }, Failed { reason } }
with broadcast::Sender<BitEvent> for operator-side observability.
Sticky-pass semantics: once Pass is reached (directly or via signed
ack on a Degraded report), the controller stops re-evaluating —
BIT is a one-shot pre-flight gate, not a continuous monitor.
- BitDegradedAck arrives pre-validated by operator_bridge; the
controller only matches report_id and applies the operator id to
the audit log.
- Concrete evaluators landed today (3 of 12 spec items, the rest
depend on components still in todo/):
- StateDirFreeSpaceEvaluator (dir creatable/readable; statvfs is
documented follow-up).
- WallClockBoundEvaluator (chrono::Utc::now vs configurable bound).
- MissionLoadedEvaluator (waypoint count via Arc<Mutex<usize>>).
- MapObjectsSyncedEvaluator (maps SyncState -> BIT status per Q9).
Tests:
- ac1_all_pass_proceeds, ac2_fail_blocks_transition,
ac3_degraded_requires_signed_ack (+ mismatched_ack supplement),
ac4_degraded_ack_timeout_fails_the_bit — all 4 ACs green.
- Pure next_state table covered by lib unit tests.
- Per-evaluator unit tests for Pass/Fail/Degraded branches.
Quality gates:
- cargo fmt: clean.
- cargo clippy -p mission_executor --tests -- -D warnings: 0 warns.
- cargo test --workspace: all green.
- Pre-existing flake in state_machine::ac3_bounded_retry_then_success
(batch 7 report) remains pre-existing — passes on rerun.
Co-authored-by: Cursor <cursoragent@cursor.com>
299 lines
10 KiB
Rust
299 lines
10 KiB
Rust
//! AZ-650 acceptance criteria — Pre-flight Built-In Test (F9).
|
|
//!
|
|
//! Tests the controller via its public surface using mock
|
|
//! [`BitEvaluator`]s. The FSM integration ("machine transitions to
|
|
//! BIT_OK") is one watch-channel hop away — the controller publishes
|
|
//! `bit_ok` and the composition root pipes that into the telemetry
|
|
//! projection. We assert the controller side (`bit_ok = true` exactly
|
|
//! when state == Pass) which is the test seam the composition root
|
|
//! consumes.
|
|
|
|
use std::sync::Arc;
|
|
use std::time::{Duration, Instant as StdInstant};
|
|
|
|
use mission_executor::{
|
|
BitController, BitControllerConfig, BitDegradedAck, BitEvaluator, BitItemStatus, BitOverall,
|
|
BitState,
|
|
};
|
|
use tokio::sync::{mpsc, watch};
|
|
|
|
/// Static-status evaluator for tests.
|
|
struct StaticEvaluator {
|
|
name: &'static str,
|
|
status: std::sync::Mutex<BitItemStatus>,
|
|
}
|
|
impl StaticEvaluator {
|
|
fn new(name: &'static str, status: BitItemStatus) -> Arc<Self> {
|
|
Arc::new(Self {
|
|
name,
|
|
status: std::sync::Mutex::new(status),
|
|
})
|
|
}
|
|
#[allow(dead_code)]
|
|
fn set(&self, status: BitItemStatus) {
|
|
*self.status.lock().unwrap() = status;
|
|
}
|
|
}
|
|
impl BitEvaluator for StaticEvaluator {
|
|
fn name(&self) -> &'static str {
|
|
self.name
|
|
}
|
|
fn evaluate(&self) -> BitItemStatus {
|
|
self.status.lock().unwrap().clone()
|
|
}
|
|
}
|
|
|
|
fn fast_config(ack_timeout: Duration) -> BitControllerConfig {
|
|
BitControllerConfig {
|
|
evaluation_interval: Duration::from_millis(20),
|
|
ack_timeout,
|
|
}
|
|
}
|
|
|
|
/// Wait until `predicate` returns `true`, polling every 10 ms. Panics
|
|
/// on `deadline`.
|
|
async fn wait_for<F>(label: &str, deadline: StdInstant, mut predicate: F)
|
|
where
|
|
F: FnMut() -> bool,
|
|
{
|
|
loop {
|
|
if predicate() {
|
|
return;
|
|
}
|
|
if StdInstant::now() >= deadline {
|
|
panic!("timed out waiting for {label}");
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(10)).await;
|
|
}
|
|
}
|
|
|
|
/// AC-1 — every dependency healthy → controller transitions to Pass
|
|
/// and `bit_ok` flips to `true`.
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn ac1_all_pass_proceeds() {
|
|
// Arrange — three evaluators, all Pass
|
|
let evaluators: Vec<Arc<dyn BitEvaluator>> = vec![
|
|
StaticEvaluator::new("mavlink_link", BitItemStatus::Pass),
|
|
StaticEvaluator::new("mission_loaded", BitItemStatus::Pass),
|
|
StaticEvaluator::new("state_dir_free_space", BitItemStatus::Pass),
|
|
];
|
|
let (_ack_tx, ack_rx) = mpsc::channel::<BitDegradedAck>(8);
|
|
let controller = BitController::new(fast_config(Duration::from_secs(60)), evaluators, ack_rx);
|
|
let (shutdown_tx, shutdown_rx) = watch::channel(false);
|
|
let (handle, join) = controller.spawn(shutdown_rx);
|
|
|
|
// Act — let the controller evaluate at least once
|
|
let mut bit_ok_rx = handle.bit_ok();
|
|
let mut state_rx = handle.state();
|
|
let deadline = StdInstant::now() + Duration::from_secs(2);
|
|
wait_for("bit_ok = true", deadline, || *bit_ok_rx.borrow_and_update()).await;
|
|
|
|
// Assert
|
|
assert!(*bit_ok_rx.borrow());
|
|
assert_eq!(*state_rx.borrow_and_update(), BitState::Pass);
|
|
let report = handle.last_report().await.expect("report generated");
|
|
assert_eq!(report.overall, BitOverall::Pass);
|
|
assert_eq!(report.items.len(), 3);
|
|
|
|
// Cleanup
|
|
shutdown_tx.send(true).unwrap();
|
|
let _ = join.await;
|
|
}
|
|
|
|
/// AC-2 — `camera_rtsp` reports Fail → `bit_ok = false`; controller
|
|
/// stays Failed; FSM (downstream of `bit_ok`) does NOT transition.
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn ac2_fail_blocks_transition() {
|
|
// Arrange — one Fail evaluator
|
|
let evaluators: Vec<Arc<dyn BitEvaluator>> = vec![
|
|
StaticEvaluator::new("mavlink_link", BitItemStatus::Pass),
|
|
StaticEvaluator::new(
|
|
"camera_rtsp",
|
|
BitItemStatus::Fail {
|
|
detail: "no RTSP peer".into(),
|
|
},
|
|
),
|
|
];
|
|
let (_ack_tx, ack_rx) = mpsc::channel::<BitDegradedAck>(8);
|
|
let controller = BitController::new(fast_config(Duration::from_secs(60)), evaluators, ack_rx);
|
|
let (shutdown_tx, shutdown_rx) = watch::channel(false);
|
|
let (handle, join) = controller.spawn(shutdown_rx);
|
|
|
|
// Act — wait for one evaluation cycle
|
|
let mut state_rx = handle.state();
|
|
let deadline = StdInstant::now() + Duration::from_secs(2);
|
|
wait_for("state != Idle", deadline, || {
|
|
!matches!(*state_rx.borrow_and_update(), BitState::Idle)
|
|
})
|
|
.await;
|
|
|
|
// Assert — bit_ok is false; state is Failed; report is observable
|
|
let bit_ok = *handle.bit_ok().borrow();
|
|
assert!(!bit_ok, "bit_ok must remain false on Fail");
|
|
match state_rx.borrow().clone() {
|
|
BitState::Failed { reason } => assert!(reason.contains("camera_rtsp")),
|
|
other => panic!("expected Failed, got {other:?}"),
|
|
}
|
|
let report = handle.last_report().await.unwrap();
|
|
assert_eq!(report.overall, BitOverall::Fail);
|
|
|
|
// Cleanup
|
|
shutdown_tx.send(true).unwrap();
|
|
let _ = join.await;
|
|
}
|
|
|
|
/// AC-3 — Degraded → controller enters AwaitingAck → signed ack with
|
|
/// matching report_id flips to Pass and `bit_ok = true`.
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn ac3_degraded_requires_signed_ack() {
|
|
// Arrange — Degraded evaluator (e.g. mapobjects on cached fallback)
|
|
let evaluators: Vec<Arc<dyn BitEvaluator>> = vec![
|
|
StaticEvaluator::new("mavlink_link", BitItemStatus::Pass),
|
|
StaticEvaluator::new(
|
|
"mapobjects_synced_or_cached_acked",
|
|
BitItemStatus::Degraded {
|
|
detail: "operating on cached fallback".into(),
|
|
},
|
|
),
|
|
];
|
|
let (ack_tx, ack_rx) = mpsc::channel::<BitDegradedAck>(8);
|
|
let controller = BitController::new(fast_config(Duration::from_secs(60)), evaluators, ack_rx);
|
|
let (shutdown_tx, shutdown_rx) = watch::channel(false);
|
|
let (handle, join) = controller.spawn(shutdown_rx);
|
|
|
|
// Act — wait for AwaitingAck state
|
|
let mut state_rx = handle.state();
|
|
let deadline = StdInstant::now() + Duration::from_secs(2);
|
|
wait_for("state == AwaitingAck", deadline, || {
|
|
matches!(*state_rx.borrow_and_update(), BitState::AwaitingAck { .. })
|
|
})
|
|
.await;
|
|
|
|
let report_id = match state_rx.borrow().clone() {
|
|
BitState::AwaitingAck { report_id } => report_id,
|
|
other => panic!("expected AwaitingAck, got {other:?}"),
|
|
};
|
|
|
|
// `bit_ok` is still false while awaiting ack
|
|
assert!(!*handle.bit_ok().borrow());
|
|
|
|
// Act — send a matching signed ack
|
|
ack_tx
|
|
.send(BitDegradedAck {
|
|
report_id,
|
|
operator_id: Some("op-A".into()),
|
|
})
|
|
.await
|
|
.unwrap();
|
|
|
|
// Wait for state → Pass
|
|
let mut bit_ok_rx = handle.bit_ok();
|
|
let deadline = StdInstant::now() + Duration::from_secs(2);
|
|
wait_for("bit_ok = true after ack", deadline, || {
|
|
*bit_ok_rx.borrow_and_update()
|
|
})
|
|
.await;
|
|
|
|
// Assert
|
|
assert!(*bit_ok_rx.borrow());
|
|
assert_eq!(*state_rx.borrow_and_update(), BitState::Pass);
|
|
|
|
// Cleanup
|
|
shutdown_tx.send(true).unwrap();
|
|
let _ = join.await;
|
|
}
|
|
|
|
/// AC-3 supplement — an ack with a *different* report_id is ignored;
|
|
/// controller stays AwaitingAck.
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn ac3_mismatched_ack_is_ignored() {
|
|
// Arrange
|
|
let evaluators: Vec<Arc<dyn BitEvaluator>> = vec![StaticEvaluator::new(
|
|
"mapobjects_synced_or_cached_acked",
|
|
BitItemStatus::Degraded {
|
|
detail: "cached fallback".into(),
|
|
},
|
|
)];
|
|
let (ack_tx, ack_rx) = mpsc::channel::<BitDegradedAck>(8);
|
|
let controller = BitController::new(fast_config(Duration::from_secs(60)), evaluators, ack_rx);
|
|
let (shutdown_tx, shutdown_rx) = watch::channel(false);
|
|
let (handle, join) = controller.spawn(shutdown_rx);
|
|
|
|
let mut state_rx = handle.state();
|
|
let deadline = StdInstant::now() + Duration::from_secs(2);
|
|
wait_for("state == AwaitingAck", deadline, || {
|
|
matches!(*state_rx.borrow_and_update(), BitState::AwaitingAck { .. })
|
|
})
|
|
.await;
|
|
|
|
// Act — send an ack with a bogus report_id
|
|
ack_tx
|
|
.send(BitDegradedAck {
|
|
report_id: uuid::Uuid::nil(),
|
|
operator_id: Some("op-A".into()),
|
|
})
|
|
.await
|
|
.unwrap();
|
|
|
|
// Give the controller time to process the mismatch
|
|
tokio::time::sleep(Duration::from_millis(100)).await;
|
|
|
|
// Assert — still AwaitingAck; bit_ok still false
|
|
assert!(matches!(
|
|
*state_rx.borrow_and_update(),
|
|
BitState::AwaitingAck { .. }
|
|
));
|
|
assert!(!*handle.bit_ok().borrow());
|
|
|
|
// Cleanup
|
|
shutdown_tx.send(true).unwrap();
|
|
let _ = join.await;
|
|
}
|
|
|
|
/// AC-4 — Degraded ack timeout transitions to Failed; `bit_ok` stays
|
|
/// false.
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
|
async fn ac4_degraded_ack_timeout_fails_the_bit() {
|
|
// Arrange — short ack timeout
|
|
let evaluators: Vec<Arc<dyn BitEvaluator>> = vec![StaticEvaluator::new(
|
|
"mapobjects_synced_or_cached_acked",
|
|
BitItemStatus::Degraded {
|
|
detail: "cached fallback".into(),
|
|
},
|
|
)];
|
|
let (_ack_tx, ack_rx) = mpsc::channel::<BitDegradedAck>(8);
|
|
let controller =
|
|
BitController::new(fast_config(Duration::from_millis(200)), evaluators, ack_rx);
|
|
let (shutdown_tx, shutdown_rx) = watch::channel(false);
|
|
let (handle, join) = controller.spawn(shutdown_rx);
|
|
|
|
// Wait for AwaitingAck
|
|
let mut state_rx = handle.state();
|
|
let deadline = StdInstant::now() + Duration::from_secs(2);
|
|
wait_for("state == AwaitingAck", deadline, || {
|
|
matches!(*state_rx.borrow_and_update(), BitState::AwaitingAck { .. })
|
|
})
|
|
.await;
|
|
|
|
// Act — don't ack; let the timeout fire (200 ms ack_timeout + slack)
|
|
let deadline = StdInstant::now() + Duration::from_secs(2);
|
|
wait_for("state == Failed", deadline, || {
|
|
matches!(*state_rx.borrow_and_update(), BitState::Failed { .. })
|
|
})
|
|
.await;
|
|
|
|
// Assert
|
|
match state_rx.borrow().clone() {
|
|
BitState::Failed { reason } => assert!(
|
|
reason.contains("ack_timeout"),
|
|
"Failed reason should mention ack_timeout, got {reason}"
|
|
),
|
|
other => panic!("expected Failed, got {other:?}"),
|
|
}
|
|
assert!(!*handle.bit_ok().borrow());
|
|
|
|
// Cleanup
|
|
shutdown_tx.send(true).unwrap();
|
|
let _ = join.await;
|
|
}
|