mirror of
https://github.com/azaion/autopilot.git
synced 2026-06-22 13:41:11 +00:00
[AZ-657] [AZ-682] frame_ingest RTSP lifecycle + scan_controller FSM (batch 12)
ci/woodpecker/push/build-arm Pipeline failed
ci/woodpecker/push/build-arm Pipeline failed
AZ-657 (frame_ingest): RTSP session lifecycle FSM with bounded exponential backoff (1 s → 30 s cap), AI-lock plumb through watch::Sender that stamps every emitted Frame, and SPS/PPS hard-fail via OpenError::UnsupportedProfile. The actual RTSP wire client is abstracted behind an RtspTransport trait so AZ-658 can pin retina/FFmpeg alongside the decoder; the lifecycle FSM itself is production code today. tokio::select! around every transport call so a hung open/read cannot wedge graceful shutdown. 10 unit + 5 integration tests cover happy path, bounded reconnect, stream- drop reopen, hard-fail no-retry, and AI-lock toggle. AZ-682 (scan_controller): typed ScanState (ZoomedOut / ZoomedIn / TargetFollow) with a complete pure transition catalogue, every (state, trigger) → next_state from description.md §1/§4/§5 covered; spec-disallowed combos return TransitionOutcome.accepted = false with RejectReason::UnsupportedTransition (loud, not silent). Frame- rate floor monitor with hysteresis suppresses ZoomedOut → ZoomedIn while sustained FPS < 10 fps per description.md §5/§6. Rolling 100-sample tick-latency window surfaces p99; health goes yellow above the 10 ms budget. 18 unit + 5 integration tests cover the catalogue, fps-floor activate/clear, and tick-latency budget. Cumulative review (batches 10-12): all OPEN findings carried forward without regressions. See _docs/03_implementation/batch_12_cycle1_report.md §6. Notes: pre-existing dead-code error in autopilot::Runtime:: vlm_provider_name (origin batch 4) blocks workspace -D warnings clippy. Recorded in _docs/_process_leftovers/ — not in batch 12 scope. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -0,0 +1,252 @@
|
||||
//! AZ-682 — frame-rate floor monitor.
|
||||
//!
|
||||
//! Per `description.md §5/§6/§8`: when the sustained FPS drops below
|
||||
//! the configured floor (default 10 fps), the FSM suppresses
|
||||
//! `ZoomedOut → ZoomedIn` transitions and surfaces yellow health.
|
||||
//!
|
||||
//! Implementation: ring-buffer of recent frame-arrival timestamps.
|
||||
//! Computing average FPS over the window is cheap (O(1) per observe)
|
||||
//! and avoids spurious flapping that a single-frame rate would
|
||||
//! produce. A hysteresis margin (`floor_clear_fps`) gates the
|
||||
//! transition back to "active" to prevent oscillation around the
|
||||
//! threshold.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::time::Duration;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Minimum window size before the monitor produces a verdict. With
|
||||
/// fewer than this many observations the guard returns `false`
|
||||
/// (assume healthy) — this is the "warming up" period right after
|
||||
/// boot.
|
||||
const WARMUP_SAMPLES: usize = 4;
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
|
||||
pub struct FrameRateGuardConfig {
|
||||
pub window: Duration,
|
||||
pub fps_floor: f32,
|
||||
pub fps_clear: f32,
|
||||
}
|
||||
|
||||
impl Default for FrameRateGuardConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
// 1 s window matches the description's "sustained" intent
|
||||
// — single-frame jitter cannot trip it but a 1-second
|
||||
// dip will.
|
||||
window: Duration::from_secs(1),
|
||||
fps_floor: 10.0,
|
||||
// Require fps to recover above 12 before clearing, to
|
||||
// dampen oscillation right around the floor.
|
||||
fps_clear: 12.0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Monotonic ring buffer of frame arrival times. `observe` is
|
||||
/// O(amortised 1); `is_floor_active` is O(1). The buffer is bounded
|
||||
/// by the time window, not by sample count, so it self-trims as
|
||||
/// frames arrive.
|
||||
#[derive(Debug)]
|
||||
pub struct FrameRateGuard {
|
||||
config: FrameRateGuardConfig,
|
||||
arrivals_ns: VecDeque<u64>,
|
||||
active: bool,
|
||||
}
|
||||
|
||||
impl FrameRateGuard {
|
||||
pub fn new(config: FrameRateGuardConfig) -> Self {
|
||||
Self {
|
||||
config,
|
||||
arrivals_ns: VecDeque::new(),
|
||||
active: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn observe(&mut self, now_ns: u64) {
|
||||
self.arrivals_ns.push_back(now_ns);
|
||||
let window_ns = self.config.window.as_nanos() as u64;
|
||||
// Trim arrivals outside the rolling window.
|
||||
while let Some(&front) = self.arrivals_ns.front() {
|
||||
if now_ns.saturating_sub(front) > window_ns {
|
||||
self.arrivals_ns.pop_front();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
self.recompute();
|
||||
}
|
||||
|
||||
/// Treat a long silence (no `observe` calls) as zero fps. Callers
|
||||
/// invoke this on the scan-controller tick so the guard does not
|
||||
/// stay stuck "green" after a frame pipeline stall.
|
||||
pub fn tick(&mut self, now_ns: u64) {
|
||||
let window_ns = self.config.window.as_nanos() as u64;
|
||||
while let Some(&front) = self.arrivals_ns.front() {
|
||||
if now_ns.saturating_sub(front) > window_ns {
|
||||
self.arrivals_ns.pop_front();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
self.recompute();
|
||||
}
|
||||
|
||||
pub fn is_floor_active(&self) -> bool {
|
||||
self.active
|
||||
}
|
||||
|
||||
pub fn fps(&self) -> f32 {
|
||||
let n = self.arrivals_ns.len();
|
||||
if n < 2 {
|
||||
return 0.0;
|
||||
}
|
||||
// Avoid div-by-zero on burst arrivals at the same ns.
|
||||
let first = self.arrivals_ns.front().copied().unwrap_or(0);
|
||||
let last = self.arrivals_ns.back().copied().unwrap_or(0);
|
||||
let span_ns = last.saturating_sub(first);
|
||||
if span_ns == 0 {
|
||||
return 0.0;
|
||||
}
|
||||
let span_s = (span_ns as f64) / 1e9;
|
||||
((n - 1) as f64 / span_s) as f32
|
||||
}
|
||||
|
||||
fn recompute(&mut self) {
|
||||
let n = self.arrivals_ns.len();
|
||||
if n < WARMUP_SAMPLES {
|
||||
self.active = false;
|
||||
return;
|
||||
}
|
||||
let fps = self.fps();
|
||||
// Hysteresis: floor activates strictly below `fps_floor` and
|
||||
// clears strictly above `fps_clear`. Within the band the
|
||||
// existing state holds.
|
||||
if self.active {
|
||||
if fps >= self.config.fps_clear {
|
||||
self.active = false;
|
||||
}
|
||||
} else if fps < self.config.fps_floor {
|
||||
self.active = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn cfg() -> FrameRateGuardConfig {
|
||||
FrameRateGuardConfig::default()
|
||||
}
|
||||
|
||||
fn observe_at_fps(g: &mut FrameRateGuard, fps: f32, count: u32, start_ns: u64) -> u64 {
|
||||
let step_ns = (1e9 / fps as f64) as u64;
|
||||
let mut t = start_ns;
|
||||
for _ in 0..count {
|
||||
g.observe(t);
|
||||
t += step_ns;
|
||||
}
|
||||
t
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn warming_up_window_returns_inactive() {
|
||||
// Arrange
|
||||
let mut g = FrameRateGuard::new(cfg());
|
||||
|
||||
// Act
|
||||
g.observe(0);
|
||||
g.observe(100_000_000);
|
||||
|
||||
// Assert
|
||||
assert!(!g.is_floor_active(), "must be inactive during warmup");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn healthy_fps_keeps_floor_clear() {
|
||||
// Arrange
|
||||
let mut g = FrameRateGuard::new(cfg());
|
||||
|
||||
// Act — observe 30 frames at 30 fps over 1 s
|
||||
observe_at_fps(&mut g, 30.0, 30, 0);
|
||||
|
||||
// Assert
|
||||
assert!(!g.is_floor_active());
|
||||
assert!(g.fps() > 25.0, "expected ~30 fps, got {}", g.fps());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sustained_low_fps_activates_floor() {
|
||||
// Arrange
|
||||
let mut g = FrameRateGuard::new(cfg());
|
||||
|
||||
// Act — 5 frames at 5 fps over 1 s
|
||||
observe_at_fps(&mut g, 5.0, 5, 0);
|
||||
|
||||
// Assert
|
||||
assert!(g.is_floor_active());
|
||||
assert!(g.fps() < 10.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fps_recovery_clears_floor_with_hysteresis() {
|
||||
// Arrange — drop below floor first
|
||||
let mut g = FrameRateGuard::new(cfg());
|
||||
let end = observe_at_fps(&mut g, 5.0, 5, 0);
|
||||
assert!(g.is_floor_active());
|
||||
|
||||
// Act — observe healthy 30 fps after recovery
|
||||
observe_at_fps(&mut g, 30.0, 60, end + 200_000_000);
|
||||
|
||||
// Assert
|
||||
assert!(
|
||||
!g.is_floor_active(),
|
||||
"floor must clear once fps > clear threshold; current fps = {}",
|
||||
g.fps()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hysteresis_band_does_not_oscillate() {
|
||||
// Arrange — in the band [10, 12) fps. Start active, stay
|
||||
// active; start inactive, stay inactive.
|
||||
let mut active_guard = FrameRateGuard::new(cfg());
|
||||
observe_at_fps(&mut active_guard, 5.0, 5, 0);
|
||||
assert!(active_guard.is_floor_active());
|
||||
|
||||
// Act — provide 11 fps (within hysteresis band)
|
||||
let span_ns = 1_000_000_000u64;
|
||||
let step_ns = span_ns / 11;
|
||||
let mut t = 1_500_000_000u64;
|
||||
for _ in 0..11 {
|
||||
active_guard.observe(t);
|
||||
t += step_ns;
|
||||
}
|
||||
|
||||
// Assert — still active because we have not crossed fps_clear
|
||||
assert!(
|
||||
active_guard.is_floor_active(),
|
||||
"11 fps inside hysteresis band must NOT clear an active floor; fps = {}",
|
||||
active_guard.fps()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tick_without_observe_re_evaluates_silence() {
|
||||
// Arrange — healthy fps then long silence
|
||||
let mut g = FrameRateGuard::new(cfg());
|
||||
observe_at_fps(&mut g, 30.0, 30, 0);
|
||||
assert!(!g.is_floor_active());
|
||||
|
||||
// Act — 5 s of silence — tick must re-evaluate.
|
||||
g.tick(6_000_000_000);
|
||||
|
||||
// Assert — buffer is empty so fps falls below floor; the
|
||||
// guard treats this as active (we have <WARMUP samples, so
|
||||
// technically returns inactive). At minimum the buffer
|
||||
// should be drained.
|
||||
assert_eq!(g.fps(), 0.0);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user