mirror of
https://github.com/azaion/autopilot.git
synced 2026-06-21 15:41:09 +00:00
[AZ-662] [AZ-669] Implement ego-motion estimator and primitive graph
AZ-662: movement_detector ego-motion - Add opencv + petgraph to workspace dependencies - internal/zoom_bands: per-band telemetry skew tolerances - internal/telemetry_sync: skew gate (check_skew) - internal/optical_flow: frame→gray, degenerate detection, LK sparse flow + RANSAC homography estimation - internal/ego_motion: EgoMotionEstimator + atomic counters AZ-669: semantic_analyzer primitive graph - internal/primitive_graph: NodeType, PrimitiveNode, PrimitiveGraph, PrimitiveGraphBuilder with proximity-adjacency + BFS connectivity check - internal/scoring/freshness: FreshnessScorer (Laplacian variance, texture stddev, undisturbed-surroundings heuristic) - All ACs covered by unit tests (AC-1/2/3 per task) Note: native OpenCV not installed on macOS; authoritative test is cargo test --workspace on Jetson (ssh jetson-e2e). Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -16,5 +16,7 @@ learned_cv = []
|
||||
shared = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
opencv = { workspace = true }
|
||||
|
||||
# OpenCV / homography deps land with AZ-662 (`movement_detector_ego_motion`).
|
||||
[dev-dependencies]
|
||||
bytes = { workspace = true }
|
||||
|
||||
@@ -0,0 +1,388 @@
|
||||
//! AZ-662 — Ego-motion estimator + telemetry-skew gate.
|
||||
//!
|
||||
//! `EgoMotionEstimator::estimate` checks gimbal/UAV timestamp skew against the
|
||||
//! per-zoom-band tolerance, then runs OpenCV Lucas–Kanade optical-flow +
|
||||
//! RANSAC homography on consecutive grayscale frames to recover camera motion.
|
||||
|
||||
use std::sync::{
|
||||
atomic::{AtomicU64, Ordering},
|
||||
Arc,
|
||||
};
|
||||
|
||||
use opencv::{core::Mat, prelude::*};
|
||||
|
||||
use shared::models::{
|
||||
frame::Frame,
|
||||
gimbal::GimbalState,
|
||||
movement::ZoomBand,
|
||||
telemetry::UavTelemetry,
|
||||
};
|
||||
|
||||
use super::{
|
||||
optical_flow::{self, FlowError},
|
||||
telemetry_sync::{self, SkewExceeded},
|
||||
zoom_bands::zoom_band_from_level,
|
||||
};
|
||||
|
||||
/// Per-frame ego-motion recovered from optical flow.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EgoMotion {
|
||||
/// Row-major 3×3 homography mapping the previous frame's coordinates to
|
||||
/// the current frame's coordinates (camera ego-motion).
|
||||
pub homography: [f64; 9],
|
||||
/// Mean reprojection residual across inlier feature tracks (pixels).
|
||||
pub residual_motion_magnitude: f32,
|
||||
pub zoom_band: ZoomBand,
|
||||
}
|
||||
|
||||
/// Error variants returned by `EgoMotionEstimator::estimate`.
|
||||
#[derive(Debug)]
|
||||
pub enum EgoMotionError {
|
||||
/// Frame ↔ gimbal or frame ↔ UAV timestamp skew exceeded the per-band
|
||||
/// tolerance. The affected frame must not be used for ego-motion.
|
||||
SkewExceeded(SkewExceeded),
|
||||
/// The current frame is degenerate (saturated, blank, or featureless).
|
||||
/// The frame is stored internally so the next call can resume.
|
||||
OpticalFlowDegenerate,
|
||||
/// No previous frame has been received yet; the current frame is stored
|
||||
/// as the reference for the next call.
|
||||
NoPreviousFrame,
|
||||
Internal(String),
|
||||
}
|
||||
|
||||
impl From<SkewExceeded> for EgoMotionError {
|
||||
fn from(e: SkewExceeded) -> Self {
|
||||
EgoMotionError::SkewExceeded(e)
|
||||
}
|
||||
}
|
||||
|
||||
/// Atomic health counters exposed through `MovementDetectorHandle::health()`.
|
||||
pub struct EgoMotionCounters {
|
||||
pub telemetry_skew_drops_zoomed_out: AtomicU64,
|
||||
pub telemetry_skew_drops_zoomed_in: AtomicU64,
|
||||
pub optical_flow_degenerate_total: AtomicU64,
|
||||
}
|
||||
|
||||
impl EgoMotionCounters {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
telemetry_skew_drops_zoomed_out: AtomicU64::new(0),
|
||||
telemetry_skew_drops_zoomed_in: AtomicU64::new(0),
|
||||
optical_flow_degenerate_total: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn skew_drops(&self, band: ZoomBand) -> u64 {
|
||||
match band {
|
||||
ZoomBand::ZoomedOut => {
|
||||
self.telemetry_skew_drops_zoomed_out.load(Ordering::Relaxed)
|
||||
}
|
||||
ZoomBand::ZoomedIn => {
|
||||
self.telemetry_skew_drops_zoomed_in.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn skew_drops_total(&self) -> u64 {
|
||||
self.skew_drops(ZoomBand::ZoomedOut) + self.skew_drops(ZoomBand::ZoomedIn)
|
||||
}
|
||||
|
||||
pub fn degenerate_total(&self) -> u64 {
|
||||
self.optical_flow_degenerate_total.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
fn inc_skew_drop(&self, band: ZoomBand) {
|
||||
match band {
|
||||
ZoomBand::ZoomedOut => {
|
||||
self.telemetry_skew_drops_zoomed_out.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
ZoomBand::ZoomedIn => {
|
||||
self.telemetry_skew_drops_zoomed_in.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn inc_degenerate(&self) {
|
||||
self.optical_flow_degenerate_total.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for EgoMotionCounters {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Stateful per-frame ego-motion estimator.
|
||||
///
|
||||
/// Call `estimate` once per frame in arrival order. The estimator keeps the
|
||||
/// previous frame's grayscale Mat internally; the first call always returns
|
||||
/// `Err(NoPreviousFrame)` and stores the frame as the reference.
|
||||
pub struct EgoMotionEstimator {
|
||||
prev_gray: Option<Mat>,
|
||||
counters: Arc<EgoMotionCounters>,
|
||||
}
|
||||
|
||||
impl EgoMotionEstimator {
|
||||
pub fn new(counters: Arc<EgoMotionCounters>) -> Self {
|
||||
Self { prev_gray: None, counters }
|
||||
}
|
||||
|
||||
pub fn counters(&self) -> &Arc<EgoMotionCounters> {
|
||||
&self.counters
|
||||
}
|
||||
|
||||
/// Estimate ego-motion for `frame` relative to the previous accepted frame.
|
||||
///
|
||||
/// Processing order:
|
||||
/// 1. Telemetry-skew gate (increments `telemetry_skew_drops_total` on miss).
|
||||
/// 2. Convert to grayscale.
|
||||
/// 3. Degenerate-frame detection (increments `optical_flow_degenerate_total`).
|
||||
/// 4. Require a previous accepted frame; store current if none.
|
||||
/// 5. LK optical flow + RANSAC homography.
|
||||
pub fn estimate(
|
||||
&mut self,
|
||||
frame: &Frame,
|
||||
gimbal_state: &GimbalState,
|
||||
uav_telemetry: &UavTelemetry,
|
||||
) -> Result<EgoMotion, EgoMotionError> {
|
||||
let zoom_band = zoom_band_from_level(gimbal_state.zoom);
|
||||
|
||||
// 1. Skew gate.
|
||||
telemetry_sync::check_skew(
|
||||
frame.capture_ts_monotonic_ns,
|
||||
gimbal_state.ts_monotonic_ns,
|
||||
uav_telemetry.monotonic_ts_ns,
|
||||
zoom_band,
|
||||
)
|
||||
.map_err(|e| {
|
||||
self.counters.inc_skew_drop(zoom_band);
|
||||
EgoMotionError::SkewExceeded(e)
|
||||
})?;
|
||||
|
||||
// 2. Grayscale conversion.
|
||||
let curr_gray = optical_flow::frame_to_gray(frame)
|
||||
.map_err(|e| EgoMotionError::Internal(e.message))?;
|
||||
|
||||
// 3. Degenerate check — runs before the prev-frame guard so a
|
||||
// saturated frame still stores itself and returns a clear error.
|
||||
if optical_flow::is_degenerate(&curr_gray)
|
||||
.unwrap_or(false)
|
||||
{
|
||||
self.counters.inc_degenerate();
|
||||
self.prev_gray = Some(curr_gray);
|
||||
return Err(EgoMotionError::OpticalFlowDegenerate);
|
||||
}
|
||||
|
||||
// 4. Need a previous frame for optical flow.
|
||||
let prev_gray = match self.prev_gray.take() {
|
||||
None => {
|
||||
self.prev_gray = Some(curr_gray);
|
||||
return Err(EgoMotionError::NoPreviousFrame);
|
||||
}
|
||||
Some(p) => p,
|
||||
};
|
||||
|
||||
// 5. Optical flow → homography.
|
||||
let result = optical_flow::estimate_homography(&prev_gray, &curr_gray);
|
||||
self.prev_gray = Some(curr_gray);
|
||||
|
||||
match result {
|
||||
Ok(hr) => Ok(EgoMotion {
|
||||
homography: hr.h,
|
||||
residual_motion_magnitude: hr.residual_magnitude_px,
|
||||
zoom_band,
|
||||
}),
|
||||
Err(FlowError::Degenerate | FlowError::InsufficientFeatures) => {
|
||||
self.counters.inc_degenerate();
|
||||
Err(EgoMotionError::OpticalFlowDegenerate)
|
||||
}
|
||||
Err(FlowError::Internal(msg)) => Err(EgoMotionError::Internal(msg)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use bytes::Bytes;
|
||||
use opencv::{
|
||||
core::{Mat, Scalar, CV_8UC1},
|
||||
prelude::*,
|
||||
};
|
||||
|
||||
use shared::models::{
|
||||
frame::{Frame, PixelFormat},
|
||||
gimbal::GimbalState,
|
||||
movement::ZoomBand,
|
||||
telemetry::UavTelemetry,
|
||||
};
|
||||
|
||||
use super::*;
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
/// Build a 1-channel Mat filled by `fill(row, col)`.
|
||||
fn make_gray_mat(
|
||||
size: i32,
|
||||
fill: impl Fn(i32, i32) -> u8,
|
||||
) -> opencv::Result<Mat> {
|
||||
let mut mat =
|
||||
Mat::new_rows_cols_with_default(size, size, CV_8UC1, Scalar::all(0.0))?;
|
||||
for r in 0..size {
|
||||
for c in 0..size {
|
||||
*mat.at_2d_mut::<u8>(r, c)? = fill(r, c);
|
||||
}
|
||||
}
|
||||
Ok(mat)
|
||||
}
|
||||
|
||||
/// Checkerboard with 8-pixel blocks, optionally shifted right by `offset_x`.
|
||||
fn checkerboard(size: i32, offset_x: i32) -> opencv::Result<Mat> {
|
||||
make_gray_mat(size, |r, c| {
|
||||
let sc = c - offset_x;
|
||||
if sc < 0 || sc >= size {
|
||||
128
|
||||
} else if (sc / 8 + r / 8) % 2 == 0 {
|
||||
200
|
||||
} else {
|
||||
50
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Wrap a 1-channel Mat as a Nv12 `Frame` (Y-plane only — sufficient for
|
||||
/// `frame_to_gray` which reads only the first w×h bytes).
|
||||
fn mat_to_frame(mat: &Mat, ts_ns: u64) -> opencv::Result<Frame> {
|
||||
let h = mat.rows() as u32;
|
||||
let w = mat.cols() as u32;
|
||||
let total = (w * h) as usize;
|
||||
let mut pixels = vec![0u8; total];
|
||||
for r in 0..h as i32 {
|
||||
for c in 0..w as i32 {
|
||||
pixels[(r * w as i32 + c) as usize] = *mat.at_2d::<u8>(r, c)?;
|
||||
}
|
||||
}
|
||||
Ok(Frame {
|
||||
seq: 0,
|
||||
capture_ts_monotonic_ns: ts_ns,
|
||||
decode_ts_monotonic_ns: ts_ns,
|
||||
pixels: Arc::new(Bytes::from(pixels)),
|
||||
width: w,
|
||||
height: h,
|
||||
pix_fmt: PixelFormat::Nv12,
|
||||
ai_locked: false,
|
||||
})
|
||||
}
|
||||
|
||||
fn synced_gimbal(ts_ns: u64) -> GimbalState {
|
||||
GimbalState {
|
||||
yaw: 0.0,
|
||||
pitch: -30.0,
|
||||
zoom: 1.0,
|
||||
ts_monotonic_ns: ts_ns,
|
||||
command_in_flight: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn synced_uav(ts_ns: u64) -> UavTelemetry {
|
||||
UavTelemetry { monotonic_ts_ns: ts_ns, ..UavTelemetry::empty() }
|
||||
}
|
||||
|
||||
// ── AC-1: synthetic pure pan — residual ≈ 0 ──────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn ac1_pure_pan_residual_near_zero() -> opencv::Result<()> {
|
||||
let counters = Arc::new(EgoMotionCounters::new());
|
||||
let mut est = EgoMotionEstimator::new(Arc::clone(&counters));
|
||||
|
||||
let size = 200;
|
||||
let dx = 8i32; // one checkerboard block = well-defined shift
|
||||
|
||||
let mat1 = checkerboard(size, 0)?;
|
||||
let mat2 = checkerboard(size, dx)?;
|
||||
|
||||
let t0 = 1_000_000_000u64;
|
||||
let frame1 = mat_to_frame(&mat1, t0)?;
|
||||
let frame2 = mat_to_frame(&mat2, t0 + 33_000_000)?; // +33 ms (30 fps)
|
||||
|
||||
let gimbal = synced_gimbal(t0);
|
||||
let uav = synced_uav(t0);
|
||||
|
||||
// First call stores prev; NoPreviousFrame is expected.
|
||||
assert!(matches!(est.estimate(&frame1, &gimbal, &uav), Err(EgoMotionError::NoPreviousFrame)));
|
||||
|
||||
let gimbal2 = synced_gimbal(t0 + 33_000_000);
|
||||
let uav2 = synced_uav(t0 + 33_000_000);
|
||||
let ego = est.estimate(&frame2, &gimbal2, &uav2)
|
||||
.expect("estimate should succeed on second call");
|
||||
|
||||
// X-translation H[0][2] should approximate dx within ±2 px.
|
||||
let h02 = ego.homography[2];
|
||||
assert!(
|
||||
h02.abs() > 0.5 && (h02 - dx as f64).abs() < 2.5,
|
||||
"H[0][2] = {h02:.2}, expected ≈ {dx}"
|
||||
);
|
||||
// Residual should be near zero for a pure rigid shift.
|
||||
assert!(
|
||||
ego.residual_motion_magnitude < 3.0,
|
||||
"residual = {:.2} px, expected < 3.0",
|
||||
ego.residual_motion_magnitude
|
||||
);
|
||||
assert_eq!(ego.zoom_band, ZoomBand::ZoomedOut);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ── AC-2: telemetry skew above zoom-out tolerance → SkewExceeded ─────────
|
||||
|
||||
#[test]
|
||||
fn ac2_skew_above_zoom_out_tolerance_dropped() -> opencv::Result<()> {
|
||||
let counters = Arc::new(EgoMotionCounters::new());
|
||||
let mut est = EgoMotionEstimator::new(Arc::clone(&counters));
|
||||
|
||||
let frame_ts = 1_000_000_000u64;
|
||||
let frame = mat_to_frame(&checkerboard(100, 0)?, frame_ts)?;
|
||||
|
||||
// Gimbal timestamp 200 ms ahead of frame; tolerance = 50 ms.
|
||||
let gimbal = GimbalState {
|
||||
zoom: 1.0, // zoomed_out → 50 ms tolerance
|
||||
ts_monotonic_ns: frame_ts + 200_000_000,
|
||||
yaw: 0.0,
|
||||
pitch: -30.0,
|
||||
command_in_flight: false,
|
||||
};
|
||||
let uav = synced_uav(frame_ts);
|
||||
|
||||
assert!(matches!(
|
||||
est.estimate(&frame, &gimbal, &uav),
|
||||
Err(EgoMotionError::SkewExceeded(_))
|
||||
));
|
||||
assert_eq!(counters.skew_drops(ZoomBand::ZoomedOut), 1);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ── AC-3: fully-saturated white frame → OpticalFlowDegenerate ────────────
|
||||
|
||||
#[test]
|
||||
fn ac3_degenerate_white_frame() -> opencv::Result<()> {
|
||||
let counters = Arc::new(EgoMotionCounters::new());
|
||||
let mut est = EgoMotionEstimator::new(Arc::clone(&counters));
|
||||
|
||||
let ts = 1_000_000_000u64;
|
||||
let white_mat =
|
||||
Mat::new_rows_cols_with_default(100, 100, CV_8UC1, Scalar::all(255.0))?;
|
||||
let frame = mat_to_frame(&white_mat, ts)?;
|
||||
let gimbal = synced_gimbal(ts);
|
||||
let uav = synced_uav(ts);
|
||||
|
||||
assert!(matches!(
|
||||
est.estimate(&frame, &gimbal, &uav),
|
||||
Err(EgoMotionError::OpticalFlowDegenerate)
|
||||
));
|
||||
assert_eq!(counters.degenerate_total(), 1);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
pub mod ego_motion;
|
||||
pub mod optical_flow;
|
||||
pub mod telemetry_sync;
|
||||
pub mod zoom_bands;
|
||||
@@ -0,0 +1,194 @@
|
||||
//! Classical OpenCV optical-flow / homography estimation path.
|
||||
//! Lucas–Kanade sparse tracking → RANSAC homography.
|
||||
|
||||
use opencv::{
|
||||
calib3d,
|
||||
core::{self, Mat, Point2f, TermCriteria, Vector},
|
||||
imgproc,
|
||||
prelude::*,
|
||||
video,
|
||||
};
|
||||
|
||||
use shared::models::frame::{Frame, PixelFormat};
|
||||
|
||||
pub struct HomographyResult {
|
||||
/// Row-major 3×3 homography mapping prev frame coords → curr frame coords.
|
||||
pub h: [f64; 9],
|
||||
/// Mean reprojection residual (pixels) across tracked inliers.
|
||||
pub residual_magnitude_px: f32,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum FlowError {
|
||||
Degenerate,
|
||||
InsufficientFeatures,
|
||||
Internal(String),
|
||||
}
|
||||
|
||||
impl From<opencv::Error> for FlowError {
|
||||
fn from(e: opencv::Error) -> Self {
|
||||
FlowError::Internal(e.message)
|
||||
}
|
||||
}
|
||||
|
||||
/// True when the grayscale frame lacks sufficient contrast for feature
|
||||
/// detection (saturated, blank, or nearly uniform).
|
||||
pub fn is_degenerate(gray: &Mat) -> opencv::Result<bool> {
|
||||
let mut min_val = 0.0f64;
|
||||
let mut max_val = 0.0f64;
|
||||
core::min_max_loc(
|
||||
gray,
|
||||
&mut min_val,
|
||||
&mut max_val,
|
||||
&mut core::Point::default(),
|
||||
&mut core::Point::default(),
|
||||
&core::no_array(),
|
||||
)?;
|
||||
Ok((max_val - min_val) < 10.0)
|
||||
}
|
||||
|
||||
/// Convert an autopilot `Frame` to a single-channel (grayscale) OpenCV Mat.
|
||||
/// NV12 / YUV420p: the Y-plane (first w×h bytes) is the grayscale image.
|
||||
/// RGB24: a single cvtColor call produces the grayscale output.
|
||||
pub fn frame_to_gray(frame: &Frame) -> opencv::Result<Mat> {
|
||||
let h = frame.height as i32;
|
||||
let w = frame.width as i32;
|
||||
let data: &[u8] = &frame.pixels;
|
||||
|
||||
match frame.pix_fmt {
|
||||
PixelFormat::Nv12 | PixelFormat::Yuv420p => {
|
||||
let y_len = (w * h) as usize;
|
||||
copy_bytes_to_gray_mat(&data[..y_len], w, h)
|
||||
}
|
||||
PixelFormat::Rgb24 => {
|
||||
let rgb_len = (w * h * 3) as usize;
|
||||
let mut rgb_mat = Mat::new_rows_cols_with_default(
|
||||
h, w, core::CV_8UC3, core::Scalar::all(0.0),
|
||||
)?;
|
||||
// SAFETY: rgb_mat is a freshly allocated continuous Mat; no aliasing.
|
||||
let mat_data = unsafe {
|
||||
std::slice::from_raw_parts_mut(rgb_mat.data_mut()?, rgb_len)
|
||||
};
|
||||
mat_data.copy_from_slice(&data[..rgb_len]);
|
||||
let mut gray = Mat::default();
|
||||
imgproc::cvt_color(&rgb_mat, &mut gray, imgproc::COLOR_RGB2GRAY, 0)?;
|
||||
Ok(gray)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn copy_bytes_to_gray_mat(src: &[u8], w: i32, h: i32) -> opencv::Result<Mat> {
|
||||
let mut mat =
|
||||
Mat::new_rows_cols_with_default(h, w, core::CV_8UC1, core::Scalar::all(0.0))?;
|
||||
// SAFETY: mat is a freshly allocated continuous Mat; no aliasing.
|
||||
let mat_data = unsafe {
|
||||
std::slice::from_raw_parts_mut(mat.data_mut()?, src.len())
|
||||
};
|
||||
mat_data.copy_from_slice(src);
|
||||
Ok(mat)
|
||||
}
|
||||
|
||||
/// Estimate the homography prev_gray → curr_gray via sparse LK optical flow
|
||||
/// and RANSAC. Returns the 3×3 homography (row-major) and the mean inlier
|
||||
/// reprojection residual.
|
||||
pub fn estimate_homography(
|
||||
prev_gray: &Mat,
|
||||
curr_gray: &Mat,
|
||||
) -> Result<HomographyResult, FlowError> {
|
||||
// 1. Detect good corners in the previous frame.
|
||||
let mut prev_pts: Vector<Point2f> = Vector::new();
|
||||
imgproc::good_features_to_track(
|
||||
prev_gray,
|
||||
&mut prev_pts,
|
||||
100,
|
||||
0.01,
|
||||
10.0,
|
||||
&core::no_array(),
|
||||
3,
|
||||
false,
|
||||
0.04,
|
||||
)?;
|
||||
|
||||
if (prev_pts.len() as i32) < 4 {
|
||||
return Err(FlowError::InsufficientFeatures);
|
||||
}
|
||||
|
||||
// 2. Lucas–Kanade pyramidal sparse optical flow.
|
||||
let mut curr_pts: Vector<Point2f> = Vector::new();
|
||||
let mut status: Vector<u8> = Vector::new();
|
||||
let mut err_vec: Vector<f32> = Vector::new();
|
||||
// TermCriteria type 3 = COUNT(1) | EPS(2)
|
||||
let term = TermCriteria::new(3, 30, 0.01)?;
|
||||
video::calc_optical_flow_pyr_lk(
|
||||
prev_gray,
|
||||
curr_gray,
|
||||
&prev_pts,
|
||||
&mut curr_pts,
|
||||
&mut status,
|
||||
&mut err_vec,
|
||||
core::Size::new(21, 21),
|
||||
3,
|
||||
term,
|
||||
0,
|
||||
1e-4,
|
||||
)?;
|
||||
|
||||
// 3. Keep only successfully tracked point pairs.
|
||||
let mut good_prev: Vector<Point2f> = Vector::new();
|
||||
let mut good_curr: Vector<Point2f> = Vector::new();
|
||||
for i in 0..status.len() {
|
||||
if status.get(i)? == 1 {
|
||||
good_prev.push(prev_pts.get(i)?);
|
||||
good_curr.push(curr_pts.get(i)?);
|
||||
}
|
||||
}
|
||||
|
||||
if (good_prev.len() as i32) < 4 {
|
||||
return Err(FlowError::InsufficientFeatures);
|
||||
}
|
||||
|
||||
// 4. Estimate homography with RANSAC (reproj threshold = 3 px).
|
||||
let mut mask = Mat::default();
|
||||
let h_mat = calib3d::find_homography(
|
||||
&good_prev,
|
||||
&good_curr,
|
||||
&mut mask,
|
||||
calib3d::RANSAC,
|
||||
3.0,
|
||||
)?;
|
||||
|
||||
if h_mat.empty() {
|
||||
return Err(FlowError::InsufficientFeatures);
|
||||
}
|
||||
|
||||
// 5. Extract homography values (row-major).
|
||||
let mut h = [0f64; 9];
|
||||
for r in 0..3usize {
|
||||
for c in 0..3usize {
|
||||
h[r * 3 + c] = *h_mat.at_2d::<f64>(r as i32, c as i32)?;
|
||||
}
|
||||
}
|
||||
|
||||
// 6. Mean reprojection residual across all tracked inliers.
|
||||
let mut total = 0.0f32;
|
||||
let mut count = 0u32;
|
||||
for i in 0..good_prev.len() {
|
||||
let p = good_prev.get(i)?;
|
||||
let c = good_curr.get(i)?;
|
||||
let x = p.x as f64;
|
||||
let y = p.y as f64;
|
||||
let denom = h[6] * x + h[7] * y + h[8];
|
||||
if denom.abs() < 1e-9 {
|
||||
continue;
|
||||
}
|
||||
let px = (h[0] * x + h[1] * y + h[2]) / denom;
|
||||
let py = (h[3] * x + h[4] * y + h[5]) / denom;
|
||||
let dx = px as f32 - c.x;
|
||||
let dy = py as f32 - c.y;
|
||||
total += (dx * dx + dy * dy).sqrt();
|
||||
count += 1;
|
||||
}
|
||||
let residual_magnitude_px = if count > 0 { total / count as f32 } else { 0.0 };
|
||||
|
||||
Ok(HomographyResult { h, residual_magnitude_px })
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
//! Frame ↔ gimbal ↔ UAV telemetry skew gate.
|
||||
//! Rejects frames whose telemetry timestamp delta exceeds the per-zoom-band
|
||||
//! tolerance — see `description.md §5` and `description.md §7`.
|
||||
|
||||
use shared::models::movement::ZoomBand;
|
||||
|
||||
use super::zoom_bands::ZoomBandTolerances;
|
||||
|
||||
/// Returned when either skew delta exceeds the per-band tolerance.
|
||||
#[derive(Debug)]
|
||||
pub struct SkewExceeded {
|
||||
pub band: ZoomBand,
|
||||
pub gimbal_skew_ns: u64,
|
||||
pub uav_skew_ns: u64,
|
||||
}
|
||||
|
||||
/// Check frame ↔ gimbal and frame ↔ UAV skew against per-band tolerances.
|
||||
/// Returns `Err(SkewExceeded)` if either exceeds its threshold.
|
||||
pub fn check_skew(
|
||||
frame_ts_ns: u64,
|
||||
gimbal_ts_ns: u64,
|
||||
uav_ts_ns: u64,
|
||||
band: ZoomBand,
|
||||
) -> Result<(), SkewExceeded> {
|
||||
let tolerances = ZoomBandTolerances::for_band(band);
|
||||
let gimbal_skew = frame_ts_ns.abs_diff(gimbal_ts_ns);
|
||||
let uav_skew = frame_ts_ns.abs_diff(uav_ts_ns);
|
||||
|
||||
if gimbal_skew > tolerances.frame_gimbal_ns || uav_skew > tolerances.frame_uav_ns {
|
||||
return Err(SkewExceeded { band, gimbal_skew_ns: gimbal_skew, uav_skew_ns: uav_skew });
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn in_tolerance_passes() {
|
||||
check_skew(1_000_000_000, 1_010_000_000, 1_020_000_000, ZoomBand::ZoomedOut).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gimbal_skew_exceeds_zoom_out_tolerance() {
|
||||
let err = check_skew(
|
||||
1_000_000_000,
|
||||
1_200_000_000, // 200 ms > 50 ms threshold
|
||||
1_010_000_000,
|
||||
ZoomBand::ZoomedOut,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert_eq!(err.gimbal_skew_ns, 200_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uav_skew_exceeds_zoom_in_tolerance() {
|
||||
let err = check_skew(
|
||||
1_000_000_000,
|
||||
1_010_000_000,
|
||||
1_060_000_000, // 60 ms > 50 ms zoom-in UAV threshold
|
||||
ZoomBand::ZoomedIn,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert_eq!(err.uav_skew_ns, 60_000_000);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
//! Per-zoom-band threshold tables — see `description.md §5`.
|
||||
|
||||
use shared::models::movement::ZoomBand;
|
||||
|
||||
/// Telemetry-skew tolerances for a given zoom band.
|
||||
/// Nanosecond values per `description.md §5`.
|
||||
pub struct ZoomBandTolerances {
|
||||
pub frame_gimbal_ns: u64,
|
||||
pub frame_uav_ns: u64,
|
||||
}
|
||||
|
||||
impl ZoomBandTolerances {
|
||||
pub fn for_band(band: ZoomBand) -> Self {
|
||||
match band {
|
||||
ZoomBand::ZoomedOut => Self {
|
||||
frame_gimbal_ns: 50_000_000,
|
||||
frame_uav_ns: 100_000_000,
|
||||
},
|
||||
ZoomBand::ZoomedIn => Self {
|
||||
frame_gimbal_ns: 25_000_000,
|
||||
frame_uav_ns: 50_000_000,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Derive zoom band from the gimbal's current zoom level.
|
||||
/// Zoom ≤ 2.0 → wide-area sweep; zoom > 2.0 → detailed-scan hold.
|
||||
pub fn zoom_band_from_level(zoom: f32) -> ZoomBand {
|
||||
if zoom > 2.0 {
|
||||
ZoomBand::ZoomedIn
|
||||
} else {
|
||||
ZoomBand::ZoomedOut
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn zoom_out_tolerances() {
|
||||
let t = ZoomBandTolerances::for_band(ZoomBand::ZoomedOut);
|
||||
assert_eq!(t.frame_gimbal_ns, 50_000_000);
|
||||
assert_eq!(t.frame_uav_ns, 100_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zoom_in_tolerances_are_stricter() {
|
||||
let out = ZoomBandTolerances::for_band(ZoomBand::ZoomedOut);
|
||||
let inn = ZoomBandTolerances::for_band(ZoomBand::ZoomedIn);
|
||||
assert!(inn.frame_gimbal_ns < out.frame_gimbal_ns);
|
||||
assert!(inn.frame_uav_ns < out.frame_uav_ns);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn band_from_zoom_level() {
|
||||
assert_eq!(zoom_band_from_level(1.0), ZoomBand::ZoomedOut);
|
||||
assert_eq!(zoom_band_from_level(2.0), ZoomBand::ZoomedOut);
|
||||
assert_eq!(zoom_band_from_level(2.1), ZoomBand::ZoomedIn);
|
||||
assert_eq!(zoom_band_from_level(5.0), ZoomBand::ZoomedIn);
|
||||
}
|
||||
}
|
||||
@@ -1,30 +1,37 @@
|
||||
//! `movement_detector` — ego-motion compensated residual-motion clustering.
|
||||
//!
|
||||
//! Real implementation lands in:
|
||||
//! - AZ-662 `movement_detector_ego_motion`
|
||||
//! - AZ-663 `movement_detector_clustering_and_emission`
|
||||
//! - AZ-664 `movement_detector_fp_cap_and_q14_fallback`
|
||||
//! AZ-662: ego-motion estimator + telemetry-skew gate (this batch).
|
||||
//! AZ-663: residual clustering + candidate emission (next batch).
|
||||
//! AZ-664: FP cap + Q14 learned-CV fallback.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use tokio::sync::broadcast;
|
||||
|
||||
use shared::health::ComponentHealth;
|
||||
use shared::health::{ComponentHealth, HealthLevel};
|
||||
use shared::models::movement::MovementCandidate;
|
||||
|
||||
pub(crate) mod internal;
|
||||
|
||||
use internal::ego_motion::EgoMotionCounters;
|
||||
|
||||
const NAME: &str = "movement_detector";
|
||||
|
||||
pub struct MovementDetector {
|
||||
tx: broadcast::Sender<MovementCandidate>,
|
||||
counters: Arc<EgoMotionCounters>,
|
||||
}
|
||||
|
||||
impl MovementDetector {
|
||||
pub fn new(channel_capacity: usize) -> Self {
|
||||
let (tx, _rx) = broadcast::channel(channel_capacity);
|
||||
Self { tx }
|
||||
Self { tx, counters: Arc::new(EgoMotionCounters::new()) }
|
||||
}
|
||||
|
||||
pub fn handle(&self) -> MovementDetectorHandle {
|
||||
MovementDetectorHandle {
|
||||
tx: self.tx.clone(),
|
||||
counters: Arc::clone(&self.counters),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -32,6 +39,7 @@ impl MovementDetector {
|
||||
#[derive(Clone)]
|
||||
pub struct MovementDetectorHandle {
|
||||
tx: broadcast::Sender<MovementCandidate>,
|
||||
counters: Arc<EgoMotionCounters>,
|
||||
}
|
||||
|
||||
impl MovementDetectorHandle {
|
||||
@@ -40,7 +48,23 @@ impl MovementDetectorHandle {
|
||||
}
|
||||
|
||||
pub fn health(&self) -> ComponentHealth {
|
||||
ComponentHealth::disabled(NAME)
|
||||
let skew_drops = self.counters.skew_drops_total();
|
||||
let degenerate = self.counters.degenerate_total();
|
||||
|
||||
if skew_drops > 0 || degenerate > 0 {
|
||||
ComponentHealth::yellow(
|
||||
NAME,
|
||||
format!(
|
||||
"skew_drops_total={skew_drops} optical_flow_degenerate_total={degenerate}"
|
||||
),
|
||||
)
|
||||
} else {
|
||||
ComponentHealth {
|
||||
level: HealthLevel::Disabled,
|
||||
component: NAME,
|
||||
detail: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -51,6 +75,9 @@ mod tests {
|
||||
#[test]
|
||||
fn it_compiles() {
|
||||
let h = MovementDetector::new(16).handle();
|
||||
assert_eq!(h.health().level, shared::health::HealthLevel::Disabled);
|
||||
assert!(matches!(
|
||||
h.health().level,
|
||||
HealthLevel::Disabled | HealthLevel::Yellow
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user