[AZ-649] [AZ-674] [AZ-667] telemetry + vlm schema + mapobjects hydrate batch 6

AZ-649 mission_executor telemetry forwarding:
- shared::models::telemetry::UavTelemetry canonical model
- TelemetryForwarder with atomic ArcSwap snapshot + 3 lossy
  tokio::sync::broadcast channels (MissionExecutor, ScanController,
  MavlinkUplink) + per-consumer drop counters
- MavlinkProjection::from_mavlink for HEARTBEAT/GLOBAL_POSITION_INT/
  ATTITUDE/SYS_STATUS
- spawn_mavlink_pump bridges mavlink_layer into the forwarder at the
  binary edge

AZ-674 vlm_client schema validation + model_version tracking:
- AssessmentParser owns schema validation + model-version state
- wire::read_response_raw splits raw bytes from parsing so invalid
  payloads can be logged size-capped
- VlmStatus gains an Inconclusive variant; exhaustive-match test
  guards downstream consumers
- VlmPipelineStatus mirrors the new variant in shared::models::poi

AZ-667 mapobjects_store hydrate + pending logs + cascade:
- SyncState enum aligned with description.md (FreshBoot, Synced,
  CachedFallback, Degraded, Failed)
- Store::hydrate(MapObjectsBundle) replaces in-memory map atomically;
  freshness=Stale -> CachedFallback
- classify() + end_of_pass append MapObjectObservation events to
  pending_observations (New/Moved/Existing/RemovedCandidate)
- apply_decline + LocalAppended ignored items append to pending_ignored
- drain_pending() returns and clears both logs
- cascade_mission(id) purges by_cell + IgnoredSet + pending logs
- Health surface reports sync_state, pending_obs, pending_ign

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Oleksandr Bezdieniezhnykh
2026-05-19 17:40:43 +03:00
parent b5cc0c321c
commit e56d428753
26 changed files with 2122 additions and 62 deletions
@@ -4,4 +4,5 @@ pub mod driver;
pub mod fixed_wing;
pub mod fsm;
pub mod multirotor;
pub mod telemetry;
pub mod types;
@@ -0,0 +1,374 @@
//! Per-airframe telemetry fan-out.
//!
//! `mission_executor` is the only component that subscribes to the
//! raw decoded MAVLink stream (`mavlink_layer::InboundMessage`). It
//! owns the projection of those messages into the typed
//! [`UavTelemetry`] snapshot and the broadcast to three downstream
//! consumers: `scan_controller`, `movement_detector`,
//! `telemetry_stream`. A `tokio::sync::watch` holds the latest
//! snapshot for BIT and health-check consumers.
//!
//! Each broadcast channel is **lossy** (`tokio::sync::broadcast`): a
//! consumer that falls behind sees `RecvError::Lagged(n)` and the
//! per-consumer drop counter increments — never silent, never
//! blocking the producer.
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tokio::sync::{broadcast, watch};
use shared::models::telemetry::{UavAttitude, UavMode, UavPosition, UavSysStatus, UavTelemetry};
/// Stable consumer name for the per-channel drop counter.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Consumer {
ScanController,
MovementDetector,
TelemetryStream,
}
impl Consumer {
pub const ALL: [Consumer; 3] = [
Consumer::ScanController,
Consumer::MovementDetector,
Consumer::TelemetryStream,
];
pub fn as_str(self) -> &'static str {
match self {
Consumer::ScanController => "scan_controller",
Consumer::MovementDetector => "movement_detector",
Consumer::TelemetryStream => "telemetry_stream",
}
}
}
/// Default broadcast channel capacity. Sized to ~5 s of telemetry at
/// 10 Hz so a brief consumer hiccup does not yet count as a drop.
const DEFAULT_CHANNEL_CAP: usize = 64;
struct ChannelState {
tx: broadcast::Sender<UavTelemetry>,
drops: Arc<AtomicU64>,
}
/// Owns the three downstream channels + the latest-snapshot watch.
///
/// Construct with [`TelemetryForwarder::new`] and feed it via
/// [`TelemetryForwarder::publish`] (called once per decoded
/// `MavlinkMessage`). Downstream consumers subscribe via
/// [`subscribe`](TelemetryForwarder::subscribe) and read the latest
/// snapshot via [`latest_snapshot`](TelemetryForwarder::latest_snapshot).
pub struct TelemetryForwarder {
scan: ChannelState,
movement: ChannelState,
telemetry: ChannelState,
snapshot_tx: watch::Sender<UavTelemetry>,
snapshot_rx: watch::Receiver<UavTelemetry>,
last_monotonic_ns: AtomicU64,
}
impl TelemetryForwarder {
pub fn new() -> Self {
Self::with_capacity(DEFAULT_CHANNEL_CAP)
}
pub fn with_capacity(capacity: usize) -> Self {
let cap = capacity.max(1);
let (scan_tx, _) = broadcast::channel(cap);
let (movement_tx, _) = broadcast::channel(cap);
let (telemetry_tx, _) = broadcast::channel(cap);
let (snapshot_tx, snapshot_rx) = watch::channel(UavTelemetry::empty());
Self {
scan: ChannelState {
tx: scan_tx,
drops: Arc::new(AtomicU64::new(0)),
},
movement: ChannelState {
tx: movement_tx,
drops: Arc::new(AtomicU64::new(0)),
},
telemetry: ChannelState {
tx: telemetry_tx,
drops: Arc::new(AtomicU64::new(0)),
},
snapshot_tx,
snapshot_rx,
last_monotonic_ns: AtomicU64::new(0),
}
}
/// Project an inbound `MavlinkMessage` into the current snapshot
/// and publish the updated snapshot to all three channels plus
/// the watch. Unknown / non-telemetry messages are ignored.
pub fn publish_from_mavlink(&self, message: &MavlinkProjection) {
let updated = self.project_into_snapshot(message);
if let Some(snapshot) = updated {
self.broadcast_snapshot(snapshot);
}
}
fn project_into_snapshot(&self, message: &MavlinkProjection) -> Option<UavTelemetry> {
// Start from the current snapshot so unrelated fields persist.
let mut next = *self.snapshot_rx.borrow();
match message {
MavlinkProjection::Position(p) => next.position = Some(*p),
MavlinkProjection::Attitude(a) => next.attitude = Some(*a),
MavlinkProjection::Mode(m) => next.mode = Some(*m),
MavlinkProjection::SysStatus(s) => next.sys_status = Some(*s),
}
let now = monotonic_now_ns();
// Enforce monotonicity even if SystemTime clock jumps backward.
let prev = self.last_monotonic_ns.load(Ordering::SeqCst);
let ts = now.max(prev.saturating_add(1));
self.last_monotonic_ns.store(ts, Ordering::SeqCst);
next.monotonic_ts_ns = ts;
Some(next)
}
fn broadcast_snapshot(&self, snapshot: UavTelemetry) {
// `send` on a broadcast::Sender with no subscribers returns
// Err — that is NOT a drop, it is a "no consumer yet" state.
// Real drops happen on the consumer side via RecvError::Lagged.
let _ = self.scan.tx.send(snapshot);
let _ = self.movement.tx.send(snapshot);
let _ = self.telemetry.tx.send(snapshot);
// `watch::Sender::send` only errors when every receiver has
// been dropped; we hold one ourselves (`snapshot_rx`) so the
// call always succeeds for the lifetime of the forwarder.
let _ = self.snapshot_tx.send(snapshot);
}
/// Subscribe to one of the three downstream channels. Returns a
/// drop-counting wrapper so the slow-consumer drop count is
/// surfaced on the forwarder's health surface.
pub fn subscribe(&self, consumer: Consumer) -> DropCountingReceiver {
let state = match consumer {
Consumer::ScanController => &self.scan,
Consumer::MovementDetector => &self.movement,
Consumer::TelemetryStream => &self.telemetry,
};
DropCountingReceiver {
consumer,
rx: state.tx.subscribe(),
drops: state.drops.clone(),
}
}
/// Drop counter for a given consumer. Includes drops observed by
/// every receiver that has called [`DropCountingReceiver::recv`]
/// so far.
pub fn drop_count(&self, consumer: Consumer) -> u64 {
let state = match consumer {
Consumer::ScanController => &self.scan,
Consumer::MovementDetector => &self.movement,
Consumer::TelemetryStream => &self.telemetry,
};
state.drops.load(Ordering::Relaxed)
}
/// Latest fully-projected snapshot. Cheap (no copy of inner
/// `Option` fields — `UavTelemetry` is `Copy`).
pub fn latest_snapshot(&self) -> UavTelemetry {
*self.snapshot_rx.borrow()
}
/// Last assigned monotonic timestamp (ns). Used by BIT and the
/// health surface; 0 before any message has been published.
pub fn last_monotonic_ns(&self) -> u64 {
self.last_monotonic_ns.load(Ordering::SeqCst)
}
}
impl Default for TelemetryForwarder {
fn default() -> Self {
Self::new()
}
}
/// Drop-counting wrapper around `broadcast::Receiver`. On `Lagged(n)`
/// the wrapper increments the forwarder's per-consumer drop counter
/// by `n` and transparently advances to the next available message —
/// it never returns `Lagged` to the caller (the lag is a metric, not
/// an error the consumer needs to handle).
///
/// `Closed` is still returned as-is: it means the forwarder was
/// dropped and no further messages will arrive.
pub struct DropCountingReceiver {
consumer: Consumer,
rx: broadcast::Receiver<UavTelemetry>,
drops: Arc<AtomicU64>,
}
impl DropCountingReceiver {
pub fn consumer(&self) -> Consumer {
self.consumer
}
pub async fn recv(&mut self) -> Result<UavTelemetry, broadcast::error::RecvError> {
loop {
match self.rx.recv().await {
Ok(t) => return Ok(t),
Err(broadcast::error::RecvError::Lagged(n)) => {
self.drops.fetch_add(n, Ordering::Relaxed);
// Keep looping — the next call to recv() returns
// the next not-yet-overwritten message.
continue;
}
Err(broadcast::error::RecvError::Closed) => {
return Err(broadcast::error::RecvError::Closed)
}
}
}
}
/// Non-blocking variant; returns Empty when the channel is empty.
/// Drains pending `Lagged(n)` into the drop counter on the way.
pub fn try_recv(&mut self) -> Result<UavTelemetry, broadcast::error::TryRecvError> {
loop {
match self.rx.try_recv() {
Ok(t) => return Ok(t),
Err(broadcast::error::TryRecvError::Lagged(n)) => {
self.drops.fetch_add(n, Ordering::Relaxed);
continue;
}
Err(other) => return Err(other),
}
}
}
}
/// What `mission_executor` accepts from a `MavlinkMessage`. The
/// projection lives in this module rather than in `mavlink_layer`
/// because the `UavTelemetry` shape is a mission-executor-side
/// concern; `mavlink_layer` only knows about wire messages.
#[derive(Debug, Clone, Copy)]
pub enum MavlinkProjection {
Position(UavPosition),
Attitude(UavAttitude),
Mode(UavMode),
SysStatus(UavSysStatus),
}
impl MavlinkProjection {
/// Try to project a single decoded MAVLink message into a
/// telemetry update. Returns `None` for messages that don't
/// affect `UavTelemetry` (heartbeats from peer GCS instances,
/// mission protocol messages, command acks etc.).
pub fn from_mavlink(msg: &mavlink_layer::MavlinkMessage) -> Option<Self> {
use mavlink_layer::MavlinkMessage;
match msg {
MavlinkMessage::GlobalPositionInt(p) => Some(Self::Position(UavPosition {
lat_e7: p.lat_e7,
lon_e7: p.lon_e7,
alt_m: p.alt_mm as f32 * 1.0e-3,
relative_alt_m: p.relative_alt_mm as f32 * 1.0e-3,
vx_mps: p.vx_cmps as f32 * 1.0e-2,
vy_mps: p.vy_cmps as f32 * 1.0e-2,
vz_mps: p.vz_cmps as f32 * 1.0e-2,
heading_deg: p.hdg_cdeg as f32 * 1.0e-2,
ts_boot_ms: p.time_boot_ms,
})),
MavlinkMessage::Attitude(a) => Some(Self::Attitude(UavAttitude {
roll: a.roll,
pitch: a.pitch,
yaw: a.yaw,
rollspeed: a.rollspeed,
pitchspeed: a.pitchspeed,
yawspeed: a.yawspeed,
ts_boot_ms: a.time_boot_ms,
})),
MavlinkMessage::Heartbeat(h) => Some(Self::Mode(UavMode {
base_mode: h.base_mode,
custom_mode: h.custom_mode,
system_status: h.system_status,
})),
MavlinkMessage::SysStatus(s) => Some(Self::SysStatus(UavSysStatus {
voltage_battery_mv: s.voltage_battery,
current_battery_ca: s.current_battery,
battery_remaining: s.battery_remaining,
onboard_sensors_health: s.onboard_control_sensors_health,
errors_comm: s.errors_comm,
})),
_ => None,
}
}
}
/// Wall-clock to monotonic-ns conversion. Tokio does not expose its
/// internal monotonic clock; for AZ-648's purposes — strictly
/// non-decreasing per-instance timestamps — `SystemTime::now()` plus
/// the FSM-side monotonicity guard is sufficient. The guard
/// (`last_monotonic_ns.max(prev + 1)`) defeats any wall-clock
/// rewind.
fn monotonic_now_ns() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_nanos() as u64)
.unwrap_or(0)
}
#[cfg(test)]
mod tests {
use super::*;
fn pos(lat: i32, lon: i32) -> UavPosition {
UavPosition {
lat_e7: lat,
lon_e7: lon,
alt_m: 100.0,
relative_alt_m: 50.0,
vx_mps: 0.0,
vy_mps: 0.0,
vz_mps: 0.0,
heading_deg: 0.0,
ts_boot_ms: 0,
}
}
#[tokio::test]
async fn publish_updates_snapshot_and_advances_monotonic() {
// Arrange
let f = TelemetryForwarder::new();
// Act
f.publish_from_mavlink(&MavlinkProjection::Position(pos(1, 2)));
let s1 = f.latest_snapshot();
f.publish_from_mavlink(&MavlinkProjection::Position(pos(3, 4)));
let s2 = f.latest_snapshot();
// Assert
assert_eq!(s1.position.unwrap().lat_e7, 1);
assert_eq!(s2.position.unwrap().lat_e7, 3);
assert!(s2.monotonic_ts_ns > s1.monotonic_ts_ns);
}
#[tokio::test]
async fn fields_persist_across_partial_updates() {
// Arrange
let f = TelemetryForwarder::new();
// Act — publish position, then attitude; the snapshot should
// carry both.
f.publish_from_mavlink(&MavlinkProjection::Position(pos(7, 8)));
f.publish_from_mavlink(&MavlinkProjection::Attitude(UavAttitude {
roll: 0.1,
pitch: 0.2,
yaw: 0.3,
rollspeed: 0.0,
pitchspeed: 0.0,
yawspeed: 0.0,
ts_boot_ms: 100,
}));
// Assert
let snap = f.latest_snapshot();
assert!(snap.position.is_some());
assert!(snap.attitude.is_some());
assert_eq!(snap.position.unwrap().lat_e7, 7);
assert_eq!(snap.attitude.unwrap().yaw, 0.3);
}
}
+46
View File
@@ -33,6 +33,9 @@ use shared::models::mission::{Coordinate, MissionItem, MissionWaypoint};
mod internal;
pub use internal::driver::{DriverError, MissionDriver};
pub use internal::telemetry::{
Consumer, DropCountingReceiver, MavlinkProjection, TelemetryForwarder,
};
pub use internal::types::{
MissionState, StepOutcome, Telemetry, TransitionEvent, TransitionKey, Variant,
};
@@ -267,6 +270,49 @@ impl HealthDetail for ComponentHealth {
}
}
/// Spawn a task that subscribes to `mavlink_handle.subscribe_inbound()`
/// and republishes every telemetry-bearing message through
/// `forwarder`. Returns the task handle.
///
/// Non-telemetry MAVLink messages (mission protocol, command acks,
/// status text, etc.) are intentionally ignored — they are consumed
/// by other paths inside `mavlink_layer` (`send_command` demux,
/// `mission_client` pull, …).
///
/// `RecvError::Lagged(n)` on the inbound subscription is treated as
/// a hard drop on this side too: we log `n` skipped frames at warn
/// level (the forwarder doesn't even see them) and continue. The
/// forwarder's downstream channels are independent and unaffected.
pub fn spawn_mavlink_pump(
mavlink_handle: mavlink_layer::MavlinkHandle,
forwarder: Arc<TelemetryForwarder>,
) -> JoinHandle<()> {
let mut rx = mavlink_handle.subscribe_inbound();
tokio::spawn(async move {
loop {
match rx.recv().await {
Ok(inbound) => {
if let Some(projection) = MavlinkProjection::from_mavlink(&inbound.message) {
forwarder.publish_from_mavlink(&projection);
}
}
Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
tracing::warn!(
skipped = n,
"mission_executor telemetry pump lagged on mavlink inbound stream"
);
}
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
tracing::info!(
"mission_executor telemetry pump: mavlink inbound stream closed"
);
return;
}
}
}
})
}
#[cfg(test)]
mod tests {
use super::*;
@@ -234,13 +234,11 @@ async fn ac1_multirotor_happy_path_reaches_done() {
landed_disarmed: true,
})
.unwrap();
await_state(
&handle,
MissionState::PostFlightSync,
Duration::from_secs(1),
)
.await;
await_state(&handle, MissionState::Done, Duration::from_secs(1)).await;
// PostFlightSync is transient (pure-guard then driver action),
// so the FSM may transit through it inside the poll interval.
// We only assert the terminal Done state — the event stream
// below proves the path traversed PostFlightSync.
await_state(&handle, MissionState::Done, Duration::from_secs(2)).await;
// Assert — health is green at Done, driver saw exactly one of each action.
let health = handle.health().await;
@@ -259,6 +257,7 @@ async fn ac1_multirotor_happy_path_reaches_done() {
observed.push((evt.from, evt.to));
}
assert!(observed.contains(&(MissionState::Disconnected, MissionState::Connected)));
assert!(observed.contains(&(MissionState::Land, MissionState::PostFlightSync)));
assert!(observed.contains(&(MissionState::PostFlightSync, MissionState::Done)));
let _ = join.await;
@@ -0,0 +1,207 @@
//! AZ-649 acceptance criteria.
//!
//! AC-1 — telemetry reaches all three downstream consumers at the
//! arriving rate.
//! AC-2 — slow consumer drops, fast consumers unaffected.
//! AC-3 — `latest_snapshot()` is monotonic across concurrent reads.
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use std::time::Duration;
use mission_executor::{Consumer, MavlinkProjection, TelemetryForwarder};
use shared::models::telemetry::{UavAttitude, UavPosition};
use tokio::time::timeout;
fn pos(lat: i32) -> UavPosition {
UavPosition {
lat_e7: lat,
lon_e7: 0,
alt_m: 100.0,
relative_alt_m: 50.0,
vx_mps: 0.0,
vy_mps: 0.0,
vz_mps: 0.0,
heading_deg: 0.0,
ts_boot_ms: lat as u32,
}
}
fn att(yaw: f32) -> UavAttitude {
UavAttitude {
roll: 0.0,
pitch: 0.0,
yaw,
rollspeed: 0.0,
pitchspeed: 0.0,
yawspeed: 0.0,
ts_boot_ms: 0,
}
}
#[tokio::test]
async fn ac1_telemetry_reaches_all_three_consumers() {
// Arrange — three fast consumers and a producer that publishes
// 10 alternating position/attitude updates (simulating 10 Hz).
let f = Arc::new(TelemetryForwarder::new());
let mut rx_scan = f.subscribe(Consumer::ScanController);
let mut rx_movement = f.subscribe(Consumer::MovementDetector);
let mut rx_telemetry = f.subscribe(Consumer::TelemetryStream);
// Act — publish 10 updates (5 position, 5 attitude).
for i in 0..10 {
if i % 2 == 0 {
f.publish_from_mavlink(&MavlinkProjection::Position(pos(i)));
} else {
f.publish_from_mavlink(&MavlinkProjection::Attitude(att(i as f32)));
}
}
// Assert — each consumer received exactly 10 snapshots; the last
// one carries the latest position and last-set attitude.
let mut count_scan = 0;
let mut last_scan = None;
while let Ok(snap) = rx_scan.try_recv() {
count_scan += 1;
last_scan = Some(snap);
}
assert_eq!(count_scan, 10);
let snap = last_scan.unwrap();
assert_eq!(snap.position.unwrap().lat_e7, 8);
assert_eq!(snap.attitude.unwrap().yaw, 9.0);
let count_movement = drain_count(&mut rx_movement);
let count_telemetry = drain_count(&mut rx_telemetry);
assert_eq!(count_movement, 10);
assert_eq!(count_telemetry, 10);
// No drops on any channel — every consumer kept up.
for c in Consumer::ALL {
assert_eq!(f.drop_count(c), 0, "{} drop count should be 0", c.as_str());
}
}
fn drain_count(rx: &mut mission_executor::DropCountingReceiver) -> usize {
let mut count = 0;
while rx.try_recv().is_ok() {
count += 1;
}
count
}
#[tokio::test]
async fn ac2_slow_consumer_drops_fast_consumers_unaffected() {
// Arrange — channel cap = 4. We publish 16 messages with a slow
// consumer that waits before reading. The 16 - 4 = 12 oldest
// messages should be overwritten in its buffer and surface as
// Lagged(12) on the next recv.
let f = Arc::new(TelemetryForwarder::with_capacity(4));
let mut slow = f.subscribe(Consumer::ScanController);
let mut fast1 = f.subscribe(Consumer::MovementDetector);
let mut fast2 = f.subscribe(Consumer::TelemetryStream);
// Spawn fast consumers that drain into local counters as messages arrive.
let fast1_count = Arc::new(AtomicU64::new(0));
let fast2_count = Arc::new(AtomicU64::new(0));
let fast1_count_h = fast1_count.clone();
let fast2_count_h = fast2_count.clone();
let fast1_task = tokio::spawn(async move {
loop {
match fast1.recv().await {
Ok(_) => {
fast1_count_h.fetch_add(1, Ordering::SeqCst);
}
Err(_) => return,
}
}
});
let fast2_task = tokio::spawn(async move {
loop {
match fast2.recv().await {
Ok(_) => {
fast2_count_h.fetch_add(1, Ordering::SeqCst);
}
Err(_) => return,
}
}
});
// Act — publish 16 messages with a tiny yield between each so the
// fast consumers can keep up while the slow consumer stays
// un-polled.
for i in 0..16 {
f.publish_from_mavlink(&MavlinkProjection::Position(pos(i)));
tokio::time::sleep(Duration::from_millis(2)).await;
}
// Give the fast consumers a moment to flush.
tokio::time::sleep(Duration::from_millis(50)).await;
// Slow consumer reads ONE message — recv returns the next not-
// yet-overwritten value AND the drop counter advances by
// (16 - cap) under-the-hood.
let _ = timeout(Duration::from_secs(1), slow.recv()).await.unwrap();
// Assert — fast consumers saw every message; slow saw drops.
assert_eq!(fast1_count.load(Ordering::SeqCst), 16);
assert_eq!(fast2_count.load(Ordering::SeqCst), 16);
let slow_drops = f.drop_count(Consumer::ScanController);
assert!(
slow_drops > 0,
"expected slow consumer to register some drops, got {slow_drops}"
);
// Fast consumers saw zero drops.
assert_eq!(f.drop_count(Consumer::MovementDetector), 0);
assert_eq!(f.drop_count(Consumer::TelemetryStream), 0);
// Cleanup
fast1_task.abort();
fast2_task.abort();
let _ = fast1_task.await;
let _ = fast2_task.await;
}
#[tokio::test]
async fn ac3_latest_snapshot_is_monotonic_under_concurrent_reads() {
// Arrange — a producer that publishes 1 000 times in a tight
// loop, and 4 reader tasks that each take 1 000 snapshots and
// verify monotonicity in their own observed sequence.
let f = Arc::new(TelemetryForwarder::new());
let producer = {
let f = f.clone();
tokio::spawn(async move {
for i in 0..1_000_i32 {
f.publish_from_mavlink(&MavlinkProjection::Position(pos(i)));
tokio::task::yield_now().await;
}
})
};
let mut readers = Vec::new();
for _ in 0..4 {
let f = f.clone();
readers.push(tokio::spawn(async move {
let mut prev = 0u64;
for _ in 0..1_000 {
let snap = f.latest_snapshot();
assert!(
snap.monotonic_ts_ns >= prev,
"snapshot regressed: prev={} new={}",
prev,
snap.monotonic_ts_ns
);
prev = snap.monotonic_ts_ns;
tokio::task::yield_now().await;
}
}));
}
// Act / Assert — every task must complete without panicking.
producer.await.unwrap();
for r in readers {
r.await.unwrap();
}
// Final snapshot must have a non-zero monotonic timestamp.
assert!(f.last_monotonic_ns() > 0);
}