[AZ-676] [AZ-677] [AZ-678] [AZ-679] telemetry+operator foundation

Batch 15 ships the four foundation tickets sitting on top of AZ-675
(gRPC server) and AZ-667 (mapobjects_store hydrate):

* AZ-676: telemetry_stream video path (rtsp_forward + bytes_inline)
  with ai_locked atomic + session counter, SubscribeVideo RPC.
* AZ-677: MapObjects snapshot-on-subscribe + diff broadcast +
  reconnect-resync (StartThen stream-prepend pattern).
* AZ-678: HmacOperatorValidator with per-session monotonic seq,
  in-process session registry + TTL, constant-time HMAC compare,
  rejection-reason counters, sliding 60 s sig-failure red-health gate.
  Trait OperatorCommandValidator in shared::contracts::operator_auth.
* AZ-679: PoiSurfaceMapper produces OperatorPoiEvent per architecture
  §7.10; PoiDequeued events on rotate/age-out/complete; pushed via
  new TelemetrySink::push_operator_event extension on Topic::OperatorEvent.

Cross-task wiring: TelemetrySink trait extended with
push_operator_event; OperatorBridge gets optional builder methods
with_telemetry_sink / with_validator (composition root wires in
AZ-680). Workspace deps: hmac = "0.12"; per-crate adds bytes,
serde_json, parking_lot, chrono, uuid, sha2, thiserror.

Tests: 14/14 ACs verified locally (4 + 3 + 5 + 3 by AC) plus
6 supporting unit tests + 7 integration tests + 2 shared serde
roundtrips. cargo clippy clean on touched crates. Cumulative
review for batches 13-15 produced; verdict PASS_WITH_WARNINGS
(0 Critical, 0 High, 1 Medium, 4 Low — all carry-overs or
deferred-producer notes for AZ-680/AZ-684).

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Oleksandr Bezdieniezhnykh
2026-05-20 16:18:40 +03:00
parent 0eb09eec2d
commit ccf929af69
29 changed files with 3495 additions and 68 deletions
@@ -0,0 +1,380 @@
//! AZ-677 integration tests — snapshot on subscribe, diff stream while
//! connected, fresh snapshot on reconnect (no diff replay).
use std::net::TcpListener;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::Duration;
use chrono::Utc;
use tokio::time::timeout;
use tokio_stream::StreamExt;
use tonic::transport::{Channel, Endpoint};
use tonic::Request;
use uuid::Uuid;
use shared::models::mapobject::{
BundleFreshness, DiffKind, IgnoredItem, IgnoredItemSource, MapObject, MapObjectObservation,
MapObjectSource, MapObjectsBundle, RetentionScope,
};
use shared::models::mission::Coordinate;
use telemetry_stream::internal::mapobjects::MapObjectsDiff;
use telemetry_stream::{
MapObjectsSnapshotSource, MapObjectsTopicMessage, SubscribeRequest, TelemetryStream,
TelemetryStreamClient, TelemetryTopic,
};
fn bind_ephemeral() -> (TcpListener, u16) {
let l = TcpListener::bind("127.0.0.1:0").expect("bind ephemeral");
let port = l.local_addr().unwrap().port();
(l, port)
}
async fn connect(port: u16) -> TelemetryStreamClient<Channel> {
let url = format!("http://127.0.0.1:{port}");
let endpoint = Endpoint::from_shared(url)
.unwrap()
.connect_timeout(Duration::from_secs(2));
for _ in 0..50 {
if let Ok(c) = TelemetryStreamClient::connect(endpoint.clone()).await {
return c;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
panic!("gRPC client failed to connect");
}
fn coord(lat: f64, lon: f64) -> Coordinate {
Coordinate {
latitude: lat,
longitude: lon,
altitude_m: 0.0,
}
}
fn make_mapobject(class: &str) -> MapObject {
let now = Utc::now();
MapObject {
h3_cell: 0,
mgrs_key: "33UWP00".to_string(),
class: class.to_string(),
class_group: "vehicle".to_string(),
gps_lat: 0.0,
gps_lon: 0.0,
size_width_m: 2.0,
size_length_m: 4.0,
confidence: 0.8,
first_seen: now,
last_seen: now,
mission_id: "m1".to_string(),
source: MapObjectSource::LocalObserved,
pending_upload: true,
}
}
fn make_ignored() -> IgnoredItem {
IgnoredItem {
id: Uuid::new_v4(),
mgrs: "33UWP01".to_string(),
h3_cell: 0,
class_group: "vehicle".to_string(),
decline_time: Utc::now(),
operator_id: None,
mission_id: "m1".to_string(),
retention_scope: RetentionScope::Mission,
expires_at: None,
source: IgnoredItemSource::LocalAppended,
pending_upload: true,
}
}
fn make_observation(class: &str) -> MapObjectObservation {
MapObjectObservation {
id: Uuid::new_v4(),
h3_cell: 0,
class: class.to_string(),
class_group: "vehicle".to_string(),
mission_id: "m1".to_string(),
uav_id: "uav_1".to_string(),
observed_at_monotonic_ns: 1,
observed_at_wallclock: Utc::now(),
gps_lat: 0.0,
gps_lon: 0.0,
mgrs: "33UWP02".to_string(),
size_width_m: 2.0,
size_length_m: 4.0,
confidence: 0.7,
diff_kind: DiffKind::New,
photo_ref: None,
raw_evidence: None,
}
}
/// Snapshot source whose `snapshot()` content can be mutated between
/// subscribes. Tracks how many times it has been called so the
/// reconnect test can verify the source was re-queried (rather than a
/// cached snapshot).
struct MutableSource {
bundle: parking_lot::Mutex<MapObjectsBundle>,
snapshots_emitted: AtomicUsize,
}
impl MutableSource {
fn new(initial: MapObjectsBundle) -> Self {
Self {
bundle: parking_lot::Mutex::new(initial),
snapshots_emitted: AtomicUsize::new(0),
}
}
fn set(&self, b: MapObjectsBundle) {
*self.bundle.lock() = b;
}
}
impl MapObjectsSnapshotSource for MutableSource {
fn snapshot(&self) -> MapObjectsBundle {
self.snapshots_emitted.fetch_add(1, Ordering::Relaxed);
self.bundle.lock().clone()
}
}
fn empty_bundle() -> MapObjectsBundle {
MapObjectsBundle {
schema_version: "1.0".to_string(),
mission_id: "m1".to_string(),
bbox: [coord(0.0, 0.0), coord(1.0, 1.0)],
map_objects: Vec::new(),
observations: Vec::new(),
ignored_items: Vec::new(),
as_of: Utc::now(),
freshness: Some(BundleFreshness::Fresh),
}
}
/// AC-1 — A fresh subscriber to MapObjectsBundle receives exactly one
/// snapshot, populated from the configured source.
#[tokio::test]
async fn ac1_first_subscribe_receives_snapshot() {
// Arrange
let (listener, port) = bind_ephemeral();
let server = TelemetryStream::new(64);
let handle = server.handle();
let initial = MapObjectsBundle {
map_objects: (0..50).map(|_| make_mapobject("tank")).collect(),
ignored_items: (0..10).map(|_| make_ignored()).collect(),
..empty_bundle()
};
let src = Arc::new(MutableSource::new(initial));
server.set_mapobjects_snapshot_source(src.clone());
let (_join, _guard) = server.spawn_grpc_server_on(listener).unwrap();
let mut client = connect(port).await;
let mut stream = client
.subscribe(Request::new(SubscribeRequest {
client_id: "op_a".to_string(),
topics: vec![TelemetryTopic::MapObjectsBundle as i32],
}))
.await
.unwrap()
.into_inner();
// Act — pull the first message off the stream.
let msg = timeout(Duration::from_secs(2), stream.next())
.await
.unwrap()
.unwrap()
.unwrap();
let payload: MapObjectsTopicMessage = serde_json::from_slice(&msg.payload_json).unwrap();
// Assert
let snap = match payload {
MapObjectsTopicMessage::Snapshot(s) => s,
MapObjectsTopicMessage::Diff(_) => panic!("expected Snapshot first; got Diff"),
};
assert_eq!(snap.bundle.map_objects.len(), 50);
assert_eq!(snap.bundle.ignored_items.len(), 10);
assert_eq!(src.snapshots_emitted.load(Ordering::Relaxed), 1);
// Drop client + handle scope (cleanup).
drop(stream);
drop(client);
drop(handle);
}
/// AC-2 — While connected, diffs appended by the composition root are
/// received by the client.
#[tokio::test]
async fn ac2_inflight_changes_emit_diffs() {
// Arrange
let (listener, port) = bind_ephemeral();
let server = TelemetryStream::new(64);
let handle = server.handle();
let src = Arc::new(MutableSource::new(empty_bundle()));
server.set_mapobjects_snapshot_source(src);
let (_join, _guard) = server.spawn_grpc_server_on(listener).unwrap();
let mut client = connect(port).await;
let mut stream = client
.subscribe(Request::new(SubscribeRequest {
client_id: "op_b".to_string(),
topics: vec![TelemetryTopic::MapObjectsBundle as i32],
}))
.await
.unwrap()
.into_inner();
// Drain the snapshot first.
let snap_msg = timeout(Duration::from_secs(2), stream.next())
.await
.unwrap()
.unwrap()
.unwrap();
let snap_payload: MapObjectsTopicMessage =
serde_json::from_slice(&snap_msg.payload_json).unwrap();
assert!(matches!(snap_payload, MapObjectsTopicMessage::Snapshot(_)));
// Wait for the client to register before publishing diffs.
for _ in 0..50 {
if handle.snapshot().subscribed_clients == 1 {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
// Act — push one diff with 3 added observations + 1 ignored item.
let diff = MapObjectsDiff {
added: vec![
make_observation("tank"),
make_observation("apc"),
make_observation("truck"),
],
moved: vec![],
removed_candidates: vec![],
ignored: vec![make_ignored()],
};
handle.push_mapobjects_diff(diff).unwrap();
// Pull the next message — must be a Diff carrying our content.
let diff_msg = timeout(Duration::from_secs(2), stream.next())
.await
.unwrap()
.unwrap()
.unwrap();
let diff_payload: MapObjectsTopicMessage =
serde_json::from_slice(&diff_msg.payload_json).unwrap();
let received_diff = match diff_payload {
MapObjectsTopicMessage::Diff(d) => d,
MapObjectsTopicMessage::Snapshot(_) => panic!("expected Diff; got Snapshot"),
};
// Assert
assert_eq!(received_diff.added.len(), 3);
assert_eq!(received_diff.ignored.len(), 1);
assert!(received_diff.moved.is_empty());
assert!(received_diff.removed_candidates.is_empty());
}
/// AC-3 — Reconnect after disconnect emits a fresh snapshot reflecting
/// current state; diffs that flew during the gap are NOT replayed.
#[tokio::test]
async fn ac3_reconnect_resnaps_without_replay() {
// Arrange
let (listener, port) = bind_ephemeral();
let server = TelemetryStream::new(64);
let handle = server.handle();
let src = Arc::new(MutableSource::new(empty_bundle()));
server.set_mapobjects_snapshot_source(src.clone());
let (_join, _guard) = server.spawn_grpc_server_on(listener).unwrap();
// Subscribe once, drain snapshot, drop.
let mut client = connect(port).await;
let mut stream = client
.subscribe(Request::new(SubscribeRequest {
client_id: "op_c".to_string(),
topics: vec![TelemetryTopic::MapObjectsBundle as i32],
}))
.await
.unwrap()
.into_inner();
let first_snap = timeout(Duration::from_secs(2), stream.next())
.await
.unwrap()
.unwrap()
.unwrap();
let first: MapObjectsTopicMessage = serde_json::from_slice(&first_snap.payload_json).unwrap();
let first_snap = match first {
MapObjectsTopicMessage::Snapshot(s) => s,
MapObjectsTopicMessage::Diff(_) => panic!("first must be Snapshot"),
};
assert!(first_snap.bundle.map_objects.is_empty());
// Disconnect.
drop(stream);
drop(client);
for _ in 0..50 {
if handle.snapshot().subscribed_clients == 0 {
break;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
// Act — store grew by 5 while client was disconnected. Also push
// a couple of diffs that the reconnecting client must NOT see.
src.set(MapObjectsBundle {
map_objects: (0..5).map(|_| make_mapobject("tank")).collect(),
..empty_bundle()
});
handle
.push_mapobjects_diff(MapObjectsDiff {
added: vec![make_observation("ghost_during_gap")],
..MapObjectsDiff::default()
})
.unwrap();
// Reconnect with the same client_id.
let mut client2 = connect(port).await;
let mut stream2 = client2
.subscribe(Request::new(SubscribeRequest {
client_id: "op_c".to_string(),
topics: vec![TelemetryTopic::MapObjectsBundle as i32],
}))
.await
.unwrap()
.into_inner();
let resnap_msg = timeout(Duration::from_secs(2), stream2.next())
.await
.unwrap()
.unwrap()
.unwrap();
let resnap_payload: MapObjectsTopicMessage =
serde_json::from_slice(&resnap_msg.payload_json).unwrap();
// Assert — first message after reconnect is a snapshot reflecting
// the new bundle. The skipped-during-gap diff is NOT in the
// stream (we read with a short timeout to prove no replay).
let resnap = match resnap_payload {
MapObjectsTopicMessage::Snapshot(s) => s,
MapObjectsTopicMessage::Diff(_) => {
panic!("first message after reconnect MUST be Snapshot, not Diff");
}
};
assert_eq!(
resnap.bundle.map_objects.len(),
5,
"snapshot must reflect post-gap store"
);
assert!(
src.snapshots_emitted.load(Ordering::Relaxed) >= 2,
"snapshot source must have been re-queried on reconnect"
);
// The reconnected stream should NOT immediately deliver another
// message — the gap diff was broadcast before reconnect and a
// late subscriber MUST not see it.
let maybe_extra = timeout(Duration::from_millis(300), stream2.next()).await;
assert!(
maybe_extra.is_err(),
"reconnect MUST NOT replay gap diff (got unexpected message)"
);
}
+307
View File
@@ -0,0 +1,307 @@
//! AZ-676 integration tests — SubscribeVideo RPC, ai_locked atomic
//! coordination, and per-mode delivery semantics.
use std::net::TcpListener;
use std::sync::atomic::Ordering;
use std::sync::Arc;
use std::time::Duration;
use bytes::Bytes;
use tokio::time::timeout;
use tokio_stream::StreamExt;
use tonic::transport::{Channel, Endpoint};
use tonic::Request;
use shared::contracts::TelemetrySink;
use shared::models::frame::{Frame, PixelFormat as SharedPixelFormat};
use telemetry_stream::internal::video::VideoPath;
use telemetry_stream::{
video_message, SubscribeVideoRequest, TelemetryStream, TelemetryStreamClient,
TelemetryStreamConfig, VideoMode,
};
fn bind_ephemeral() -> (TcpListener, u16) {
let l = TcpListener::bind("127.0.0.1:0").expect("bind ephemeral");
let port = l.local_addr().unwrap().port();
(l, port)
}
async fn connect(port: u16) -> TelemetryStreamClient<Channel> {
let url = format!("http://127.0.0.1:{port}");
let endpoint = Endpoint::from_shared(url)
.unwrap()
.connect_timeout(Duration::from_secs(2));
for _ in 0..50 {
if let Ok(c) = TelemetryStreamClient::connect(endpoint.clone()).await {
return c;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
panic!("gRPC client failed to connect");
}
fn make_frame(seq: u64, payload_len: usize) -> Frame {
Frame {
seq,
capture_ts_monotonic_ns: seq * 1_000_000,
decode_ts_monotonic_ns: seq * 1_000_000 + 10_000,
pixels: Arc::new(Bytes::from(vec![(seq & 0xff) as u8; payload_len])),
width: 1920,
height: 1080,
pix_fmt: SharedPixelFormat::Nv12,
ai_locked: false,
}
}
/// AC-1 — rtsp_forward emits exactly the configured URL in the
/// session-start message; no frames flow.
#[tokio::test]
async fn ac1_rtsp_forward_emits_url_only() {
// Arrange
let (listener, port) = bind_ephemeral();
let cfg = TelemetryStreamConfig {
video_path: VideoPath::RtspForward {
url: "rtsp://camera.local:8554/stream0".to_string(),
},
..TelemetryStreamConfig::default()
};
let server = TelemetryStream::with_config(cfg);
let handle = server.handle();
let (_join, _guard) = server.spawn_grpc_server_on(listener).unwrap();
let mut client = connect(port).await;
// Act
let mut stream = client
.subscribe_video(Request::new(SubscribeVideoRequest {
client_id: "op_1".to_string(),
}))
.await
.unwrap()
.into_inner();
let first = timeout(Duration::from_secs(2), stream.next())
.await
.expect("session-start within 2s")
.expect("stream open")
.expect("ok status");
// Push a frame anyway — in rtsp_forward mode it must NOT flow.
handle.push_frame(make_frame(1, 1024)).await.unwrap();
let second = timeout(Duration::from_millis(500), stream.next()).await;
// Assert
let kind = first.kind.unwrap();
match kind {
video_message::Kind::Start(start) => {
assert_eq!(start.mode, VideoMode::RtspForward as i32);
assert_eq!(start.rtsp_url, "rtsp://camera.local:8554/stream0");
}
other => panic!("expected Start, got {other:?}"),
}
assert!(
second.is_err(),
"no further messages expected in rtsp_forward mode"
);
assert_eq!(handle.video_snapshot().mode, "rtsp_forward");
}
/// AC-2 — bytes_inline forwards encoded frames to subscribed clients.
#[tokio::test]
async fn ac2_bytes_inline_forwards_frames() {
// Arrange
let (listener, port) = bind_ephemeral();
let cfg = TelemetryStreamConfig {
video_path: VideoPath::BytesInline,
// Generous capacity so the test client keeps up without lag.
video_capacity: 256,
..TelemetryStreamConfig::default()
};
let server = TelemetryStream::with_config(cfg);
let handle = server.handle();
let (_join, _guard) = server.spawn_grpc_server_on(listener).unwrap();
let mut client = connect(port).await;
let mut stream = client
.subscribe_video(Request::new(SubscribeVideoRequest {
client_id: "op_inline".to_string(),
}))
.await
.unwrap()
.into_inner();
// Drain the session-start first.
let start = timeout(Duration::from_secs(2), stream.next())
.await
.unwrap()
.unwrap()
.unwrap();
assert!(matches!(start.kind.unwrap(), video_message::Kind::Start(_)));
// Wait until the server has registered the session before
// publishing so no frames are emitted before the broadcast has a
// receiver.
for _ in 0..100 {
if handle.video_snapshot().video_session_count == 1 {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
assert_eq!(handle.video_snapshot().video_session_count, 1);
// Act — publish 100 frames; verify the client gets each one in
// monotonically increasing sequence.
let total: u64 = 100;
for seq in 0..total {
// Tiny pixel payload so the test isn't expensive.
handle.push_frame(make_frame(seq, 64)).await.unwrap();
}
let mut received = 0u64;
let mut last_seq: Option<u64> = None;
while received < total {
let msg = timeout(Duration::from_secs(2), stream.next())
.await
.expect("ac2 stalled — frame not received in 2s")
.expect("stream open")
.expect("ok status");
match msg.kind.unwrap() {
video_message::Kind::Frame(f) => {
if let Some(prev) = last_seq {
assert!(f.seq > prev, "monotonic seq violated: {prev} → {}", f.seq);
}
last_seq = Some(f.seq);
received += 1;
}
video_message::Kind::Start(_) => panic!("unexpected second Start"),
}
}
// Assert
assert_eq!(received, total);
let snap = handle.video_snapshot();
assert_eq!(snap.published_frames, total);
assert_eq!(snap.bytes_inline_drops_total, 0);
}
/// AC-3 — ai_locked flips true on first subscriber, false when the
/// last subscriber disconnects.
#[tokio::test]
async fn ac3_ai_locked_toggles_on_session_start_and_stop() {
// Arrange
let (listener, port) = bind_ephemeral();
let cfg = TelemetryStreamConfig {
video_path: VideoPath::BytesInline,
..TelemetryStreamConfig::default()
};
let server = TelemetryStream::with_config(cfg);
let handle = server.handle();
let ai_locked = server.ai_locked_handle();
let (_join, _guard) = server.spawn_grpc_server_on(listener).unwrap();
// No clients yet → false.
assert!(!ai_locked.load(Ordering::Acquire));
// Act 1 — first subscriber connects; flag must flip to true.
let mut c1 = connect(port).await;
let mut s1 = c1
.subscribe_video(Request::new(SubscribeVideoRequest {
client_id: "op_a".to_string(),
}))
.await
.unwrap()
.into_inner();
let _start = timeout(Duration::from_secs(2), s1.next())
.await
.unwrap()
.unwrap()
.unwrap();
for _ in 0..100 {
if ai_locked.load(Ordering::Acquire) {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
assert!(
ai_locked.load(Ordering::Acquire),
"ai_locked MUST be true once first session is active"
);
// Act 2 — second subscriber connects; flag stays true.
let mut c2 = connect(port).await;
let mut s2 = c2
.subscribe_video(Request::new(SubscribeVideoRequest {
client_id: "op_b".to_string(),
}))
.await
.unwrap()
.into_inner();
let _start = timeout(Duration::from_secs(2), s2.next())
.await
.unwrap()
.unwrap()
.unwrap();
for _ in 0..100 {
if handle.video_snapshot().video_session_count == 2 {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
assert!(ai_locked.load(Ordering::Acquire));
assert_eq!(handle.video_snapshot().video_session_count, 2);
// Act 3 — drop second client; one session left, still locked.
drop(s2);
drop(c2);
for _ in 0..100 {
if handle.video_snapshot().video_session_count == 1 {
break;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
assert_eq!(handle.video_snapshot().video_session_count, 1);
assert!(ai_locked.load(Ordering::Acquire));
// Act 4 — drop last client; ai_locked flips to false.
drop(s1);
drop(c1);
for _ in 0..100 {
if !ai_locked.load(Ordering::Acquire) {
break;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
assert!(
!ai_locked.load(Ordering::Acquire),
"ai_locked MUST be false after last session leaves"
);
assert_eq!(handle.video_snapshot().video_session_count, 0);
}
/// Empty client_id is rejected at the boundary (parity with Subscribe).
#[tokio::test]
async fn empty_client_id_rejected() {
// Arrange
let (listener, port) = bind_ephemeral();
let cfg = TelemetryStreamConfig {
video_path: VideoPath::BytesInline,
..TelemetryStreamConfig::default()
};
let server = TelemetryStream::with_config(cfg);
let _h = server.handle();
let (_join, _guard) = server.spawn_grpc_server_on(listener).unwrap();
let mut client = connect(port).await;
// Act
let err = client
.subscribe_video(Request::new(SubscribeVideoRequest {
client_id: String::new(),
}))
.await
.expect_err("empty client_id must error");
// Assert
assert_eq!(err.code(), tonic::Code::InvalidArgument);
}