[AZ-651] [AZ-668] lost-link failsafe ladder + mapobjects persistence (batch 7)

AZ-651 (mission_executor lost-link ladder): - LostLinkLadder pure-logic state machine (LinkOk -> Degraded -> Lost -> LinkLostInFollow + MavlinkLost branch). Configurable thresholds via LostLinkConfig. - LostLinkCommandIssuer trait + MavlinkCommandIssuer production impl emitting MAV_CMD_NAV_RETURN_TO_LAUNCH via MavlinkHandle::send_command. - LostLinkDriver task wires the ladder to operator-link watch, MAVLink LinkEvent broadcast, and optional target-follow signal. On RTL, driver calls the issuer THEN MissionExecutorHandle::failsafe_trigger. - failsafe_trigger(LinkLost | LinkLostInFollow) short-circuits FlyMission -> Land via direct FSM state mutation + TransitionEvent emission; Paused state is intentionally NOT overridden. - Tests: 4/4 ACs locally green (degraded-no-rtl; lost-fires-once; follow-grace; mavlink-loss-no-rtl) plus driver + FSM integration. AZ-668 (mapobjects_store persistence): - Snapshot serializable shape + Store::{to_snapshot,from_snapshot} round trip. - MapObjectsPersistence async trait + JsonSnapshotEngine default impl (write to .tmp, sync_all, atomic rename, best-effort parent fsync). - PersistenceError::{Corrupt, SchemaMismatch} surfaces explicit errors on bad blob; PersistenceMetrics tracks last_snapshot_ts, snapshot_size_bytes, snapshot_errors_total. - MapObjectsStore::from_snapshot factory for crash recovery from the composition root. - Tests: 4/4 ACs locally green (round-trip; atomic rename ignores partial .tmp; crash recovery preserves pending; corruption returns explicit error) plus schema-mismatch + metrics smoke checks. Quality gates: - cargo fmt: clean. - cargo clippy -p mission_executor -p mapobjects_store --tests: 0 warns. - cargo test --workspace: all green. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-06-21 20:31:09 +00:00 · 2026-05-19 18:59:28 +03:00
parent 23366a5c6d
commit 2bcd4a8059
16 changed files with 1940 additions and 8 deletions
@@ -0,0 +1,308 @@
+//! AZ-668 acceptance criteria — in-memory + JSON snapshot persistence.
+//!
+//! Covers:
+//! - AC-1 snapshot + reload round-trip
+//! - AC-2 atomic rename prevents partial writes
+//! - AC-3 crash recovery loads pending
+//! - AC-4 corruption returns explicit error (never silently empty)
+//!
+//! Plus a metrics smoke-check (`last_snapshot_ts`,
+//! `snapshot_size_bytes`, `snapshot_errors_total`) since the AC requires
+//! those three to be surfaced.
+
+use std::path::PathBuf;
+
+use chrono::Utc;
+use mapobjects_store::{
+    ClassifyInput, JsonSnapshotEngine, MapObjectsPersistence, MapObjectsStore,
+    MapObjectsStoreConfig, PersistenceError,
+};
+use shared::models::mapobject::{IgnoredItem, IgnoredItemSource, RetentionScope};
+use tempfile::TempDir;
+use uuid::Uuid;
+
+fn input(lat: f64, lon: f64, class: &str, mission_id: &str) -> ClassifyInput {
+    ClassifyInput {
+        gps_lat: lat,
+        gps_lon: lon,
+        mgrs: format!("MGRS({lat},{lon})"),
+        class: class.into(),
+        size_width_m: 1.0,
+        size_length_m: 1.0,
+        confidence: 0.9,
+        mission_id: mission_id.into(),
+        observed_at: Utc::now(),
+        uav_id: "uav1".into(),
+        observed_at_monotonic_ns: 0,
+    }
+}
+
+fn ignored_item(mgrs: &str, class_group: &str, mission_id: &str) -> IgnoredItem {
+    IgnoredItem {
+        id: Uuid::new_v4(),
+        mgrs: mgrs.into(),
+        h3_cell: 0,
+        class_group: class_group.into(),
+        decline_time: Utc::now(),
+        operator_id: Some("op-A".into()),
+        mission_id: mission_id.into(),
+        retention_scope: RetentionScope::Mission,
+        expires_at: None,
+        source: IgnoredItemSource::LocalAppended,
+        pending_upload: true,
+    }
+}
+
+/// AC-1 — snapshot + reload round-trip preserves indexed objects,
+/// ignored items, and pending observations.
+#[tokio::test]
+async fn ac1_snapshot_reload_round_trip() {
+    // Arrange — store with 100 MapObjects across a square of latitudes,
+    // 10 IgnoredItems, and 5 pending observations (the latter come "for
+    // free" from the first 5 classify calls).
+    let tmp = TempDir::new().unwrap();
+    let mission_id = "ac1-mission";
+    let engine = JsonSnapshotEngine::new(tmp.path());
+
+    let store = MapObjectsStore::new(MapObjectsStoreConfig::default());
+    let h = store.handle();
+    for i in 0..100 {
+        let lat = 50.45 + (i as f64) * 0.001;
+        let lon = 30.52 + (i as f64) * 0.001;
+        h.classify(input(lat, lon, "tank", mission_id)).unwrap();
+    }
+    for i in 0..10 {
+        h.append_ignored(ignored_item(
+            &format!("MGRS-{i}"),
+            "concealed_position",
+            mission_id,
+        ))
+        .unwrap();
+    }
+    assert_eq!(h.len().unwrap(), 100);
+
+    // Act — capture, save, then load into a brand-new store
+    let snap = h.to_snapshot(mission_id).unwrap();
+    engine.save_snapshot(&snap).await.unwrap();
+
+    let loaded = engine
+        .load_snapshot(mission_id)
+        .await
+        .expect("load ok")
+        .expect("file present");
+    let restored = MapObjectsStore::from_snapshot(MapObjectsStoreConfig::default(), loaded).unwrap();
+    let rh = restored.handle();
+
+    // Assert — counts match and pending log survived
+    assert_eq!(rh.len().unwrap(), 100);
+    assert_eq!(rh.pending_observations_count().unwrap(), 100);
+    // The 10 LocalAppended IgnoredItems went into pending_ignored too.
+    assert_eq!(rh.pending_ignored_count().unwrap(), 10);
+    // Verify the ignored-set survived the round trip with a probe.
+    assert!(rh.is_ignored("MGRS-0", "concealed_position").unwrap());
+    assert!(rh.is_ignored("MGRS-9", "concealed_position").unwrap());
+    assert!(!rh.is_ignored("MGRS-42", "concealed_position").unwrap());
+}
+
+/// AC-2 — atomic rename prevents partial writes.
+///
+/// We simulate a kill-9 mid-write by creating a leftover `.tmp` file
+/// alongside a valid `.json` snapshot. The engine must still load the
+/// good snapshot (NOT the partial `.tmp`).
+#[tokio::test]
+async fn ac2_atomic_rename_ignores_partial_tmp_file() {
+    // Arrange — write a real snapshot, then poison its sibling `.tmp`
+    let tmp = TempDir::new().unwrap();
+    let mission_id = "ac2-mission";
+    let engine = JsonSnapshotEngine::new(tmp.path());
+
+    let store = MapObjectsStore::new(MapObjectsStoreConfig::default());
+    let h = store.handle();
+    h.classify(input(50.45, 30.52, "tank", mission_id)).unwrap();
+    let snap = h.to_snapshot(mission_id).unwrap();
+    engine.save_snapshot(&snap).await.unwrap();
+
+    // Poison: write a half-finished blob to the .tmp sibling
+    let tmp_path: PathBuf = tmp
+        .path()
+        .join("mapobjects")
+        .join(format!("{mission_id}.json.tmp"));
+    tokio::fs::write(&tmp_path, b"{\"partial\":")
+        .await
+        .expect("write poisoned tmp");
+    assert!(tmp_path.exists(), "partial .tmp file should exist");
+
+    // Act — fresh engine loads from the same dir
+    let engine2 = JsonSnapshotEngine::new(tmp.path());
+    let loaded = engine2
+        .load_snapshot(mission_id)
+        .await
+        .expect("load ok")
+        .expect("good snapshot present");
+
+    // Assert — got the good snapshot, ignoring the partial .tmp
+    assert_eq!(loaded.mission_id, mission_id);
+    assert_eq!(loaded.map_objects.len(), 1);
+    // .tmp file is still on disk — the loader never touches it.
+    assert!(tmp_path.exists());
+}
+
+/// AC-3 — crash recovery loads pending observations.
+#[tokio::test]
+async fn ac3_crash_recovery_loads_pending() {
+    // Arrange — first process: classify, save
+    let tmp = TempDir::new().unwrap();
+    let mission_id = "ac3-mission";
+    let engine = JsonSnapshotEngine::new(tmp.path());
+    let store = MapObjectsStore::new(MapObjectsStoreConfig::default());
+    let h = store.handle();
+    for i in 0..7 {
+        let lat = 50.45 + (i as f64) * 0.001;
+        h.classify(input(lat, 30.52, "tank", mission_id)).unwrap();
+    }
+    let pre_crash_count = h.pending_observations_count().unwrap();
+    assert_eq!(pre_crash_count, 7);
+    engine
+        .save_snapshot(&h.to_snapshot(mission_id).unwrap())
+        .await
+        .unwrap();
+    drop(store); // simulate process death
+
+    // Act — second process: fresh engine, load
+    let engine2 = JsonSnapshotEngine::new(tmp.path());
+    let snap = engine2
+        .load_snapshot(mission_id)
+        .await
+        .unwrap()
+        .expect("snapshot present");
+    let recovered =
+        MapObjectsStore::from_snapshot(MapObjectsStoreConfig::default(), snap).unwrap();
+
+    // Assert — pending log matches pre-crash count
+    assert_eq!(
+        recovered.handle().pending_observations_count().unwrap(),
+        pre_crash_count
+    );
+}
+
+/// AC-4 — corruption surfaces an explicit error; metrics increment.
+#[tokio::test]
+async fn ac4_corruption_returns_explicit_error() {
+    // Arrange — write a known-truncated blob into the snapshot path
+    let tmp = TempDir::new().unwrap();
+    let mission_id = "ac4-mission";
+    let engine = JsonSnapshotEngine::new(tmp.path());
+
+    let dir = tmp.path().join("mapobjects");
+    tokio::fs::create_dir_all(&dir).await.unwrap();
+    let path = dir.join(format!("{mission_id}.json"));
+    // Truncated JSON: opening brace + half a key, no closing brace.
+    tokio::fs::write(&path, b"{\"schema_version\":1,\"mission_id\":\"trunc")
+        .await
+        .unwrap();
+
+    // Act
+    let result = engine.load_snapshot(mission_id).await;
+
+    // Assert — explicit Corrupt error; the store does NOT silently
+    // come up empty (caller surfaces to operator and refuses to start)
+    match result {
+        Err(PersistenceError::Corrupt { path: p, reason }) => {
+            assert_eq!(p, path);
+            assert!(reason.contains("deserialize"));
+        }
+        other => panic!("expected Corrupt, got {other:?}"),
+    }
+    // snapshot_errors_total incremented
+    let m = engine.metrics();
+    assert!(m.snapshot_errors_total >= 1);
+}
+
+/// Schema-mismatch is also treated as corruption — a future engine
+/// version bump on disk must not be silently accepted by the running
+/// binary.
+#[tokio::test]
+async fn schema_mismatch_returns_explicit_error() {
+    // Arrange — write a valid-shape JSON but with a future schema_version
+    let tmp = TempDir::new().unwrap();
+    let mission_id = "schema-mismatch-mission";
+    let engine = JsonSnapshotEngine::new(tmp.path());
+
+    let dir = tmp.path().join("mapobjects");
+    tokio::fs::create_dir_all(&dir).await.unwrap();
+    let path = dir.join(format!("{mission_id}.json"));
+    tokio::fs::write(
+        &path,
+        br#"{
+            "schema_version": 999,
+            "mission_id": "schema-mismatch-mission",
+            "as_of": "2026-01-01T00:00:00Z",
+            "map_objects": [],
+            "ignored_items": [],
+            "pending_observations": [],
+            "pending_ignored": [],
+            "sync_state": "fresh_boot"
+        }"#,
+    )
+    .await
+    .unwrap();
+
+    // Act
+    let result = engine.load_snapshot(mission_id).await;
+
+    // Assert
+    match result {
+        Err(PersistenceError::SchemaMismatch {
+            expected, found, ..
+        }) => {
+            assert_eq!(expected, 1);
+            assert_eq!(found, 999);
+        }
+        other => panic!("expected SchemaMismatch, got {other:?}"),
+    }
+}
+
+/// Metrics smoke-check — `last_snapshot_ts` + `snapshot_size_bytes`
+/// populated after a successful save.
+#[tokio::test]
+async fn metrics_populated_after_successful_save() {
+    // Arrange
+    let tmp = TempDir::new().unwrap();
+    let engine = JsonSnapshotEngine::new(tmp.path());
+    let store = MapObjectsStore::new(MapObjectsStoreConfig::default());
+    let h = store.handle();
+    h.classify(input(50.45, 30.52, "tank", "metrics-mission"))
+        .unwrap();
+
+    // Pre-save metrics empty
+    let pre = engine.metrics();
+    assert!(pre.last_snapshot_ts.is_none());
+    assert!(pre.snapshot_size_bytes.is_none());
+    assert_eq!(pre.snapshot_errors_total, 0);
+
+    // Act
+    let snap = h.to_snapshot("metrics-mission").unwrap();
+    engine.save_snapshot(&snap).await.unwrap();
+
+    // Assert
+    let post = engine.metrics();
+    assert!(post.last_snapshot_ts.is_some());
+    let size = post.snapshot_size_bytes.expect("size recorded");
+    assert!(size > 0);
+    assert_eq!(post.snapshot_errors_total, 0);
+}
+
+/// `load_snapshot` for an unknown mission returns `Ok(None)` (not
+/// `Err`). This is the "first boot, no prior state" case.
+#[tokio::test]
+async fn load_missing_returns_none() {
+    // Arrange
+    let tmp = TempDir::new().unwrap();
+    let engine = JsonSnapshotEngine::new(tmp.path());
+
+    // Act
+    let result = engine.load_snapshot("never-saved").await.unwrap();
+
+    // Assert
+    assert!(result.is_none());
+}