mirror of
https://github.com/azaion/gps-denied-onboard.git
synced 2026-04-23 02:06:36 +00:00
feat(testing): per-frame JSONL trace in E2EHarness
Opt-in trace_path parameter dumps one JSON record per processed frame with the fields diagnostics need: frame_idx, timestamp_ns, vo_success, alignment_success, tracking_state, confidence, eskf_initialized, eskf_position_enu (or None), eskf_pos_sigma_m, estimate_lat/lon, gt_lat/lon/alt No perf cost when trace_path is None. File is rotated per run — safe to point at /tmp/foo.jsonl for ad-hoc debugging. First real run on EuRoC MH_01 (100 frames) immediately exposes the concrete divergence: vo_success=0/100 (VO never engages on EuRoC grayscale imagery with current SP+LG adapter), eskf_initialized=0/100, alignment_success=77/100 (satellite-fallback path fires). Diagnosis that was hidden behind a single "ATE=10.9 km" number is now machine- readable per frame. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -13,7 +13,9 @@ What the harness does NOT do:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
@@ -29,7 +31,11 @@ from gps_denied.core.processor import FlightProcessor
|
||||
from gps_denied.core.recovery import FailureRecoveryCoordinator
|
||||
from gps_denied.core.vo import SequentialVisualOdometry
|
||||
from gps_denied.schemas.graph import FactorGraphConfig
|
||||
from gps_denied.testing.datasets.base import DatasetAdapter, PlatformClass
|
||||
from gps_denied.testing.datasets.base import (
|
||||
DatasetAdapter,
|
||||
DatasetPose,
|
||||
PlatformClass,
|
||||
)
|
||||
|
||||
EARTH_R = 6_378_137.0
|
||||
|
||||
@@ -59,10 +65,12 @@ class E2EHarness:
|
||||
adapter: DatasetAdapter,
|
||||
flight_id: str = "e2e-flight",
|
||||
max_frames: Optional[int] = None,
|
||||
trace_path: Optional[Path] = None,
|
||||
) -> None:
|
||||
self._adapter = adapter
|
||||
self._flight_id = flight_id
|
||||
self._max_frames = max_frames
|
||||
self._trace_path: Optional[Path] = Path(trace_path) if trace_path else None
|
||||
self._estimates: list[tuple[int, Optional[tuple[float, float, float]]]] = []
|
||||
|
||||
async def run(self) -> HarnessResult:
|
||||
@@ -74,15 +82,35 @@ class E2EHarness:
|
||||
frames = frames[: self._max_frames]
|
||||
gt_poses = gt_poses[: self._max_frames]
|
||||
|
||||
for frame in frames:
|
||||
image = self._load_or_synth_image(frame.image_path)
|
||||
result = await processor.process_frame(
|
||||
self._flight_id, frame.frame_idx, image
|
||||
)
|
||||
est = None
|
||||
if result.gps is not None:
|
||||
est = (result.gps.lat, result.gps.lon, 0.0) # alt not returned here
|
||||
self._estimates.append((frame.frame_idx, est))
|
||||
# Align GT by index so trace records can pair each frame with the
|
||||
# corresponding pose without an expensive timestamp search.
|
||||
gt_by_idx: dict[int, DatasetPose] = {}
|
||||
for i, pose in enumerate(gt_poses):
|
||||
gt_by_idx[i] = pose
|
||||
|
||||
trace_fh = None
|
||||
if self._trace_path is not None:
|
||||
self._trace_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
trace_fh = self._trace_path.open("w")
|
||||
|
||||
try:
|
||||
for frame in frames:
|
||||
image = self._load_or_synth_image(frame.image_path)
|
||||
result = await processor.process_frame(
|
||||
self._flight_id, frame.frame_idx, image
|
||||
)
|
||||
est = None
|
||||
if result.gps is not None:
|
||||
est = (result.gps.lat, result.gps.lon, 0.0)
|
||||
self._estimates.append((frame.frame_idx, est))
|
||||
|
||||
if trace_fh is not None:
|
||||
gt = gt_by_idx.get(frame.frame_idx)
|
||||
record = self._trace_record(processor, frame, result, gt)
|
||||
trace_fh.write(json.dumps(record) + "\n")
|
||||
finally:
|
||||
if trace_fh is not None:
|
||||
trace_fh.close()
|
||||
|
||||
gt_enu = self._poses_to_enu(gt_poses)
|
||||
est_enu = self._estimates_to_enu(gt_poses[0] if gt_poses else None)
|
||||
@@ -96,6 +124,51 @@ class E2EHarness:
|
||||
platform_class=self._adapter.capabilities.platform_class,
|
||||
)
|
||||
|
||||
def _trace_record(
|
||||
self,
|
||||
processor: FlightProcessor,
|
||||
frame,
|
||||
result,
|
||||
gt: Optional[DatasetPose],
|
||||
) -> dict:
|
||||
"""Build one JSONL record describing the product's state after a frame.
|
||||
|
||||
Captures VO success, ESKF state (position + trace(cov) as a scalar
|
||||
sigma), and the estimate/GT pair. Enough to diagnose *where* the
|
||||
pipeline diverges without dumping raw images or covariance matrices.
|
||||
"""
|
||||
eskf = processor._eskf.get(self._flight_id) # noqa: SLF001 — test harness
|
||||
eskf_initialized = bool(eskf and eskf.initialized)
|
||||
eskf_position = None
|
||||
eskf_sigma = None
|
||||
if eskf_initialized:
|
||||
pos = eskf.position
|
||||
eskf_position = [float(pos[0]), float(pos[1]), float(pos[2])]
|
||||
cov = eskf.covariance
|
||||
if cov is not None:
|
||||
# Scalar horizontal+vertical position uncertainty summary.
|
||||
eskf_sigma = float(np.sqrt(np.trace(cov[0:3, 0:3]) / 3.0))
|
||||
|
||||
est_lat = result.gps.lat if result.gps is not None else None
|
||||
est_lon = result.gps.lon if result.gps is not None else None
|
||||
|
||||
return {
|
||||
"frame_idx": frame.frame_idx,
|
||||
"timestamp_ns": frame.timestamp_ns,
|
||||
"vo_success": bool(result.vo_success),
|
||||
"alignment_success": bool(result.alignment_success),
|
||||
"tracking_state": result.tracking_state.value,
|
||||
"confidence": float(result.confidence),
|
||||
"eskf_initialized": eskf_initialized,
|
||||
"eskf_position_enu": eskf_position,
|
||||
"eskf_pos_sigma_m": eskf_sigma,
|
||||
"estimate_lat": est_lat,
|
||||
"estimate_lon": est_lon,
|
||||
"gt_lat": gt.lat if gt else None,
|
||||
"gt_lon": gt.lon if gt else None,
|
||||
"gt_alt": gt.alt if gt else None,
|
||||
}
|
||||
|
||||
def _build_processor(self) -> FlightProcessor:
|
||||
repo = MagicMock()
|
||||
streamer = MagicMock()
|
||||
|
||||
Reference in New Issue
Block a user