azaion/gps-denied-onboard
1
0
Fork 0
mirror of https://github.com/azaion/gps-denied-onboard.git synced 2026-06-23 02:41:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

[AZ-600] Batch 80: refactor sitl_replay_builder to strategy pattern

Browse Source 
Replace per-scenario fixture builders with a parameterized strategy
framework so future Derkachi-based scenarios compose existing pieces
instead of duplicating ~200 lines of orchestration per scenario.

New e2e/fixtures/sitl_replay_builder/builder.py:
- VideoSource ABC + StillImagesSource, Mp4PassthroughSource
- TlogSource ABC + SyntheticStationaryTlog, ImuCsvTlog
- FdrProjection ABC + RawFdrPassthrough, OutboundMessagesProjection
- FixtureBuilderConfig + build_fixtures(cfg) orchestrator
- Consolidated MAVLink pack_raw_imu / pack_attitude helpers
- Consolidated run_gps_denied_replay + write_observer_fixture

build_p01_fixtures.py: 423 -> 107 lines (75% reduction).
build_p02_fixtures.py: 292 -> 98 lines (66% reduction).
_common.py: deleted (folded into builder.py).

Tests reorganized:
- test_sitl_replay_builder_builder.py (new, 33 strategy-level tests)
- test_sitl_replay_builder.py (slimmed, 6 FT-P-01 integration)
- test_sitl_replay_builder_p02.py (slimmed, 7 FT-P-02 integration)

README documents the strategy framework + a worked example for
adding FT-P-04 in ~30 lines (no new strategy code required).

Regression gate: 700 passing (was 686; +14 from finer-grained
coverage of new strategy classes and the build_fixtures orchestrator).

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Oleksandr Bezdieniezhnykh
2026-05-17 14:19:08 +03:00
parent 4e0717e543
commit 7fb3cb3f34
13 changed files with 2050 additions and 1272 deletions
Download Patch File Download Diff File Expand all files Collapse all files
e2e/fixtures/sitl_replay_builder/build_p01_fixtures.py
+40 -348
View File
@@ -1,56 +1,44 @@
"""FT-P-01 fixture builder (AZ-598).
"""FT-P-01 fixture builder (AZ-598; refactored to strategy pattern in AZ-600).
Produces:
Composes the parameterized fixture-builder framework
(``e2e.fixtures.sitl_replay_builder.builder``) into the FT-P-01 scenario:
* ``outbound_messages_<fc_kind>_<host>.json`` — per-image SUT outbound GPS
estimates, in image-order. ``null`` entries encode per-image timeouts.
* ``observer_<fc_kind>_<host>.json`` — minimal observer config so
``sitl_observer.get_observer`` succeeds when the fixtures are activated.
* Video source: 60 ``AD000NN.jpg`` still images encoded at ``fps``.
* Tlog source: synthetic stationary RAW_IMU + ATTITUDE pairs.
* FDR projection: parse ``outbound_position_estimate`` records and write
``outbound_messages_<fc_kind>_<host>.json`` (the FT-P-01 fixture shape).
Strategy: drive the production ``gps-denied-replay`` CLI against a 1 fps
MP4 encoded from the FT-P-01 still-image set and a synthetic stationary
tlog, then read the resulting FDR JSONL for per-frame outbound estimates.
Compared with the rejected "live SITL docker capture" path this:
* Adds no new SUT-side frame-ingestion code (reuses
``ReplayInputAdapter`` + ``VideoFileFrameSource``).
* Bypasses the SITL container entirely (FT-P-01 tests upstream
geo-estimate accuracy; the FC is just a delivery channel).
* Runs as a single subprocess instead of a multi-container compose.
The helpers below are intentionally dependency-injectable so the unit
tests can mock OpenCV / pymavlink / subprocess / filesystem without
touching real hardware or libraries.
This module is intentionally thin — strategy implementations + the
orchestrator live in ``builder.py``. Adding a new scenario typically only
requires writing a similar ~60-line config factory + CLI module.
"""
from __future__ import annotations
import argparse
import json
import logging
import subprocess
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Callable, Iterable, Sequence
from typing import Callable, Sequence
from e2e.fixtures.sitl_replay_builder._common import (
from e2e.fixtures.sitl_replay_builder.builder import (
DEFAULT_CLI_BIN,
run_gps_denied_replay,
write_observer_fixture,
DEFAULT_FPS,
DEFAULT_TLOG_DURATION_S,
DEFAULT_TLOG_HZ,
FixtureBuilderConfig,
OutboundMessagesProjection,
StillImagesSource,
SyntheticStationaryTlog,
build_fixtures,
)
_LOG = logging.getLogger(__name__)
DEFAULT_FPS = 1.0
DEFAULT_TLOG_DURATION_S = 120
DEFAULT_TLOG_HZ = 200
DEFAULT_FDR_KIND = "outbound_position_estimate"
@dataclass(frozen=True)
class BuilderConfig:
"""Per-invocation builder configuration."""
"""Per-invocation FT-P-01 builder configuration."""
input_dir: Path
output_dir: Path
@@ -59,261 +47,11 @@ class BuilderConfig:
fps: float = DEFAULT_FPS
tlog_duration_s: int = DEFAULT_TLOG_DURATION_S
tlog_hz: int = DEFAULT_TLOG_HZ
fdr_kind: str = DEFAULT_FDR_KIND
cli_bin: str = DEFAULT_CLI_BIN
# Step 1 — encode the still images into a 1 fps MP4
def encode_stills_to_mp4(
image_paths: Sequence[Path],
output_mp4: Path,
*,
fps: float = DEFAULT_FPS,
_video_writer_factory: Callable | None = None,
_imread: Callable | None = None,
) -> int:
"""Encode `image_paths` (in order) as an MP4 at `fps`. Returns frame count.
Raises ``FileNotFoundError`` when no image paths are supplied or when
any input image cannot be read.
The OpenCV dependencies are injected via the underscore-prefixed
parameters so unit tests can run without OpenCV being available.
"""
if not image_paths:
raise FileNotFoundError(
"encode_stills_to_mp4: image_paths is empty; nothing to encode"
)
if _video_writer_factory is None or _imread is None:
import cv2
_imread = _imread or (lambda path: cv2.imread(str(path), cv2.IMREAD_COLOR))
if _video_writer_factory is None:
_fourcc = cv2.VideoWriter_fourcc(*"mp4v")
def _video_writer_factory(out: Path, width: int, height: int):
return cv2.VideoWriter(str(out), _fourcc, fps, (width, height))
first_frame = _imread(image_paths[0])
if first_frame is None:
raise FileNotFoundError(
f"encode_stills_to_mp4: failed to read {image_paths[0]}"
)
height, width = first_frame.shape[:2]
output_mp4.parent.mkdir(parents=True, exist_ok=True)
writer = _video_writer_factory(output_mp4, width, height)
try:
writer.write(first_frame)
for path in image_paths[1:]:
frame = _imread(path)
if frame is None:
raise FileNotFoundError(
f"encode_stills_to_mp4: failed to read {path}"
)
writer.write(frame)
finally:
writer.release()
return len(image_paths)
# Step 2 — generate a synthetic stationary tlog
def generate_stationary_tlog(
output_tlog: Path,
*,
duration_s: int = DEFAULT_TLOG_DURATION_S,
hz: int = DEFAULT_TLOG_HZ,
_mavlink_writer_factory: Callable | None = None,
) -> int:
"""Write a tlog with `duration_s * hz` stationary RAW_IMU + ATTITUDE pairs.
The output is the minimum tlog content ``ReplayInputAdapter`` requires:
monotonic-timestamp RAW_IMU + ATTITUDE messages so the AZ-405 tlog
pre-validator (`AC-13`) doesn't reject the input.
The samples encode zero accel/gyro/attitude — auto-sync will refuse to
find a take-off, so callers MUST drive ``gps-denied-replay`` with an
explicit ``--time-offset-ms 0`` to bypass auto-sync.
Returns the number of message PAIRS written.
"""
if duration_s <= 0:
raise ValueError(f"duration_s must be positive; got {duration_s}")
if hz <= 0:
raise ValueError(f"hz must be positive; got {hz}")
if _mavlink_writer_factory is None:
from pymavlink import mavutil
def _mavlink_writer_factory(out: Path):
return mavutil.mavlogfile(str(out), write=True)
output_tlog.parent.mkdir(parents=True, exist_ok=True)
pairs = 0
writer = _mavlink_writer_factory(output_tlog)
try:
period_us = int(1_000_000 / hz)
total_pairs = duration_s * hz
for i in range(total_pairs):
time_us = i * period_us
writer.write(_pack_raw_imu_zero(time_us))
writer.write(_pack_attitude_zero(time_us // 1000))
pairs += 1
finally:
close = getattr(writer, "close", None)
if callable(close):
close()
return pairs
def _pack_raw_imu_zero(time_usec: int) -> bytes:
"""Pack a zero-motion RAW_IMU MAVLink frame (msg id 27).
Constructed with pymavlink's MAVLink2 packer so the produced bytes are
a wire-compatible MAVLink frame including header + CRC. Stationary
semantics: all accel/gyro/mag fields are zero except the Z accel which
carries one g (gravity, ~9.81 m/s² × 1000 in mg).
"""
from pymavlink.dialects.v20 import ardupilotmega as mavlink
packer = mavlink.MAVLink(file=None, srcSystem=1, srcComponent=1)
msg = mavlink.MAVLink_raw_imu_message(
time_usec=time_usec,
xacc=0,
yacc=0,
zacc=-9810,
xgyro=0,
ygyro=0,
zgyro=0,
xmag=0,
ymag=0,
zmag=0,
id=0,
temperature=0,
)
return msg.pack(packer)
def _pack_attitude_zero(time_boot_ms: int) -> bytes:
"""Pack a zero-motion ATTITUDE MAVLink frame (msg id 30)."""
from pymavlink.dialects.v20 import ardupilotmega as mavlink
packer = mavlink.MAVLink(file=None, srcSystem=1, srcComponent=1)
msg = mavlink.MAVLink_attitude_message(
time_boot_ms=time_boot_ms,
roll=0.0,
pitch=0.0,
yaw=0.0,
rollspeed=0.0,
pitchspeed=0.0,
yawspeed=0.0,
)
return msg.pack(packer)
# Step 3 — drive `gps-denied-replay` against the generated video+tlog
# (`run_gps_denied_replay` is re-exported from `_common.py` so b78 + b79 share one impl.)
# Step 4 — extract per-frame outbound estimates from the FDR JSONL
def parse_fdr_for_outbound_estimates(
fdr_path: Path,
*,
fdr_kind: str = DEFAULT_FDR_KIND,
lat_key: str = "lat_deg",
lon_key: str = "lon_deg",
) -> list[dict]:
"""Walk `fdr_path` (JSONL) and return outbound-estimate payloads in order.
A record contributes one entry when its ``kind`` matches `fdr_kind` AND
its payload carries both `lat_key` and `lon_key`. Other records are
silently skipped (the FDR carries many record types per the
`_docs/02_document/contracts/fdr/` schema). Malformed JSON lines raise
``ValueError`` with the line number.
"""
if not fdr_path.is_file():
raise FileNotFoundError(f"FDR JSONL not found: {fdr_path}")
out: list[dict] = []
with fdr_path.open("r", encoding="utf-8") as fp:
for line_no, line in enumerate(fp, start=1):
line = line.strip()
if not line:
continue
try:
record = json.loads(line)
except json.JSONDecodeError as exc:
raise ValueError(
f"malformed FDR JSON at {fdr_path}:{line_no}: {exc.msg}"
) from exc
if record.get("kind") != fdr_kind:
continue
payload = record.get("payload", {})
if not isinstance(payload, dict):
continue
if lat_key not in payload or lon_key not in payload:
continue
out.append(
{
"lat_deg": float(payload[lat_key]),
"lon_deg": float(payload[lon_key]),
}
)
return out
# Step 5 — write the two fixture files in the b75/b78 schema
def write_outbound_messages_fixture(
output_path: Path,
image_ids: Sequence[str],
estimates: Sequence[dict | None],
) -> None:
"""Write `outbound_messages_<fc_kind>_<host>.json`.
`image_ids` and `estimates` must have the same length. `None` entries
in `estimates` are persisted as JSON `null` (timeout markers); other
entries must carry `lat_deg`/`lon_deg`.
"""
if len(image_ids) != len(estimates):
raise ValueError(
f"length mismatch: {len(image_ids)} image_ids vs "
f"{len(estimates)} estimates"
)
messages: list[dict | None] = []
for image_id, estimate in zip(image_ids, estimates):
if estimate is None:
messages.append(None)
continue
messages.append(
{
"image_id": image_id,
"lat_deg": float(estimate["lat_deg"]),
"lon_deg": float(estimate["lon_deg"]),
}
)
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(json.dumps({"messages": messages}, indent=2))
# `write_observer_fixture` is re-exported from `_common.py` (used by both b78 + b79).
# Orchestration
def _resolve_p01_image_paths(input_dir: Path) -> list[Path]:
"""Return the AD0000NN.jpg images under `input_dir`, sorted by name."""
def resolve_p01_image_paths(input_dir: Path) -> list[Path]:
"""Return the ``AD000NN.jpg`` images under ``input_dir``, sorted by name."""
if not input_dir.is_dir():
raise FileNotFoundError(f"input dir not found: {input_dir}")
return sorted(input_dir.glob("AD??????.jpg"))
@@ -327,67 +65,25 @@ def build_p01_fixtures(
_imread: Callable | None = None,
_mavlink_writer_factory: Callable | None = None,
) -> Path:
"""End-to-end FT-P-01 fixture build. Returns the output directory.
Steps (matches the module docstring):
1. Resolve the 60 AD0000NN.jpg images under ``cfg.input_dir``.
2. Encode them at ``cfg.fps`` into ``stills.mp4`` under ``cfg.output_dir``.
3. Generate a stationary ``stationary.tlog`` under ``cfg.output_dir``.
4. Run ``gps-denied-replay`` against the pair; write FDR JSONL.
5. Project FDR outbound-estimate records into the two fixture files.
Per-frame timeout handling: if the FDR yields fewer estimates than
images, the trailing image_ids get `null` (timeout) entries. If the
FDR yields MORE estimates than images (multiple emissions per frame),
only the first ``len(image_paths)`` estimates are kept and a WARN is
logged so the operator notices the schema mismatch.
"""
image_paths = _resolve_p01_image_paths(cfg.input_dir)
"""End-to-end FT-P-01 fixture build. Returns the output directory."""
image_paths = resolve_p01_image_paths(cfg.input_dir)
if not image_paths:
raise FileNotFoundError(
f"no AD??????.jpg images found under {cfg.input_dir}"
)
raise FileNotFoundError(f"no AD??????.jpg images found under {cfg.input_dir}")
cfg.output_dir.mkdir(parents=True, exist_ok=True)
stills_mp4 = cfg.output_dir / "stills.mp4"
stationary_tlog = cfg.output_dir / "stationary.tlog"
fdr_jsonl = cfg.output_dir / "fdr.jsonl"
encode_stills_to_mp4(
image_paths, stills_mp4, fps=cfg.fps,
_video_writer_factory=_video_writer_factory, _imread=_imread,
builder_cfg = FixtureBuilderConfig(
video_source=StillImagesSource(image_paths=image_paths, fps=cfg.fps),
tlog_source=SyntheticStationaryTlog(duration_s=cfg.tlog_duration_s, hz=cfg.tlog_hz),
fdr_projection=OutboundMessagesProjection(image_ids=[p.name for p in image_paths]),
output_dir=cfg.output_dir,
fc_kind=cfg.fc_kind, host=cfg.host, cli_bin=cfg.cli_bin,
video_filename="stills.mp4", tlog_filename="stationary.tlog",
fdr_subdir=".", fdr_filename="fdr.jsonl",
)
generate_stationary_tlog(
stationary_tlog,
duration_s=cfg.tlog_duration_s,
hz=cfg.tlog_hz,
_mavlink_writer_factory=_mavlink_writer_factory,
return build_fixtures(
builder_cfg,
_runner=_runner, _video_writer_factory=_video_writer_factory,
_imread=_imread, _mavlink_writer_factory=_mavlink_writer_factory,
)
run_gps_denied_replay(
stills_mp4, stationary_tlog, fdr_jsonl,
cli_bin=cfg.cli_bin, _runner=_runner,
)
raw_estimates = parse_fdr_for_outbound_estimates(fdr_jsonl, fdr_kind=cfg.fdr_kind)
estimates: list[dict | None] = list(raw_estimates[: len(image_paths)])
if len(raw_estimates) > len(image_paths):
_LOG.warning(
"FDR carried %d outbound estimates but only %d images were pushed; "
"truncating to the per-frame count", len(raw_estimates), len(image_paths)
)
while len(estimates) < len(image_paths):
estimates.append(None)
outbound_path = cfg.output_dir / f"outbound_messages_{cfg.fc_kind}_{cfg.host}.json"
observer_path = cfg.output_dir / f"observer_{cfg.fc_kind}_{cfg.host}.json"
write_outbound_messages_fixture(
outbound_path,
image_ids=[p.name for p in image_paths],
estimates=estimates,
)
write_observer_fixture(observer_path)
return cfg.output_dir
def _main(argv: Sequence[str] | None = None) -> int:
@@ -407,12 +103,8 @@ def _main(argv: Sequence[str] | None = None) -> int:
logging.basicConfig(level=logging.INFO)
cfg = BuilderConfig(
input_dir=args.input_dir,
output_dir=args.output_dir,
fc_kind=args.fc_kind,
host=args.host,
fps=args.fps,
cli_bin=args.cli_bin,
input_dir=args.input_dir, output_dir=args.output_dir,
fc_kind=args.fc_kind, host=args.host, fps=args.fps, cli_bin=args.cli_bin,
)
build_p01_fixtures(cfg)
return 0