gps-denied-onboard/e2e/jetson/run-tier2.sh

#!/usr/bin/env bash
# Tier-2 Jetson hardware-loop entrypoint.
#
# Usage:
#   ./run-tier2.sh --fc-adapter <ardupilot|inav> --vio-strategy <okvis2|klt_ransac> [--duration <5min|8h>] [--enable-chamber]
#
# Pre-requisites (verified at startup):
#   * The Jetson is provisioned per `_docs/02_document/tests/environment.md`
#     § Execution instructions — Tier-2 (JetPack 6.2, CUDA, TensorRT 10.3, cuDNN).
#   * `gps-denied-onboard.service` is installed via systemd
#     (`tier2.service` is the template; operator copies it to /etc/systemd/system).
#   * SITLs + mock + listener + runner reachable on the same network via
#     `docker compose -f e2e/docker/docker-compose.test.yml -f e2e/docker/docker-compose.tier2-bridge.yml up ...`
#     on a paired x86 host. (Same-Jetson SITL is also supported — set JETSON_HOST=localhost.)
#
# Outputs the same CSV format as Tier-1 to ./e2e-results/run-${RUN_ID}/report.csv
# plus the per-sample tegrastats + jtop CSVs in the evidence bundle.

set -euo pipefail

FC_ADAPTER=""
VIO_STRATEGY=""
DURATION="5min"
ENABLE_CHAMBER=0
JETSON_HOST_OVERRIDE=""

usage() {
    grep -E '^# ' "$0" | sed 's/^# //'
    exit 1
}

while [[ $# -gt 0 ]]; do
    case "$1" in
        --fc-adapter) FC_ADAPTER="$2"; shift 2 ;;
        --vio-strategy) VIO_STRATEGY="$2"; shift 2 ;;
        --duration) DURATION="$2"; shift 2 ;;
        --enable-chamber) ENABLE_CHAMBER=1; shift ;;
        --jetson-host) JETSON_HOST_OVERRIDE="$2"; shift 2 ;;
        -h|--help) usage ;;
        *) echo "Unknown arg: $1" >&2; usage ;;
    esac
done

if [[ -z "$FC_ADAPTER" || -z "$VIO_STRATEGY" ]]; then
    echo "ERROR: --fc-adapter and --vio-strategy are required" >&2
    usage
fi

case "$FC_ADAPTER" in
    ardupilot|inav) ;;
    *) echo "ERROR: --fc-adapter must be ardupilot or inav (got: $FC_ADAPTER)" >&2; exit 2 ;;
esac

case "$VIO_STRATEGY" in
    okvis2|klt_ransac|vins_mono) ;;
    *) echo "ERROR: --vio-strategy must be okvis2 | klt_ransac | vins_mono (got: $VIO_STRATEGY)" >&2; exit 2 ;;
esac

# RUN_ID — caller may set; default is utc-stamp + adapter pair.
: "${RUN_ID:=tier2-$(date -u +%Y%m%dT%H%M%SZ)-${FC_ADAPTER}-${VIO_STRATEGY}}"

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
RESULTS_DIR="${REPO_ROOT}/e2e-results/run-${RUN_ID}"
EVIDENCE_DIR="${RESULTS_DIR}/evidence"

mkdir -p "${EVIDENCE_DIR}"

echo "[tier2] RUN_ID=${RUN_ID}"
echo "[tier2] FC_ADAPTER=${FC_ADAPTER}  VIO_STRATEGY=${VIO_STRATEGY}  DURATION=${DURATION}"
echo "[tier2] RESULTS_DIR=${RESULTS_DIR}"

# ---------------------------------------------------------------------------
# Pre-flight: confirm the SUT systemd unit is healthy.
# ---------------------------------------------------------------------------
if ! systemctl is-active --quiet gps-denied-onboard.service; then
    echo "[tier2] gps-denied-onboard.service is not active — attempting restart..." >&2
    sudo systemctl restart gps-denied-onboard.service
    sleep 3
    if ! systemctl is-active --quiet gps-denied-onboard.service; then
        echo "[tier2] FATAL: gps-denied-onboard.service failed to start" >&2
        sudo systemctl status gps-denied-onboard.service --no-pager || true
        exit 3
    fi
fi

# ---------------------------------------------------------------------------
# Start tegrastats + jtop background samplers (evidence bundle inputs).
# ---------------------------------------------------------------------------
TEGRA_CSV="${EVIDENCE_DIR}/tegrastats.csv"
JTOP_CSV="${EVIDENCE_DIR}/jtop.csv"

# tegrastats emits at 5 Hz by default; parser converts to per-sample CSV rows.
if command -v tegrastats >/dev/null 2>&1; then
    tegrastats --interval 200 \
        | python3 "${SCRIPT_DIR}/tegrastats_parser.py" --out "${TEGRA_CSV}" &
    TEGRA_PID=$!
else
    echo "[tier2] WARNING: tegrastats not in PATH — skipping that evidence channel." >&2
    TEGRA_PID=
fi

if command -v jtop >/dev/null 2>&1; then
    python3 "${SCRIPT_DIR}/jtop_parser.py" --out "${JTOP_CSV}" --interval 1.0 &
    JTOP_PID=$!
else
    echo "[tier2] WARNING: jtop not in PATH — skipping that evidence channel." >&2
    JTOP_PID=
fi

cleanup() {
    local rc=$?
    [[ -n "${TEGRA_PID:-}" ]] && kill "${TEGRA_PID}" 2>/dev/null || true
    [[ -n "${JTOP_PID:-}" ]] && kill "${JTOP_PID}" 2>/dev/null || true
    echo "[tier2] cleanup complete (rc=${rc})"
    exit "${rc}"
}
trap cleanup EXIT INT TERM

# ---------------------------------------------------------------------------
# Run the e2e suite — the runner image is the SAME as Tier-1; only TIER differs.
# ---------------------------------------------------------------------------
JETSON_HOST_ARG="${JETSON_HOST_OVERRIDE:-localhost}"
CHAMBER_ARG=()
[[ "${ENABLE_CHAMBER}" -eq 1 ]] && CHAMBER_ARG=("--enable-chamber")

(
    cd "${REPO_ROOT}/e2e/docker"
    RUN_ID="${RUN_ID}" \
    FC_ADAPTER="${FC_ADAPTER}" \
    VIO_STRATEGY="${VIO_STRATEGY}" \
    TIER="tier2-jetson" \
    JETSON_HOST="${JETSON_HOST_ARG}" \
    docker compose \
        -f docker-compose.test.yml \
        -f docker-compose.tier2-bridge.yml \
        run --rm \
        -e TIER=tier2-jetson \
        e2e-runner \
        pytest /test-suite \
            --csv="/e2e-results/run-${RUN_ID}/report.csv" \
            --csv-columns="test_id,test_name,traces_to,fc_adapter,vio_strategy,tier,started_at_utc,execution_time_ms,result,error_message,evidence_paths" \
            --evidence-out="/e2e-results/run-${RUN_ID}/evidence" \
            --build-kind=production \
            "${CHAMBER_ARG[@]}"
)

echo "[tier2] Suite complete. Report: ${RESULTS_DIR}/report.csv"