gps-denied-onboard/e2e/jetson/tier2-on-jetson.sh

#!/usr/bin/env bash
# Tier-2 ON-JETSON delegate. NOT invoked directly by humans — `run-tier2.sh`
# ssh-orchestrates this script onto the configured Jetson host.
#
# Responsibilities:
#   * Verify `gps-denied-onboard.service` (or the `*-asan` variant) is healthy.
#   * Spawn tegrastats + jtop parallel samplers; route their output into the
#     evidence bundle.
#   * Drive the e2e-runner image via docker compose against
#     `docker-compose.test.yml + docker-compose.tier2-bridge.yml`.
#   * Tear down samplers cleanly on EXIT / INT / TERM.
#
# Required env vars (set by run-tier2.sh):
#   RUN_ID            Run identifier (utc-stamp).
#   FC_ADAPTER        ardupilot | inav
#   VIO_STRATEGY      okvis2 | klt_ransac | vins_mono
#   BUILD_KIND        production | asan
#   SELECTOR          pytest -k expression (may be empty)
#   ENABLE_CHAMBER    0 | 1
#   JETSON_HOST       host alias used by the test for SUT identification

set -euo pipefail

: "${RUN_ID:?RUN_ID must be set by run-tier2.sh}"
: "${FC_ADAPTER:?FC_ADAPTER must be set}"
: "${VIO_STRATEGY:?VIO_STRATEGY must be set}"
: "${BUILD_KIND:=production}"
: "${SELECTOR:=}"
: "${ENABLE_CHAMBER:=0}"
: "${JETSON_HOST:=localhost}"

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
RESULTS_DIR="${REPO_ROOT}/e2e-results/run-${RUN_ID}"
EVIDENCE_DIR="${RESULTS_DIR}/evidence"

mkdir -p "${EVIDENCE_DIR}"

# AC-5: the asan build is a separate systemd unit so it can run alongside
# the production one for control/treatment comparisons.
case "${BUILD_KIND}" in
    production)
        SUT_UNIT="gps-denied-onboard.service"
        ;;
    asan)
        SUT_UNIT="gps-denied-onboard-asan.service"
        # ASan stderr stream is captured into the evidence bundle (see
        # AC-5: "stderr captured into asan-fuzz-${test_id}.log"). We tail
        # the unit's journal into the evidence file via journalctl.
        ASAN_LOG="${EVIDENCE_DIR}/asan-fuzz.log"
        ;;
    *)
        echo "[tier2-on-jetson] FATAL: unknown BUILD_KIND=${BUILD_KIND}" >&2
        exit 2
        ;;
esac

# AC-3: systemd lifecycle. Restart on demand; fail loud if it doesn't
# come back up.
echo "[tier2-on-jetson] verifying ${SUT_UNIT} is active..."
if ! systemctl is-active --quiet "${SUT_UNIT}"; then
    echo "[tier2-on-jetson] ${SUT_UNIT} is not active — restarting..." >&2
    sudo systemctl restart "${SUT_UNIT}"
    # AC-3 says "restart within ≤5 s"; we poll up to 5s + 1s safety
    # margin.
    for _ in 1 2 3 4 5 6; do
        sleep 1
        if systemctl is-active --quiet "${SUT_UNIT}"; then
            break
        fi
    done
    if ! systemctl is-active --quiet "${SUT_UNIT}"; then
        echo "[tier2-on-jetson] FATAL: ${SUT_UNIT} failed to start" >&2
        sudo systemctl status "${SUT_UNIT}" --no-pager || true
        exit 3
    fi
fi

# AC-4: tegrastats + jtop parallel capture. Output streams into the
# evidence bundle.
TEGRA_CSV="${EVIDENCE_DIR}/tegrastats-${JETSON_HOST}-${RUN_ID}.csv"
JTOP_CSV="${EVIDENCE_DIR}/jtop-${JETSON_HOST}-${RUN_ID}.csv"
TEGRA_PID=""
JTOP_PID=""
ASAN_TAIL_PID=""

if command -v tegrastats >/dev/null 2>&1; then
    # 5 Hz sampling matches the parser's expected cadence.
    tegrastats --interval 200 \
        | python3 "${SCRIPT_DIR}/tegrastats_parser.py" --out "${TEGRA_CSV}" &
    TEGRA_PID=$!
    echo "[tier2-on-jetson] tegrastats sampler pid=${TEGRA_PID} → ${TEGRA_CSV}"
else
    echo "[tier2-on-jetson] WARNING: tegrastats not in PATH — skipping that evidence channel." >&2
fi

if command -v jtop >/dev/null 2>&1; then
    python3 "${SCRIPT_DIR}/jtop_parser.py" --out "${JTOP_CSV}" --interval 1.0 &
    JTOP_PID=$!
    echo "[tier2-on-jetson] jtop sampler pid=${JTOP_PID} → ${JTOP_CSV}"
else
    echo "[tier2-on-jetson] WARNING: jtop not in PATH — skipping that evidence channel." >&2
fi

if [[ "${BUILD_KIND}" == "asan" ]]; then
    journalctl -u "${SUT_UNIT}" -f --no-pager > "${ASAN_LOG}" 2>&1 &
    ASAN_TAIL_PID=$!
    echo "[tier2-on-jetson] asan journal tail pid=${ASAN_TAIL_PID} → ${ASAN_LOG}"
fi

cleanup() {
    local rc=$?
    [[ -n "${TEGRA_PID}" ]] && kill "${TEGRA_PID}" 2>/dev/null || true
    [[ -n "${JTOP_PID}" ]] && kill "${JTOP_PID}" 2>/dev/null || true
    [[ -n "${ASAN_TAIL_PID}" ]] && kill "${ASAN_TAIL_PID}" 2>/dev/null || true
    echo "[tier2-on-jetson] cleanup complete (rc=${rc})"
    exit "${rc}"
}
trap cleanup EXIT INT TERM

# AC-1: selector parity. SELECTOR is forwarded as `-k "<expr>"` to the
# pytest inside the runner image; empty SELECTOR means "all tests".
PYTEST_ARGS=("/test-suite")
PYTEST_ARGS+=("--csv=/e2e-results/run-${RUN_ID}/report.csv")
PYTEST_ARGS+=("--csv-columns=test_id,test_name,traces_to,fc_adapter,vio_strategy,tier,started_at_utc,execution_time_ms,result,error_message,evidence_paths")
PYTEST_ARGS+=("--evidence-out=/e2e-results/run-${RUN_ID}/evidence")
PYTEST_ARGS+=("--build-kind=${BUILD_KIND}")
[[ "${ENABLE_CHAMBER}" -eq 1 ]] && PYTEST_ARGS+=("--enable-chamber")
[[ -n "${SELECTOR}" ]] && PYTEST_ARGS+=("-k" "${SELECTOR}")

(
    cd "${REPO_ROOT}/e2e/docker"
    RUN_ID="${RUN_ID}" \
    FC_ADAPTER="${FC_ADAPTER}" \
    VIO_STRATEGY="${VIO_STRATEGY}" \
    TIER="tier2-jetson" \
    JETSON_HOST="${JETSON_HOST}" \
    BUILD_KIND="${BUILD_KIND}" \
    docker compose \
        -f docker-compose.test.yml \
        -f docker-compose.tier2-bridge.yml \
        run --rm \
        -e TIER=tier2-jetson \
        -e BUILD_KIND="${BUILD_KIND}" \
        e2e-runner \
        pytest "${PYTEST_ARGS[@]}"
)

echo "[tier2-on-jetson] Suite complete. Report: ${RESULTS_DIR}/report.csv"