gps-denied-onboard/e2e/jetson/run-tier2.sh

#!/usr/bin/env bash
# Tier-2 Jetson hardware-loop entrypoint (orchestrator).
#
# This script runs FROM a control host (typically x86) and ssh-orchestrates
# the on-Jetson half (`tier2-on-jetson.sh`). When invoked on the Jetson
# itself (uname -m == aarch64 AND TIER2_HOST=localhost), it delegates
# directly without going through ssh.
#
# Usage:
#   ./run-tier2.sh \
#       --fc-adapter <ardupilot|inav> \
#       --vio-strategy <okvis2|klt_ransac|vins_mono> \
#       [-k <pytest selector>] \
#       [--build-kind <production|asan>] \
#       [--duration <5min|8h>] \
#       [--enable-chamber] \
#       [--reflash] \
#       [--dry-run]
#
# Required env vars (when TIER2_HOST != localhost):
#   TIER2_HOST       Jetson hostname or IP
#   TIER2_USER       SSH user on the Jetson
#   TIER2_KEY_PATH   Path to the SSH private key
#
# Pre-requisites verified at startup:
#   * The Jetson is provisioned per `_docs/02_document/tests/environment.md`
#     § Execution instructions — Tier-2 (JetPack 6.2, CUDA, TensorRT 10.3,
#     cuDNN).
#   * `gps-denied-onboard.service` (or `gps-denied-onboard-asan.service`
#     for --build-kind=asan) is installed via systemd. `tier2.service` is
#     the template.
#   * SITLs + mock + listener + runner reachable on the same network via
#     `docker compose -f e2e/docker/docker-compose.test.yml
#                  -f e2e/docker/docker-compose.tier2-bridge.yml up ...`
#     on a paired x86 host (same as Tier-1's `docker-compose.test.yml`
#     network).
#
# Outputs the same CSV format as Tier-1 to
#   ./e2e-results/run-${RUN_ID}/report.csv
# plus the per-sample tegrastats + jtop CSVs in the evidence bundle.

set -euo pipefail

FC_ADAPTER=""
VIO_STRATEGY=""
SELECTOR=""
BUILD_KIND="production"
DURATION="5min"
ENABLE_CHAMBER=0
RUN_REFLASH=0
DRY_RUN=0

usage() {
    grep -E '^# ' "$0" | sed 's/^# //' >&2
    exit 1
}

while [[ $# -gt 0 ]]; do
    case "$1" in
        --fc-adapter) FC_ADAPTER="$2"; shift 2 ;;
        --vio-strategy) VIO_STRATEGY="$2"; shift 2 ;;
        -k|--selector) SELECTOR="$2"; shift 2 ;;
        --build-kind) BUILD_KIND="$2"; shift 2 ;;
        --duration) DURATION="$2"; shift 2 ;;
        --enable-chamber) ENABLE_CHAMBER=1; shift ;;
        --reflash) RUN_REFLASH=1; shift ;;
        --dry-run) DRY_RUN=1; shift ;;
        -h|--help) usage ;;
        *) echo "Unknown arg: $1" >&2; usage ;;
    esac
done

if [[ -z "$FC_ADAPTER" || -z "$VIO_STRATEGY" ]]; then
    echo "ERROR: --fc-adapter and --vio-strategy are required" >&2
    usage
fi

case "$FC_ADAPTER" in
    ardupilot|inav) ;;
    *) echo "ERROR: --fc-adapter must be ardupilot or inav (got: $FC_ADAPTER)" >&2; exit 2 ;;
esac

case "$VIO_STRATEGY" in
    okvis2|klt_ransac|vins_mono) ;;
    *) echo "ERROR: --vio-strategy must be okvis2 | klt_ransac | vins_mono (got: $VIO_STRATEGY)" >&2; exit 2 ;;
esac

case "$BUILD_KIND" in
    production|asan) ;;
    *) echo "ERROR: --build-kind must be production or asan (got: $BUILD_KIND)" >&2; exit 2 ;;
esac

# AC-6 (image-flash gating). Even when --reflash is requested, refuse to
# proceed unless the operator has acknowledged via TIER2_REFLASH_ACK=1.
# This is a two-key gate so a stray flag flip in CI cannot accidentally
# re-provision a development board.
if [[ "${RUN_REFLASH}" -eq 1 ]]; then
    if [[ "${TIER2_REFLASH_ACK:-0}" != "1" ]]; then
        echo "ERROR: --reflash requires TIER2_REFLASH_ACK=1 in the env" >&2
        echo "       This is a destructive operation; set the ack to" >&2
        echo "       confirm you intend to re-flash the Jetson via" >&2
        echo "       nvidia-sdkmanager-cli." >&2
        exit 4
    fi
fi

# RUN_ID — caller may set; default is utc-stamp + adapter pair.
: "${RUN_ID:=tier2-$(date -u +%Y%m%dT%H%M%SZ)-${FC_ADAPTER}-${VIO_STRATEGY}}"

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"

# ---------------------------------------------------------------------------
# Determine mode:
#   * local mode  — run on the Jetson itself; no ssh wrapper.
#     Triggered when TIER2_HOST=localhost OR is unset on an aarch64 host.
#   * remote mode — orchestrator: ssh into TIER2_HOST and execute the
#     on-Jetson delegate there.
# ---------------------------------------------------------------------------
TIER2_HOST="${TIER2_HOST:-}"
if [[ -z "${TIER2_HOST}" ]]; then
    if [[ "$(uname -m)" == "aarch64" ]]; then
        TIER2_HOST="localhost"
    else
        echo "ERROR: TIER2_HOST must be set when running from a non-Jetson host" >&2
        echo "       (uname -m is $(uname -m); this script is not running on a Jetson)" >&2
        exit 5
    fi
fi

echo "[tier2] RUN_ID=${RUN_ID}"
echo "[tier2] FC_ADAPTER=${FC_ADAPTER}  VIO_STRATEGY=${VIO_STRATEGY}  BUILD_KIND=${BUILD_KIND}"
echo "[tier2] SELECTOR='${SELECTOR}'  DURATION=${DURATION}  ENABLE_CHAMBER=${ENABLE_CHAMBER}"
echo "[tier2] TIER2_HOST=${TIER2_HOST}"

# ---------------------------------------------------------------------------
# Build the ssh command prefix for the orchestrator mode.
# ---------------------------------------------------------------------------
SSH_CMD=""
if [[ "${TIER2_HOST}" != "localhost" ]]; then
    : "${TIER2_USER:?TIER2_USER must be set for remote orchestrator mode}"
    : "${TIER2_KEY_PATH:?TIER2_KEY_PATH must be set for remote orchestrator mode}"
    if [[ ! -f "${TIER2_KEY_PATH}" ]]; then
        echo "ERROR: TIER2_KEY_PATH does not point at a real file: ${TIER2_KEY_PATH}" >&2
        exit 6
    fi
    SSH_CMD="ssh -o StrictHostKeyChecking=accept-new -i ${TIER2_KEY_PATH} ${TIER2_USER}@${TIER2_HOST}"
fi

# ---------------------------------------------------------------------------
# AC-2: idempotent provisioning. apt update + install is idempotent on
# its own; we just gate it behind a `--reflash` flag because re-running
# it on every test invocation is needlessly slow.
# ---------------------------------------------------------------------------
provision_jetson() {
    local PROVISION_CMD
    PROVISION_CMD="set -eu;
        if ! dpkg -s python3-pip >/dev/null 2>&1; then
            sudo apt-get update;
            sudo apt-get install -y --no-install-recommends \
                python3-pip docker.io openssh-client iproute2;
        fi"

    if [[ "${TIER2_HOST}" == "localhost" ]]; then
        bash -c "${PROVISION_CMD}"
    else
        # shellcheck disable=SC2086
        ${SSH_CMD} "${PROVISION_CMD}"
    fi
}

# ---------------------------------------------------------------------------
# AC-6: reflash via NVIDIA's sdkmanager-cli. This is the destructive
# path; only runs when --reflash AND TIER2_REFLASH_ACK=1 are BOTH set.
# ---------------------------------------------------------------------------
reflash_jetson() {
    local FLASH_CMD
    FLASH_CMD="set -eu;
        if ! command -v nvidia-sdkmanager-cli >/dev/null 2>&1; then
            echo 'ERROR: nvidia-sdkmanager-cli not installed on Jetson' >&2
            exit 7
        fi
        echo '[tier2] re-flashing JetPack image via nvidia-sdkmanager-cli...' >&2
        nvidia-sdkmanager-cli flash --target-spec jetson-orin-nano-super"

    if [[ "${TIER2_HOST}" == "localhost" ]]; then
        bash -c "${FLASH_CMD}"
    else
        # shellcheck disable=SC2086
        ${SSH_CMD} "${FLASH_CMD}"
    fi
}

# ---------------------------------------------------------------------------
# Execute the on-Jetson delegate.
# ---------------------------------------------------------------------------
ENV_PREFIX=(
    "RUN_ID=${RUN_ID}"
    "FC_ADAPTER=${FC_ADAPTER}"
    "VIO_STRATEGY=${VIO_STRATEGY}"
    "BUILD_KIND=${BUILD_KIND}"
    "SELECTOR=${SELECTOR}"
    "ENABLE_CHAMBER=${ENABLE_CHAMBER}"
    "JETSON_HOST=${TIER2_HOST}"
)

if [[ "${TIER2_HOST}" == "localhost" ]]; then
    DELEGATE_CMD=(env "${ENV_PREFIX[@]}" "${SCRIPT_DIR}/tier2-on-jetson.sh")
else
    # Remote mode: rsync the e2e/ tree onto the Jetson and run the
    # delegate over ssh. We mirror the repo to /opt/azaion-e2e/ on the
    # Jetson; subsequent invocations are incremental via rsync's default
    # delta-transfer.
    REMOTE_REPO="/opt/azaion-e2e"
    RSYNC_CMD="rsync -az --delete -e 'ssh -o StrictHostKeyChecking=accept-new -i ${TIER2_KEY_PATH}' ${REPO_ROOT}/e2e/ ${TIER2_USER}@${TIER2_HOST}:${REMOTE_REPO}/e2e/"
    DELEGATE_CMD=(
        bash -c
        "${RSYNC_CMD} && ${SSH_CMD} \"env $(printf '%q ' "${ENV_PREFIX[@]}")${REMOTE_REPO}/e2e/jetson/tier2-on-jetson.sh\""
    )
fi

if [[ "${DRY_RUN}" -eq 1 ]]; then
    echo "[tier2] --dry-run: showing actions that would execute, then exiting."
    echo "[tier2]   provision: ${SSH_CMD:-(local)} apt-get install -y python3-pip docker.io openssh-client iproute2"
    if [[ "${RUN_REFLASH}" -eq 1 ]]; then
        echo "[tier2]   reflash:   ${SSH_CMD:-(local)} nvidia-sdkmanager-cli flash --target-spec jetson-orin-nano-super"
    fi
    echo "[tier2]   delegate:  ${DELEGATE_CMD[*]}"
    exit 0
fi

provision_jetson
[[ "${RUN_REFLASH}" -eq 1 ]] && reflash_jetson

"${DELEGATE_CMD[@]}"

echo "[tier2] Suite complete. RUN_ID=${RUN_ID}"