gps-denied-onboard/scripts/run-tests-jetson.sh

#!/usr/bin/env bash
# AZ-615: drive the Tier-2 Reality Gate e2e harness on a remote Jetson.
#
# Runs from the developer Mac. Assumes:
#   * `ssh jetson-e2e` works via key auth + ~/.ssh/config (see
#     _docs/03_implementation/jetson_harness_setup.md for one-time setup).
#   * The Jetson has docker + nvidia-container-toolkit + ≥ 30 GB free on
#     /var/lib/docker.
#
# Flow:
#   1. rsync the working tree to the Jetson under ~/gps-denied-onboard/
#      (excluding .git, LFS pointers, build artefacts).
#   2. ssh into the Jetson and `docker compose build` the e2e-runner image
#      against tests/e2e/Dockerfile.jetson.
#   3. ssh again and `docker compose up --abort-on-container-exit
#      --exit-code-from e2e-runner` so the local exit code reflects the
#      remote test verdict.
#   4. stdout / stderr stream back to the Mac terminal.
#
# Exit code propagates the docker-compose exit code (which == the
# e2e-runner container's exit code, which == pytest's verdict).

set -euo pipefail

# ----------------------------------------------------------------------
# Configuration

SSH_ALIAS="${JETSON_SSH_ALIAS:-jetson-e2e}"
# REMOTE_DIR may contain a leading '~' for convenience. rsync expands it
# server-side, but the later `bash -s <<EOF` heredoc embeds it as a
# literal string that ends up inside `cd "..."` — and bash does NOT
# expand '~' inside double quotes. To keep one variable that works in
# both contexts we resolve '~' to the remote $HOME up-front.
REMOTE_DIR="${JETSON_REMOTE_DIR:-~/gps-denied-onboard}"
COMPOSE_FILE="docker-compose.test.jetson.yml"

# Repo root regardless of where the script is invoked from.
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"

# ----------------------------------------------------------------------
# Pre-flight

if ! command -v rsync >/dev/null 2>&1; then
    echo "ERROR: rsync not on PATH — install with 'brew install rsync' or apt" >&2
    exit 64
fi

if ! ssh -o BatchMode=yes -o ConnectTimeout=5 "${SSH_ALIAS}" true 2>/dev/null; then
    cat >&2 <<EOF
ERROR: cannot reach 'ssh ${SSH_ALIAS}' non-interactively. Configure
       ~/.ssh/config + agent-based key auth per
       _docs/03_implementation/jetson_harness_setup.md.
EOF
    exit 65
fi

# Resolve any leading '~' in REMOTE_DIR against the remote $HOME so the
# value can be safely double-quoted in later heredocs.
case "${REMOTE_DIR}" in
    "~"|"~/"*)
        REMOTE_HOME="$(ssh "${SSH_ALIAS}" 'printf %s "$HOME"')"
        if [ -z "${REMOTE_HOME}" ]; then
            echo "ERROR: failed to resolve \$HOME on ${SSH_ALIAS}" >&2
            exit 66
        fi
        REMOTE_DIR="${REMOTE_HOME}${REMOTE_DIR#\~}"
        ;;
esac

echo "[run-tests-jetson] using ssh alias: ${SSH_ALIAS}"
echo "[run-tests-jetson] remote dir:      ${REMOTE_DIR}"
echo "[run-tests-jetson] compose file:    ${COMPOSE_FILE}"

# ----------------------------------------------------------------------
# Step 1: sync source

# Exclusions kept deliberately narrow — we want the full src/, tests/,
# _docs/, docker-compose*.yml, scripts/, pyproject.toml. We exclude:
#   * .git — huge, no value on the Jetson
#   * __pycache__ / *.pyc — host-arch bytecode, regenerated on Jetson
#   * _build / build / dist — local CMake / setuptools output trees
#   * node_modules — frontend artefacts, not needed by the harness
#   * .venv / venv — host venv, would clobber the Jetson's Python env
#   * .DS_Store — macOS metadata
#   * *.tlog / *.bin / *.engine — large fixtures that exist on Jetson
#     either via a separate fixture-sync step or are produced by the SUT
# Note on LFS-tracked fixtures (e.g. flight_derkachi.mp4): Git LFS
# pointers (134-byte text files) transfer fine, but the SUT needs the
# real binary. The convention on the Mac side is to smudge the pointer
# locally BEFORE running this script (e.g. `git lfs pull`, or copy
# from `.git/lfs/objects/<sha>/...`). rsync then transfers the actual
# bytes. If a fixture arrives as a pointer the test will fail-fast
# with "Derkachi fixture missing".
#
# Flags note: macOS ships BSD rsync, which doesn't support GNU's
# `--info=progress2`. Stick to the portable subset.
echo "[run-tests-jetson] rsync → ${SSH_ALIAS}:${REMOTE_DIR}/"
rsync -az --delete --stats \
    --exclude=.git/ \
    --exclude='__pycache__/' \
    --exclude='*.pyc' \
    --exclude=_build/ \
    --exclude=build/ \
    --exclude=dist/ \
    --exclude=node_modules/ \
    --exclude=.venv/ \
    --exclude=venv/ \
    --exclude=.DS_Store \
    --exclude='*.engine' \
    "${REPO_ROOT}/" "${SSH_ALIAS}:${REMOTE_DIR}/"

# ----------------------------------------------------------------------
# Step 2: build the e2e-runner image on the Jetson

# The image MUST be built on the Jetson — see Dockerfile.jetson comment
# about Tegra-specific libs.
echo "[run-tests-jetson] docker compose build e2e-runner (on Jetson)"
# shellcheck disable=SC2087  # we want the heredoc to expand on the local side
ssh "${SSH_ALIAS}" bash -s <<EOF
set -euo pipefail
cd "${REMOTE_DIR}"
docker compose -f "${COMPOSE_FILE}" build e2e-runner
EOF

# ----------------------------------------------------------------------
# Step 3: run

# `--abort-on-container-exit` plus `--exit-code-from e2e-runner` makes
# docker-compose propagate the runner's exit code, which we propagate
# back to the local terminal via `ssh` returning that code. So `bash
# scripts/run-tests-jetson.sh && echo OK` does the right thing locally.
echo "[run-tests-jetson] docker compose up e2e-runner (on Jetson)"
ssh "${SSH_ALIAS}" bash -s <<EOF
set -euo pipefail
cd "${REMOTE_DIR}"
exec docker compose -f "${COMPOSE_FILE}" up \
    --abort-on-container-exit \
    --exit-code-from e2e-runner
EOF