gps-denied-onboard/scripts/run-tests-jetson.sh

#!/usr/bin/env bash
# AZ-615: drive the Tier-2 Reality Gate e2e harness on a remote Jetson.
#
# Runs from the developer Mac. Assumes:
#   * `ssh jetson-e2e` works via key auth + ~/.ssh/config (see
#     _docs/03_implementation/jetson_harness_setup.md for one-time setup).
#   * The Jetson has docker + nvidia-container-toolkit + ≥ 30 GB free on
#     /var/lib/docker.
#
# Flow:
#   1. rsync the working tree to the Jetson under ~/gps-denied-onboard/
#      (excluding .git, LFS pointers, build artefacts).
#   2. ssh into the Jetson and `docker compose build` the e2e-runner image
#      against tests/e2e/Dockerfile.jetson.
#   3. ssh again and `docker compose up --abort-on-container-exit
#      --exit-code-from e2e-runner` so the local exit code reflects the
#      remote test verdict.
#   4. stdout / stderr stream back to the Mac terminal.
#
# Exit code propagates the docker-compose exit code (which == the
# e2e-runner container's exit code, which == pytest's verdict).

set -euo pipefail

# ----------------------------------------------------------------------
# Configuration

SSH_ALIAS="${JETSON_SSH_ALIAS:-jetson-e2e}"
# REMOTE_DIR may contain a leading '~' for convenience. rsync expands it
# server-side, but the later `bash -s <<EOF` heredoc embeds it as a
# literal string that ends up inside `cd "..."` — and bash does NOT
# expand '~' inside double quotes. To keep one variable that works in
# both contexts we resolve '~' to the remote $HOME up-front.
REMOTE_DIR="${JETSON_REMOTE_DIR:-~/gps-denied-onboard}"
COMPOSE_FILE="docker-compose.test.jetson.yml"

# Repo root regardless of where the script is invoked from.
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"

# AZ-688: the Jetson compose `include:`s ../satellite-provider/docker-compose.yml.
# That relative path must resolve identically on the Mac (where the workstation
# clones gps-denied-onboard alongside satellite-provider) and on the Jetson
# (where this script rsyncs both). REMOTE_SATPROV_DIR is computed as a sibling
# of REMOTE_DIR so the relative `../satellite-provider` works after `cd`.
SATPROV_DIR="${REPO_ROOT}/../satellite-provider"
if [ ! -d "${SATPROV_DIR}" ]; then
    echo "ERROR: ../satellite-provider not found at ${SATPROV_DIR}" >&2
    echo "       Clone the sibling repo before running the Jetson harness." >&2
    exit 67
fi
SATPROV_DIR="$(cd "${SATPROV_DIR}" && pwd)"

# .env.test (gitignored) supplies JWT_SECRET / JWT_ISSUER / JWT_AUDIENCE /
# GOOGLE_MAPS_API_KEY. The upstream satellite-provider compose interpolates
# `${VAR}` from the docker-compose shell environment, so we must source the
# file BEFORE building the heredoc.
ENV_TEST_FILE="${REPO_ROOT}/.env.test"
if [ ! -f "${ENV_TEST_FILE}" ]; then
    echo "ERROR: ${ENV_TEST_FILE} not found." >&2
    echo "       Copy .env.test.example to .env.test and fill in the JWT/GMaps vars." >&2
    echo "       See _docs/03_implementation/jetson_harness_setup.md for details." >&2
    exit 68
fi
set -o allexport
# shellcheck disable=SC1090
source "${ENV_TEST_FILE}"
set +o allexport

for var in JWT_SECRET JWT_ISSUER JWT_AUDIENCE; do
    val="${!var:-}"
    if [ -z "${val}" ]; then
        echo "ERROR: ${var} not set after sourcing ${ENV_TEST_FILE}." >&2
        echo "       The real satellite-provider fails fast at startup without all three JWT_* vars." >&2
        exit 69
    fi
done

if [ "${#JWT_SECRET}" -lt 32 ]; then
    echo "ERROR: JWT_SECRET is ${#JWT_SECRET} bytes; HMAC-SHA256 requires ≥ 32 bytes." >&2
    exit 70
fi

# AZ-777 Phase 1: the e2e-runner needs a Bearer token to call the real
# satellite-provider. If the caller didn't pre-export SATELLITE_PROVIDER_API_KEY
# (preferred for CI / repeatable runs), mint a fresh dev JWT here using the
# same JWT_SECRET / JWT_ISSUER / JWT_AUDIENCE the producer validates against.
if [ -z "${SATELLITE_PROVIDER_API_KEY:-}" ]; then
    echo "[run-tests-jetson] minting fresh dev JWT via scripts/mint_dev_jwt.py"
    if ! SATELLITE_PROVIDER_API_KEY=$(python3 "${SCRIPT_DIR}/mint_dev_jwt.py" \
            --subject e2e-runner-jetson 2>&1); then
        echo "ERROR: mint_dev_jwt.py failed:" >&2
        echo "${SATELLITE_PROVIDER_API_KEY}" >&2
        exit 71
    fi
    export SATELLITE_PROVIDER_API_KEY
fi

# Pre-quote the env vars for safe heredoc injection. `${var@Q}` would be
# cleaner but it requires bash 4.4+; macOS ships bash 3.2 and we want to
# stay portable. `printf %q` is in bash 2+.
JWT_SECRET_Q=$(printf '%q' "${JWT_SECRET}")
JWT_ISSUER_Q=$(printf '%q' "${JWT_ISSUER}")
JWT_AUDIENCE_Q=$(printf '%q' "${JWT_AUDIENCE}")
GOOGLE_MAPS_API_KEY_Q=$(printf '%q' "${GOOGLE_MAPS_API_KEY:-}")
SATELLITE_PROVIDER_API_KEY_Q=$(printf '%q' "${SATELLITE_PROVIDER_API_KEY}")

# ----------------------------------------------------------------------
# Pre-flight

if ! command -v rsync >/dev/null 2>&1; then
    echo "ERROR: rsync not on PATH — install with 'brew install rsync' or apt" >&2
    exit 64
fi

if ! ssh -o BatchMode=yes -o ConnectTimeout=5 "${SSH_ALIAS}" true 2>/dev/null; then
    cat >&2 <<EOF
ERROR: cannot reach 'ssh ${SSH_ALIAS}' non-interactively. Configure
       ~/.ssh/config + agent-based key auth per
       _docs/03_implementation/jetson_harness_setup.md.
EOF
    exit 65
fi

# Resolve any leading '~' in REMOTE_DIR against the remote $HOME so the
# value can be safely double-quoted in later heredocs.
case "${REMOTE_DIR}" in
    "~"|"~/"*)
        REMOTE_HOME="$(ssh "${SSH_ALIAS}" 'printf %s "$HOME"')"
        if [ -z "${REMOTE_HOME}" ]; then
            echo "ERROR: failed to resolve \$HOME on ${SSH_ALIAS}" >&2
            exit 66
        fi
        REMOTE_DIR="${REMOTE_HOME}${REMOTE_DIR#\~}"
        ;;
esac

# AZ-688: place satellite-provider as a sibling of REMOTE_DIR so the
# compose `include: ../satellite-provider/docker-compose.yml` resolves.
REMOTE_PARENT_DIR="$(dirname "${REMOTE_DIR}")"
REMOTE_SATPROV_DIR="${REMOTE_PARENT_DIR}/satellite-provider"

echo "[run-tests-jetson] using ssh alias: ${SSH_ALIAS}"
echo "[run-tests-jetson] remote dir:      ${REMOTE_DIR}"
echo "[run-tests-jetson] remote satprov:  ${REMOTE_SATPROV_DIR}"
echo "[run-tests-jetson] compose file:    ${COMPOSE_FILE}"

# AZ-688: ensure the dev TLS cert exists locally before rsync so the
# satellite-provider container can mount /app/certs/api.pfx on startup.
echo "[run-tests-jetson] ensure-dev-cert (local)"
bash "${SCRIPT_DIR}/ensure-dev-cert.sh"

DERKACHI_MP4="${REPO_ROOT}/_docs/00_problem/input_data/flight_derkachi/flight_derkachi.mp4"
if [[ -f "${DERKACHI_MP4}" ]]; then
    mp4_bytes=$(wc -c < "${DERKACHI_MP4}" | tr -d ' ')
    if [[ "${mp4_bytes}" -lt 1000000 ]]; then
        echo "[run-tests-jetson] ERROR: ${DERKACHI_MP4} is ${mp4_bytes} bytes — looks like a Git LFS pointer." >&2
        echo "[run-tests-jetson] Run 'git lfs pull' (or copy the real mp4) before rsyncing to Jetson." >&2
        exit 1
    fi
fi

# ----------------------------------------------------------------------
# Step 1: sync source

# Exclusions kept deliberately narrow — we want the full src/, tests/,
# _docs/, docker-compose*.yml, scripts/, pyproject.toml. We exclude:
#   * .git — huge, no value on the Jetson
#   * __pycache__ / *.pyc — host-arch bytecode, regenerated on Jetson
#   * _build / build / dist — local CMake / setuptools output trees
#   * node_modules — frontend artefacts, not needed by the harness
#   * .venv / venv — host venv, would clobber the Jetson's Python env
#   * .DS_Store — macOS metadata
#   * *.tlog / *.bin / *.engine — large fixtures that exist on Jetson
#     either via a separate fixture-sync step or are produced by the SUT
# Note on LFS-tracked fixtures (e.g. flight_derkachi.mp4): Git LFS
# pointers (134-byte text files) transfer fine, but the SUT needs the
# real binary. The convention on the Mac side is to smudge the pointer
# locally BEFORE running this script (e.g. `git lfs pull`, or copy
# from `.git/lfs/objects/<sha>/...`). rsync then transfers the actual
# bytes. If a fixture arrives as a pointer the test will fail-fast
# with "Derkachi fixture missing".
#
# Flags note: macOS ships BSD rsync, which doesn't support GNU's
# `--info=progress2`. Stick to the portable subset.
echo "[run-tests-jetson] rsync gps-denied-onboard → ${SSH_ALIAS}:${REMOTE_DIR}/"
rsync -az --delete --stats \
    --exclude=.git/ \
    --exclude='__pycache__/' \
    --exclude='*.pyc' \
    --exclude=_build/ \
    --exclude=build/ \
    --exclude=dist/ \
    --exclude=node_modules/ \
    --exclude=.venv/ \
    --exclude=venv/ \
    --exclude=.DS_Store \
    --exclude='*.engine' \
    "${REPO_ROOT}/" "${SSH_ALIAS}:${REMOTE_DIR}/"

# AZ-688: also rsync the sibling satellite-provider repo so the
# `include:` path resolves on the Jetson. .NET artefacts (bin/, obj/,
# TestResults/) are excluded; the cert dir is included so the upstream
# api container can mount /app/certs/api.pfx.
echo "[run-tests-jetson] rsync satellite-provider → ${SSH_ALIAS}:${REMOTE_SATPROV_DIR}/"
rsync -az --delete --stats \
    --exclude=.git/ \
    --exclude=bin/ \
    --exclude=obj/ \
    --exclude=TestResults/ \
    --exclude=.vs/ \
    --exclude='*.DotSettings*' \
    --exclude='*.user' \
    --exclude=logs/ \
    --exclude=Content/ \
    --exclude=tiles/ \
    --exclude=ready/ \
    --exclude=.DS_Store \
    "${SATPROV_DIR}/" "${SSH_ALIAS}:${REMOTE_SATPROV_DIR}/"

# ----------------------------------------------------------------------
# Step 2: build the e2e-runner image on the Jetson

# Dockerfile.jetson needs Tegra libs, so e2e-runner MUST be built on-device.
# satellite-provider is NOT rebuilt here: the parent-suite image now compiles
# gRPC protos (AZ-977) and protoc segfaults on arm64 inside dotnet-sdk
# (exit 139). The cached gps-denied-onboard/satellite-provider:dev image is
# used as-is until AZ-977 ships an arm64-safe build path.
echo "[run-tests-jetson] docker compose build e2e-runner (on Jetson)"
# The compose `include:` resolves the upstream env vars from the shell, so
# pass JWT_SECRET / JWT_ISSUER / JWT_AUDIENCE / GOOGLE_MAPS_API_KEY through
# the heredoc as explicit exports. (We can't rely on `ssh -o SendEnv` —
# the Jetson sshd would have to allow the matching AcceptEnv on its side.)
# shellcheck disable=SC2087  # we want the heredoc to expand on the local side
ssh "${SSH_ALIAS}" bash -s <<EOF
set -euo pipefail
export JWT_SECRET=${JWT_SECRET_Q}
export JWT_ISSUER=${JWT_ISSUER_Q}
export JWT_AUDIENCE=${JWT_AUDIENCE_Q}
export GOOGLE_MAPS_API_KEY=${GOOGLE_MAPS_API_KEY_Q}
export SATELLITE_PROVIDER_API_KEY=${SATELLITE_PROVIDER_API_KEY_Q}
cd "${REMOTE_DIR}"
docker compose -f "${COMPOSE_FILE}" build e2e-runner
EOF

# ----------------------------------------------------------------------
# Step 3: run

# `--abort-on-container-exit` plus `--exit-code-from e2e-runner` makes
# docker-compose propagate the runner's exit code, which we propagate
# back to the local terminal via `ssh` returning that code. So `bash
# scripts/run-tests-jetson.sh && echo OK` does the right thing locally.
echo "[run-tests-jetson] docker compose up e2e-runner (on Jetson)"
ssh "${SSH_ALIAS}" bash -s <<EOF
set -euo pipefail
export JWT_SECRET=${JWT_SECRET_Q}
export JWT_ISSUER=${JWT_ISSUER_Q}
export JWT_AUDIENCE=${JWT_AUDIENCE_Q}
export GOOGLE_MAPS_API_KEY=${GOOGLE_MAPS_API_KEY_Q}
export SATELLITE_PROVIDER_API_KEY=${SATELLITE_PROVIDER_API_KEY_Q}
cd "${REMOTE_DIR}"
exec docker compose -f "${COMPOSE_FILE}" up \
    --abort-on-container-exit \
    --exit-code-from e2e-runner
EOF