gps-denied-onboard/scripts/start-services.sh

#!/usr/bin/env bash
# GPS-Denied Onboard — start the service stack.
#
# Sources environment from .env (project root) and brings up the
# appropriate compose file. Waits for every service that declares a
# HEALTHCHECK to report `healthy`. Emits AZAION_UPDATE_EVENT to journald
# when run on a systemd host (matches the suite-wide audit chain — see
# observability.md § Deploy Audit).
#
# Refuses to start the airborne stack when /run/azaion/in-flight is set —
# the FC owns flight; restarting the companion mid-flight would race the
# in-flight failsafe (deployment_procedures.md § Deployment Strategy).
#
# Usage:
#   scripts/start-services.sh [--target dev|airborne|operator-workstation]
#                             [--compose-file <path>]
#                             [--wait-secs N] [--force] [--help]
#
# Defaults:
#   target       = dev
#   compose-file = docker-compose.yml
#   wait-secs    = 120
#
# Exit codes:
#   0  all services healthy
#   64 missing prerequisite
#   65 SSH unreachable when DEPLOY_HOST is set
#   68 refused — /run/azaion/in-flight is set and --force was not passed
#   69 timed out waiting for healthchecks

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"

usage() {
    cat <<'EOF'
GPS-Denied Onboard — start the service stack.

Sources environment from .env (project root) and brings up the
appropriate compose file. Waits for every service that declares a
HEALTHCHECK to report `healthy`. Emits AZAION_UPDATE_EVENT to journald
when run on a systemd host (matches the suite-wide audit chain — see
observability.md § Deploy Audit).

Refuses to start the airborne stack when /run/azaion/in-flight is set —
the FC owns flight; restarting the companion mid-flight would race the
in-flight failsafe (deployment_procedures.md § Deployment Strategy).

Usage:
  scripts/start-services.sh [--target dev|airborne|operator-workstation]
                            [--compose-file <path>]
                            [--wait-secs N] [--force] [--help]

Defaults:
  target       = dev
  compose-file = docker-compose.yml
  wait-secs    = 120

Exit codes:
  0  all services healthy
  64 missing prerequisite
  65 SSH unreachable when DEPLOY_HOST is set
  68 refused — /run/azaion/in-flight is set and --force was not passed
  69 timed out waiting for healthchecks
EOF
    exit 0
}

TARGET="dev"
COMPOSE_FILE=""
WAIT_SECS=120
FORCE=0

while [ $# -gt 0 ]; do
    case "$1" in
        --target)       TARGET="$2"; shift 2 ;;
        --compose-file) COMPOSE_FILE="$2"; shift 2 ;;
        --wait-secs)    WAIT_SECS="$2"; shift 2 ;;
        --force)        FORCE=1; shift ;;
        --help|-h)      usage ;;
        *) echo "ERROR: unknown argument: $1" >&2; usage ;;
    esac
done

if [ -f "${REPO_ROOT}/.env" ]; then
    set -a
    # shellcheck disable=SC1091
    . "${REPO_ROOT}/.env"
    set +a
fi

if [ -z "${COMPOSE_FILE}" ]; then
    case "${TARGET}" in
        dev)                  COMPOSE_FILE="docker-compose.yml" ;;
        airborne)             COMPOSE_FILE="${AIRBORNE_COMPOSE_FILE:-/etc/gps-denied/docker-compose.airborne.yml}" ;;
        operator-workstation) COMPOSE_FILE="docker-compose.yml" ;;
        *) echo "ERROR: invalid --target: ${TARGET}" >&2; exit 64 ;;
    esac
fi

remote_exec=""
if [ -n "${DEPLOY_HOST:-}" ]; then
    if ! ssh -o BatchMode=yes -o ConnectTimeout=5 "${DEPLOY_HOST}" true 2>/dev/null; then
        echo "ERROR: cannot reach 'ssh ${DEPLOY_HOST}' non-interactively." >&2
        exit 65
    fi
    remote_exec="ssh ${DEPLOY_HOST}"
fi

if [ "${TARGET}" = "airborne" ]; then
    if ${remote_exec} test -e /run/azaion/in-flight 2>/dev/null; then
        if [ "${FORCE}" = "1" ]; then
            echo "WARNING: /run/azaion/in-flight is set but --force was passed —" >&2
            echo "         continuing. This will race the FC in-flight failsafe." >&2
        else
            echo "ERROR: /run/azaion/in-flight is set on the target host." >&2
            echo "       Refusing to (re)start the airborne stack mid-flight." >&2
            echo "       Wait for the flight to end, or pass --force to override." >&2
            exit 68
        fi
    fi
fi

if ! ${remote_exec} command -v docker >/dev/null 2>&1; then
    echo "ERROR: docker not found on target" >&2
    exit 64
fi

echo "[start-services] target:       ${TARGET}"
echo "[start-services] compose-file: ${COMPOSE_FILE}"
echo "[start-services] wait:         ${WAIT_SECS}s"

echo "[start-services] docker compose up -d --remove-orphans"
${remote_exec} docker compose -f "${COMPOSE_FILE}" up -d --remove-orphans

echo "[start-services] waiting for services to reach 'healthy' state"
deadline=$(( $(date +%s) + WAIT_SECS ))
while :; do
    now=$(date +%s)
    if [ "${now}" -ge "${deadline}" ]; then
        echo "ERROR: timed out after ${WAIT_SECS}s waiting for healthchecks" >&2
        ${remote_exec} docker compose -f "${COMPOSE_FILE}" ps
        exit 69
    fi
    unhealthy_or_starting=$(${remote_exec} docker compose -f "${COMPOSE_FILE}" ps \
        --format '{{.Service}} {{.Health}}' 2>/dev/null \
        | awk '$2 != "" && $2 != "healthy" { print }' \
        || true)
    if [ -z "${unhealthy_or_starting}" ]; then
        break
    fi
    sleep 3
done

# Audit event — suite-mandated AZAION_UPDATE_EVENT (observability.md).
# Skip on macOS / non-systemd dev hosts (logger -t still works locally).
revision="${AZAION_REVISION:-unknown}"
if ${remote_exec} command -v logger >/dev/null 2>&1; then
    ${remote_exec} logger -t gps-denied-onboard -p user.notice \
        "AZAION_UPDATE_EVENT service=gps-denied-onboard target=${TARGET} revision=${revision} action=started outcome=success" \
        || true
fi

echo "[start-services] OK"
${remote_exec} docker compose -f "${COMPOSE_FILE}" ps