mirror of
https://github.com/azaion/gps-denied-onboard.git
synced 2026-06-21 09:41:13 +00:00
[autodev] Update configuration and documentation for cycle-1
ci/woodpecker/push/02-build-push Pipeline failed
ci/woodpecker/push/02-build-push Pipeline failed
- Enhanced `.env.example` with detailed CMake build flags and replay-mode strategy flags for development and CI environments. - Updated `.gitignore` to include a new deploy rollback bookmark. - Revised `_docs/_autodev_state.md` to reflect the current task status and steps. - Added new lessons to `_docs/LESSONS.md` regarding testing and architectural improvements. - Documented changes in `_docs/02_document/deployment/ci_cd_pipeline.md` to reflect the relaxed OpenCV version pin. - Updated test data documentation in `_docs/02_document/tests/test-data.md` to clarify fixture usage and paths. This commit continues the cycle-1 documentation sync and addresses various configuration updates for improved clarity and functionality.
This commit is contained in:
Oleksandr Bezdieniezhnykh
Executable
+182
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env bash
|
||||
# GPS-Denied Onboard — main deployment orchestrator.
|
||||
#
|
||||
# Runs: pull-images → flight-state-check → stop-services → start-services
|
||||
# → health-check. Honours the /run/azaion/in-flight flag on
|
||||
# airborne targets; refuses to restart mid-flight unless --force is
|
||||
# passed. See deployment_procedures.md § Production Deployment.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/deploy.sh [--target dev|airborne|operator-workstation]
|
||||
# [--branch <branch>] [--arch <arch>]
|
||||
# [--compose-file <path>]
|
||||
# [--wait-secs N]
|
||||
# [--rollback] [--force] [--help]
|
||||
#
|
||||
# Defaults:
|
||||
# target = operator-workstation
|
||||
# branch = main
|
||||
# arch = arm
|
||||
# wait-secs = 120
|
||||
#
|
||||
# --rollback restores the image set saved by the most recent
|
||||
# stop-services.sh run (.previous-tags.env at the repo root).
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 deploy or rollback succeeded
|
||||
# 64 missing prerequisite or invalid argument
|
||||
# 65 SSH unreachable when DEPLOY_HOST is set
|
||||
# 66 registry login failed (forwarded from pull-images.sh)
|
||||
# 67 image pull failed (forwarded from pull-images.sh)
|
||||
# 68 refused — /run/azaion/in-flight is set and --force was not passed
|
||||
# 69 timed out waiting for healthchecks (forwarded from start-services.sh)
|
||||
# 70 rollback requested but .previous-tags.env not found
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
GPS-Denied Onboard — main deployment orchestrator.
|
||||
|
||||
Runs: pull-images → flight-state-check → stop-services → start-services
|
||||
→ health-check. Honours the /run/azaion/in-flight flag on
|
||||
airborne targets; refuses to restart mid-flight unless --force is
|
||||
passed. See deployment_procedures.md § Production Deployment.
|
||||
|
||||
Usage:
|
||||
scripts/deploy.sh [--target dev|airborne|operator-workstation]
|
||||
[--branch <branch>] [--arch <arch>]
|
||||
[--compose-file <path>]
|
||||
[--wait-secs N]
|
||||
[--rollback] [--force] [--help]
|
||||
|
||||
Defaults:
|
||||
target = operator-workstation
|
||||
branch = main
|
||||
arch = arm
|
||||
wait-secs = 120
|
||||
|
||||
--rollback restores the image set saved by the most recent
|
||||
stop-services.sh run (.previous-tags.env at the repo root).
|
||||
|
||||
Exit codes:
|
||||
0 deploy or rollback succeeded
|
||||
64 missing prerequisite or invalid argument
|
||||
65 SSH unreachable when DEPLOY_HOST is set
|
||||
66 registry login failed (forwarded from pull-images.sh)
|
||||
67 image pull failed (forwarded from pull-images.sh)
|
||||
68 refused — /run/azaion/in-flight is set and --force was not passed
|
||||
69 timed out waiting for healthchecks (forwarded from start-services.sh)
|
||||
70 rollback requested but .previous-tags.env not found
|
||||
EOF
|
||||
exit 0
|
||||
}
|
||||
|
||||
TARGET="operator-workstation"
|
||||
BRANCH="${BRANCH:-main}"
|
||||
ARCH="${ARCH:-arm}"
|
||||
COMPOSE_FILE=""
|
||||
WAIT_SECS="${WAIT_SECS:-120}"
|
||||
ROLLBACK=0
|
||||
FORCE=0
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--target) TARGET="$2"; shift 2 ;;
|
||||
--branch) BRANCH="$2"; shift 2 ;;
|
||||
--arch) ARCH="$2"; shift 2 ;;
|
||||
--compose-file) COMPOSE_FILE="$2"; shift 2 ;;
|
||||
--wait-secs) WAIT_SECS="$2"; shift 2 ;;
|
||||
--rollback) ROLLBACK=1; shift ;;
|
||||
--force) FORCE=1; shift ;;
|
||||
--help|-h) usage ;;
|
||||
*) echo "ERROR: unknown argument: $1" >&2; usage ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -f "${REPO_ROOT}/.env" ]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
. "${REPO_ROOT}/.env"
|
||||
set +a
|
||||
fi
|
||||
|
||||
case "${TARGET}" in
|
||||
dev|airborne|operator-workstation) ;;
|
||||
*) echo "ERROR: invalid --target: ${TARGET}" >&2; exit 64 ;;
|
||||
esac
|
||||
|
||||
PREV_TAGS_FILE="${REPO_ROOT}/.previous-tags.env"
|
||||
|
||||
# Pass-through args common to pull/start/stop/health.
|
||||
common_args=( --target "${TARGET}" )
|
||||
[ -n "${COMPOSE_FILE}" ] && common_args+=( --compose-file "${COMPOSE_FILE}" )
|
||||
|
||||
force_args=()
|
||||
[ "${FORCE}" = "1" ] && force_args+=( --force )
|
||||
|
||||
echo "═══════════════════════════════════════════════════"
|
||||
echo " GPS-Denied Onboard — deploy"
|
||||
echo "═══════════════════════════════════════════════════"
|
||||
echo " target: ${TARGET}"
|
||||
echo " branch: ${BRANCH}"
|
||||
echo " arch: ${ARCH}"
|
||||
echo " rollback: ${ROLLBACK}"
|
||||
if [ -n "${DEPLOY_HOST:-}" ]; then
|
||||
echo " remote: ${DEPLOY_HOST}"
|
||||
fi
|
||||
echo "═══════════════════════════════════════════════════"
|
||||
|
||||
if [ "${ROLLBACK}" = "1" ]; then
|
||||
if [ ! -f "${PREV_TAGS_FILE}" ]; then
|
||||
echo "ERROR: --rollback requested but ${PREV_TAGS_FILE} not found." >&2
|
||||
echo " Rollback requires a prior stop-services.sh run." >&2
|
||||
exit 70
|
||||
fi
|
||||
echo "[deploy] rollback: restoring image set from ${PREV_TAGS_FILE}"
|
||||
cat "${PREV_TAGS_FILE}"
|
||||
echo
|
||||
echo "[deploy] re-pulling the saved digests via docker pull"
|
||||
set +e
|
||||
grep -E '^PREV_[A-Z0-9_]+_IMAGE=' "${PREV_TAGS_FILE}" | while IFS='=' read -r _ image; do
|
||||
${DEPLOY_HOST:+ssh ${DEPLOY_HOST}} docker pull "${image}"
|
||||
done
|
||||
rc=$?
|
||||
set -e
|
||||
if [ "${rc}" != "0" ]; then
|
||||
echo "ERROR: rollback image pull failed (rc=${rc})" >&2
|
||||
exit "${rc}"
|
||||
fi
|
||||
else
|
||||
echo "[deploy] step 1/4 — pull-images"
|
||||
"${SCRIPT_DIR}/pull-images.sh" \
|
||||
--target "${TARGET}" \
|
||||
--branch "${BRANCH}" \
|
||||
--arch "${ARCH}"
|
||||
fi
|
||||
|
||||
echo "[deploy] step 2/4 — stop-services"
|
||||
"${SCRIPT_DIR}/stop-services.sh" "${common_args[@]}" "${force_args[@]}"
|
||||
|
||||
echo "[deploy] step 3/4 — start-services"
|
||||
"${SCRIPT_DIR}/start-services.sh" \
|
||||
"${common_args[@]}" \
|
||||
--wait-secs "${WAIT_SECS}" \
|
||||
"${force_args[@]}"
|
||||
|
||||
echo "[deploy] step 4/4 — health-check"
|
||||
if ! "${SCRIPT_DIR}/health-check.sh" "${common_args[@]}"; then
|
||||
echo "ERROR: health-check failed after deploy. See deployment_procedures.md § Rollback." >&2
|
||||
echo " To roll back: scripts/deploy.sh --rollback ${common_args[*]}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "═══════════════════════════════════════════════════"
|
||||
echo " DEPLOY OK — ${TARGET} @ ${BRANCH}-${ARCH}"
|
||||
if [ "${ROLLBACK}" = "1" ]; then
|
||||
echo " (rollback to previously-saved image digests)"
|
||||
fi
|
||||
echo "═══════════════════════════════════════════════════"
|
||||
Executable
+119
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env bash
|
||||
# GPS-Denied Onboard — verify the running service stack is healthy.
|
||||
#
|
||||
# The companion has no inbound HTTP `/health/*` endpoint (NFT-SEC-05 —
|
||||
# in-flight egress lockdown). Health is read from the Docker
|
||||
# HEALTHCHECK status (exec-based `python3 -m gps_denied_onboard.healthcheck`
|
||||
# per deployment_procedures.md § Health Checks).
|
||||
#
|
||||
# Usage:
|
||||
# scripts/health-check.sh [--target dev|airborne|operator-workstation]
|
||||
# [--compose-file <path>] [--help]
|
||||
#
|
||||
# Defaults:
|
||||
# target = dev
|
||||
# compose-file = docker-compose.yml
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 all services healthy (or running with no declared healthcheck)
|
||||
# 1 at least one service is unhealthy
|
||||
# 2 at least one service is missing (compose up was not run, or container exited)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
GPS-Denied Onboard — verify the running service stack is healthy.
|
||||
|
||||
The companion has no inbound HTTP `/health/*` endpoint (NFT-SEC-05 —
|
||||
in-flight egress lockdown). Health is read from the Docker HEALTHCHECK
|
||||
status (exec-based `python3 -m gps_denied_onboard.healthcheck` per
|
||||
deployment_procedures.md § Health Checks).
|
||||
|
||||
Usage:
|
||||
scripts/health-check.sh [--target dev|airborne|operator-workstation]
|
||||
[--compose-file <path>] [--help]
|
||||
|
||||
Defaults:
|
||||
target = dev
|
||||
compose-file = docker-compose.yml
|
||||
|
||||
Exit codes:
|
||||
0 all services healthy (or running with no declared healthcheck)
|
||||
1 at least one service is unhealthy
|
||||
2 at least one service is missing (compose up was not run, or container exited)
|
||||
EOF
|
||||
exit 0
|
||||
}
|
||||
|
||||
TARGET="dev"
|
||||
COMPOSE_FILE=""
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--target) TARGET="$2"; shift 2 ;;
|
||||
--compose-file) COMPOSE_FILE="$2"; shift 2 ;;
|
||||
--help|-h) usage ;;
|
||||
*) echo "ERROR: unknown argument: $1" >&2; usage ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -z "${COMPOSE_FILE}" ]; then
|
||||
case "${TARGET}" in
|
||||
dev) COMPOSE_FILE="docker-compose.yml" ;;
|
||||
airborne) COMPOSE_FILE="${AIRBORNE_COMPOSE_FILE:-/etc/gps-denied/docker-compose.airborne.yml}" ;;
|
||||
operator-workstation) COMPOSE_FILE="docker-compose.yml" ;;
|
||||
*) echo "ERROR: invalid --target: ${TARGET}" >&2; exit 64 ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
remote_exec=""
|
||||
if [ -n "${DEPLOY_HOST:-}" ]; then
|
||||
if ! ssh -o BatchMode=yes -o ConnectTimeout=5 "${DEPLOY_HOST}" true 2>/dev/null; then
|
||||
echo "ERROR: cannot reach 'ssh ${DEPLOY_HOST}' non-interactively." >&2
|
||||
exit 64
|
||||
fi
|
||||
remote_exec="ssh ${DEPLOY_HOST}"
|
||||
fi
|
||||
|
||||
echo "[health-check] target: ${TARGET}"
|
||||
echo "[health-check] compose-file: ${COMPOSE_FILE}"
|
||||
|
||||
# Format: <service>\t<state>\t<health>
|
||||
status_lines=$(${remote_exec} docker compose -f "${COMPOSE_FILE}" ps -a \
|
||||
--format '{{.Service}}{{"\t"}}{{.State}}{{"\t"}}{{.Health}}' 2>/dev/null \
|
||||
|| true)
|
||||
|
||||
if [ -z "${status_lines}" ]; then
|
||||
echo "ERROR: no containers found for ${COMPOSE_FILE}" >&2
|
||||
echo " Did you run scripts/start-services.sh?" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
rc=0
|
||||
printf '%-30s %-12s %-12s\n' SERVICE STATE HEALTH
|
||||
printf '%-30s %-12s %-12s\n' "------------------------------" "------------" "------------"
|
||||
while IFS=$'\t' read -r service state health; do
|
||||
[ -z "${service}" ] && continue
|
||||
if [ -z "${health}" ]; then
|
||||
health_display="(none)"
|
||||
else
|
||||
health_display="${health}"
|
||||
fi
|
||||
printf '%-30s %-12s %-12s\n' "${service}" "${state}" "${health_display}"
|
||||
if [ "${state}" != "running" ]; then
|
||||
rc=2
|
||||
elif [ -n "${health}" ] && [ "${health}" != "healthy" ]; then
|
||||
rc=1
|
||||
fi
|
||||
done <<< "${status_lines}"
|
||||
|
||||
case "${rc}" in
|
||||
0) echo "[health-check] OK" ;;
|
||||
1) echo "[health-check] UNHEALTHY — at least one service failed HEALTHCHECK" >&2 ;;
|
||||
2) echo "[health-check] MISSING — at least one service is not running" >&2 ;;
|
||||
esac
|
||||
exit "${rc}"
|
||||
Executable
+178
@@ -0,0 +1,178 @@
|
||||
#!/usr/bin/env bash
|
||||
# GPS-Denied Onboard — pull Docker images from the parent-suite registry.
|
||||
#
|
||||
# Cycle-1 images:
|
||||
# * ${REGISTRY_HOST}/azaion/gps-denied-onboard-companion-tier1:<branch>-<arch>
|
||||
# * ${REGISTRY_HOST}/azaion/gps-denied-onboard-operator-orchestrator:<branch>-<arch>
|
||||
#
|
||||
# Cycle-2 (when docker/companion-jetson.Dockerfile lands):
|
||||
# * ${REGISTRY_HOST}/azaion/gps-denied-onboard:<branch>-<arch> (airborne)
|
||||
#
|
||||
# Reads REGISTRY_HOST / REGISTRY_USER / REGISTRY_TOKEN from environment
|
||||
# or .env at the project root. Supports remote execution via SSH when
|
||||
# DEPLOY_HOST is set (delegates to the remote host's local docker pull).
|
||||
#
|
||||
# Usage:
|
||||
# scripts/pull-images.sh [--branch <branch>] [--arch <arch>]
|
||||
# [--target dev|airborne|operator-workstation]
|
||||
# [--verify] [--help]
|
||||
#
|
||||
# Defaults:
|
||||
# branch = main
|
||||
# arch = arm (Jetson + suite arm64 build agent)
|
||||
# target = operator-workstation
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 all images pulled (or already up-to-date)
|
||||
# 64 missing prerequisite (REGISTRY_HOST, docker on PATH)
|
||||
# 65 SSH unreachable when DEPLOY_HOST is set
|
||||
# 66 registry login failed
|
||||
# 67 image pull failed
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
GPS-Denied Onboard — pull Docker images from the parent-suite registry.
|
||||
|
||||
Cycle-1 images:
|
||||
* ${REGISTRY_HOST}/azaion/gps-denied-onboard-companion-tier1:<branch>-<arch>
|
||||
* ${REGISTRY_HOST}/azaion/gps-denied-onboard-operator-orchestrator:<branch>-<arch>
|
||||
|
||||
Cycle-2 (when docker/companion-jetson.Dockerfile lands):
|
||||
* ${REGISTRY_HOST}/azaion/gps-denied-onboard:<branch>-<arch> (airborne)
|
||||
|
||||
Reads REGISTRY_HOST / REGISTRY_USER / REGISTRY_TOKEN from environment
|
||||
or .env at the project root. Supports remote execution via SSH when
|
||||
DEPLOY_HOST is set (delegates to the remote host's local docker pull).
|
||||
|
||||
Usage:
|
||||
scripts/pull-images.sh [--branch <branch>] [--arch <arch>]
|
||||
[--target dev|airborne|operator-workstation]
|
||||
[--verify] [--help]
|
||||
|
||||
Defaults:
|
||||
branch = main
|
||||
arch = arm (Jetson + suite arm64 build agent)
|
||||
target = operator-workstation
|
||||
|
||||
Exit codes:
|
||||
0 all images pulled (or already up-to-date)
|
||||
64 missing prerequisite (REGISTRY_HOST, docker on PATH)
|
||||
65 SSH unreachable when DEPLOY_HOST is set
|
||||
66 registry login failed
|
||||
67 image pull failed
|
||||
EOF
|
||||
exit 0
|
||||
}
|
||||
|
||||
BRANCH="${BRANCH:-main}"
|
||||
ARCH="${ARCH:-arm}"
|
||||
TARGET="operator-workstation"
|
||||
VERIFY=0
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--branch) BRANCH="$2"; shift 2 ;;
|
||||
--arch) ARCH="$2"; shift 2 ;;
|
||||
--target) TARGET="$2"; shift 2 ;;
|
||||
--verify) VERIFY=1; shift ;;
|
||||
--help|-h) usage ;;
|
||||
*) echo "ERROR: unknown argument: $1" >&2; usage ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -f "${REPO_ROOT}/.env" ]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
. "${REPO_ROOT}/.env"
|
||||
set +a
|
||||
fi
|
||||
|
||||
if [ -z "${REGISTRY_HOST:-}" ]; then
|
||||
echo "ERROR: REGISTRY_HOST not set (e.g. git.azaion.com)" >&2
|
||||
echo " Set in .env or export before invoking." >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
remote_exec=""
|
||||
if [ -n "${DEPLOY_HOST:-}" ]; then
|
||||
if ! ssh -o BatchMode=yes -o ConnectTimeout=5 "${DEPLOY_HOST}" true 2>/dev/null; then
|
||||
echo "ERROR: cannot reach 'ssh ${DEPLOY_HOST}' non-interactively." >&2
|
||||
exit 65
|
||||
fi
|
||||
remote_exec="ssh ${DEPLOY_HOST}"
|
||||
fi
|
||||
|
||||
case "${TARGET}" in
|
||||
dev)
|
||||
IMAGES=(
|
||||
"${REGISTRY_HOST}/azaion/gps-denied-onboard-companion-tier1:${BRANCH}-${ARCH}"
|
||||
"${REGISTRY_HOST}/azaion/gps-denied-onboard-operator-orchestrator:${BRANCH}-${ARCH}"
|
||||
)
|
||||
;;
|
||||
airborne)
|
||||
IMAGES=(
|
||||
"${REGISTRY_HOST}/azaion/gps-denied-onboard:${BRANCH}-${ARCH}"
|
||||
)
|
||||
;;
|
||||
operator-workstation)
|
||||
IMAGES=(
|
||||
"${REGISTRY_HOST}/azaion/gps-denied-onboard-operator-orchestrator:${BRANCH}-${ARCH}"
|
||||
)
|
||||
;;
|
||||
*)
|
||||
echo "ERROR: invalid --target: ${TARGET}" >&2
|
||||
echo " Must be one of: dev, airborne, operator-workstation" >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "[pull-images] registry: ${REGISTRY_HOST}"
|
||||
echo "[pull-images] branch: ${BRANCH}"
|
||||
echo "[pull-images] arch: ${ARCH}"
|
||||
echo "[pull-images] target: ${TARGET}"
|
||||
if [ -n "${remote_exec}" ]; then
|
||||
echo "[pull-images] remote: ${DEPLOY_HOST}"
|
||||
fi
|
||||
|
||||
if [ -n "${REGISTRY_USER:-}" ] && [ -n "${REGISTRY_TOKEN:-}" ]; then
|
||||
echo "[pull-images] docker login → ${REGISTRY_HOST}"
|
||||
if ! echo "${REGISTRY_TOKEN}" \
|
||||
| ${remote_exec} docker login "${REGISTRY_HOST}" \
|
||||
--username "${REGISTRY_USER}" --password-stdin >/dev/null; then
|
||||
echo "ERROR: docker login failed against ${REGISTRY_HOST}" >&2
|
||||
exit 66
|
||||
fi
|
||||
else
|
||||
echo "[pull-images] REGISTRY_USER / REGISTRY_TOKEN not set — assuming"
|
||||
echo " the registry is reachable anonymously or the local"
|
||||
echo " docker config is already authenticated."
|
||||
fi
|
||||
|
||||
for image in "${IMAGES[@]}"; do
|
||||
echo "[pull-images] pulling ${image}"
|
||||
if ! ${remote_exec} docker pull "${image}"; then
|
||||
echo "ERROR: failed to pull ${image}" >&2
|
||||
exit 67
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "${VERIFY}" = "1" ]; then
|
||||
echo "[pull-images] verifying image digests + AZAION_REVISION env"
|
||||
for image in "${IMAGES[@]}"; do
|
||||
digest=$(${remote_exec} docker image inspect \
|
||||
--format '{{(index .RepoDigests 0)}}' "${image}" 2>/dev/null || echo "")
|
||||
revision=$(${remote_exec} docker image inspect \
|
||||
--format '{{range .Config.Env}}{{println .}}{{end}}' "${image}" 2>/dev/null \
|
||||
| grep '^AZAION_REVISION=' | head -n1 | cut -d= -f2 || echo "")
|
||||
echo " ${image}"
|
||||
echo " digest: ${digest:-<missing>}"
|
||||
echo " revision: ${revision:-<missing>}"
|
||||
done
|
||||
fi
|
||||
|
||||
echo "[pull-images] OK"
|
||||
Executable
+166
@@ -0,0 +1,166 @@
|
||||
#!/usr/bin/env bash
|
||||
# GPS-Denied Onboard — start the service stack.
|
||||
#
|
||||
# Sources environment from .env (project root) and brings up the
|
||||
# appropriate compose file. Waits for every service that declares a
|
||||
# HEALTHCHECK to report `healthy`. Emits AZAION_UPDATE_EVENT to journald
|
||||
# when run on a systemd host (matches the suite-wide audit chain — see
|
||||
# observability.md § Deploy Audit).
|
||||
#
|
||||
# Refuses to start the airborne stack when /run/azaion/in-flight is set —
|
||||
# the FC owns flight; restarting the companion mid-flight would race the
|
||||
# in-flight failsafe (deployment_procedures.md § Deployment Strategy).
|
||||
#
|
||||
# Usage:
|
||||
# scripts/start-services.sh [--target dev|airborne|operator-workstation]
|
||||
# [--compose-file <path>]
|
||||
# [--wait-secs N] [--force] [--help]
|
||||
#
|
||||
# Defaults:
|
||||
# target = dev
|
||||
# compose-file = docker-compose.yml
|
||||
# wait-secs = 120
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 all services healthy
|
||||
# 64 missing prerequisite
|
||||
# 65 SSH unreachable when DEPLOY_HOST is set
|
||||
# 68 refused — /run/azaion/in-flight is set and --force was not passed
|
||||
# 69 timed out waiting for healthchecks
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
GPS-Denied Onboard — start the service stack.
|
||||
|
||||
Sources environment from .env (project root) and brings up the
|
||||
appropriate compose file. Waits for every service that declares a
|
||||
HEALTHCHECK to report `healthy`. Emits AZAION_UPDATE_EVENT to journald
|
||||
when run on a systemd host (matches the suite-wide audit chain — see
|
||||
observability.md § Deploy Audit).
|
||||
|
||||
Refuses to start the airborne stack when /run/azaion/in-flight is set —
|
||||
the FC owns flight; restarting the companion mid-flight would race the
|
||||
in-flight failsafe (deployment_procedures.md § Deployment Strategy).
|
||||
|
||||
Usage:
|
||||
scripts/start-services.sh [--target dev|airborne|operator-workstation]
|
||||
[--compose-file <path>]
|
||||
[--wait-secs N] [--force] [--help]
|
||||
|
||||
Defaults:
|
||||
target = dev
|
||||
compose-file = docker-compose.yml
|
||||
wait-secs = 120
|
||||
|
||||
Exit codes:
|
||||
0 all services healthy
|
||||
64 missing prerequisite
|
||||
65 SSH unreachable when DEPLOY_HOST is set
|
||||
68 refused — /run/azaion/in-flight is set and --force was not passed
|
||||
69 timed out waiting for healthchecks
|
||||
EOF
|
||||
exit 0
|
||||
}
|
||||
|
||||
TARGET="dev"
|
||||
COMPOSE_FILE=""
|
||||
WAIT_SECS=120
|
||||
FORCE=0
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--target) TARGET="$2"; shift 2 ;;
|
||||
--compose-file) COMPOSE_FILE="$2"; shift 2 ;;
|
||||
--wait-secs) WAIT_SECS="$2"; shift 2 ;;
|
||||
--force) FORCE=1; shift ;;
|
||||
--help|-h) usage ;;
|
||||
*) echo "ERROR: unknown argument: $1" >&2; usage ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -f "${REPO_ROOT}/.env" ]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
. "${REPO_ROOT}/.env"
|
||||
set +a
|
||||
fi
|
||||
|
||||
if [ -z "${COMPOSE_FILE}" ]; then
|
||||
case "${TARGET}" in
|
||||
dev) COMPOSE_FILE="docker-compose.yml" ;;
|
||||
airborne) COMPOSE_FILE="${AIRBORNE_COMPOSE_FILE:-/etc/gps-denied/docker-compose.airborne.yml}" ;;
|
||||
operator-workstation) COMPOSE_FILE="docker-compose.yml" ;;
|
||||
*) echo "ERROR: invalid --target: ${TARGET}" >&2; exit 64 ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
remote_exec=""
|
||||
if [ -n "${DEPLOY_HOST:-}" ]; then
|
||||
if ! ssh -o BatchMode=yes -o ConnectTimeout=5 "${DEPLOY_HOST}" true 2>/dev/null; then
|
||||
echo "ERROR: cannot reach 'ssh ${DEPLOY_HOST}' non-interactively." >&2
|
||||
exit 65
|
||||
fi
|
||||
remote_exec="ssh ${DEPLOY_HOST}"
|
||||
fi
|
||||
|
||||
if [ "${TARGET}" = "airborne" ]; then
|
||||
if ${remote_exec} test -e /run/azaion/in-flight 2>/dev/null; then
|
||||
if [ "${FORCE}" = "1" ]; then
|
||||
echo "WARNING: /run/azaion/in-flight is set but --force was passed —" >&2
|
||||
echo " continuing. This will race the FC in-flight failsafe." >&2
|
||||
else
|
||||
echo "ERROR: /run/azaion/in-flight is set on the target host." >&2
|
||||
echo " Refusing to (re)start the airborne stack mid-flight." >&2
|
||||
echo " Wait for the flight to end, or pass --force to override." >&2
|
||||
exit 68
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! ${remote_exec} command -v docker >/dev/null 2>&1; then
|
||||
echo "ERROR: docker not found on target" >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
echo "[start-services] target: ${TARGET}"
|
||||
echo "[start-services] compose-file: ${COMPOSE_FILE}"
|
||||
echo "[start-services] wait: ${WAIT_SECS}s"
|
||||
|
||||
echo "[start-services] docker compose up -d --remove-orphans"
|
||||
${remote_exec} docker compose -f "${COMPOSE_FILE}" up -d --remove-orphans
|
||||
|
||||
echo "[start-services] waiting for services to reach 'healthy' state"
|
||||
deadline=$(( $(date +%s) + WAIT_SECS ))
|
||||
while :; do
|
||||
now=$(date +%s)
|
||||
if [ "${now}" -ge "${deadline}" ]; then
|
||||
echo "ERROR: timed out after ${WAIT_SECS}s waiting for healthchecks" >&2
|
||||
${remote_exec} docker compose -f "${COMPOSE_FILE}" ps
|
||||
exit 69
|
||||
fi
|
||||
unhealthy_or_starting=$(${remote_exec} docker compose -f "${COMPOSE_FILE}" ps \
|
||||
--format '{{.Service}} {{.Health}}' 2>/dev/null \
|
||||
| awk '$2 != "" && $2 != "healthy" { print }' \
|
||||
|| true)
|
||||
if [ -z "${unhealthy_or_starting}" ]; then
|
||||
break
|
||||
fi
|
||||
sleep 3
|
||||
done
|
||||
|
||||
# Audit event — suite-mandated AZAION_UPDATE_EVENT (observability.md).
|
||||
# Skip on macOS / non-systemd dev hosts (logger -t still works locally).
|
||||
revision="${AZAION_REVISION:-unknown}"
|
||||
if ${remote_exec} command -v logger >/dev/null 2>&1; then
|
||||
${remote_exec} logger -t gps-denied-onboard -p user.notice \
|
||||
"AZAION_UPDATE_EVENT service=gps-denied-onboard target=${TARGET} revision=${revision} action=started outcome=success" \
|
||||
|| true
|
||||
fi
|
||||
|
||||
echo "[start-services] OK"
|
||||
${remote_exec} docker compose -f "${COMPOSE_FILE}" ps
|
||||
Executable
+136
@@ -0,0 +1,136 @@
|
||||
#!/usr/bin/env bash
|
||||
# GPS-Denied Onboard — graceful shutdown of the service stack.
|
||||
#
|
||||
# Refuses to stop the airborne stack when /run/azaion/in-flight is set
|
||||
# (deployment_procedures.md § Deployment Strategy — ground-only safety
|
||||
# gate). Saves the current image SHAs to .previous-tags.env so deploy.sh
|
||||
# --rollback can restore them.
|
||||
#
|
||||
# Honours the 30 s stop_grace_period declared in docker-compose.yml so
|
||||
# C13 can emit flight_footer{clean_shutdown=true} per
|
||||
# deployment_procedures.md § Graceful Shutdown.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/stop-services.sh [--target dev|airborne|operator-workstation]
|
||||
# [--compose-file <path>] [--force] [--help]
|
||||
#
|
||||
# Defaults:
|
||||
# target = dev
|
||||
# compose-file = docker-compose.yml
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 stack stopped cleanly (or already stopped)
|
||||
# 64 missing prerequisite (docker / compose file)
|
||||
# 65 SSH unreachable when DEPLOY_HOST is set
|
||||
# 68 refused — /run/azaion/in-flight is set and --force was not passed
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
GPS-Denied Onboard — graceful shutdown of the service stack.
|
||||
|
||||
Refuses to stop the airborne stack when /run/azaion/in-flight is set
|
||||
(deployment_procedures.md § Deployment Strategy — ground-only safety
|
||||
gate). Saves the current image SHAs to .previous-tags.env so deploy.sh
|
||||
--rollback can restore them.
|
||||
|
||||
Honours the 30 s stop_grace_period declared in docker-compose.yml so
|
||||
C13 can emit flight_footer{clean_shutdown=true} per
|
||||
deployment_procedures.md § Graceful Shutdown.
|
||||
|
||||
Usage:
|
||||
scripts/stop-services.sh [--target dev|airborne|operator-workstation]
|
||||
[--compose-file <path>] [--force] [--help]
|
||||
|
||||
Defaults:
|
||||
target = dev
|
||||
compose-file = docker-compose.yml
|
||||
|
||||
Exit codes:
|
||||
0 stack stopped cleanly (or already stopped)
|
||||
64 missing prerequisite (docker / compose file)
|
||||
65 SSH unreachable when DEPLOY_HOST is set
|
||||
68 refused — /run/azaion/in-flight is set and --force was not passed
|
||||
EOF
|
||||
exit 0
|
||||
}
|
||||
|
||||
TARGET="dev"
|
||||
COMPOSE_FILE=""
|
||||
FORCE=0
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--target) TARGET="$2"; shift 2 ;;
|
||||
--compose-file) COMPOSE_FILE="$2"; shift 2 ;;
|
||||
--force) FORCE=1; shift ;;
|
||||
--help|-h) usage ;;
|
||||
*) echo "ERROR: unknown argument: $1" >&2; usage ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -z "${COMPOSE_FILE}" ]; then
|
||||
case "${TARGET}" in
|
||||
dev) COMPOSE_FILE="docker-compose.yml" ;;
|
||||
airborne) COMPOSE_FILE="${AIRBORNE_COMPOSE_FILE:-/etc/gps-denied/docker-compose.airborne.yml}" ;;
|
||||
operator-workstation) COMPOSE_FILE="docker-compose.yml" ;;
|
||||
*) echo "ERROR: invalid --target: ${TARGET}" >&2; exit 64 ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
remote_exec=""
|
||||
if [ -n "${DEPLOY_HOST:-}" ]; then
|
||||
if ! ssh -o BatchMode=yes -o ConnectTimeout=5 "${DEPLOY_HOST}" true 2>/dev/null; then
|
||||
echo "ERROR: cannot reach 'ssh ${DEPLOY_HOST}' non-interactively." >&2
|
||||
exit 65
|
||||
fi
|
||||
remote_exec="ssh ${DEPLOY_HOST}"
|
||||
fi
|
||||
|
||||
if [ "${TARGET}" = "airborne" ]; then
|
||||
if ${remote_exec} test -e /run/azaion/in-flight 2>/dev/null; then
|
||||
if [ "${FORCE}" = "1" ]; then
|
||||
echo "WARNING: /run/azaion/in-flight is set but --force was passed —" >&2
|
||||
echo " continuing. This will abort a live flight pipeline." >&2
|
||||
else
|
||||
echo "ERROR: /run/azaion/in-flight is set on the target host." >&2
|
||||
echo " Refusing to stop the airborne stack mid-flight." >&2
|
||||
echo " Wait for the flight to end, or pass --force to override." >&2
|
||||
exit 68
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! ${remote_exec} command -v docker >/dev/null 2>&1; then
|
||||
echo "ERROR: docker not found on $( [ -n "${remote_exec}" ] && echo "remote" || echo "local" ) host" >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
echo "[stop-services] target: ${TARGET}"
|
||||
echo "[stop-services] compose-file: ${COMPOSE_FILE}"
|
||||
|
||||
PREV_TAGS_FILE="${REPO_ROOT}/.previous-tags.env"
|
||||
echo "[stop-services] saving current image digests to ${PREV_TAGS_FILE}"
|
||||
{
|
||||
echo "# Saved by scripts/stop-services.sh on $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "# Used by deploy.sh --rollback to restore the previous image set."
|
||||
echo "# Service tag layout: PREV_<SERVICE>_IMAGE=<repo>@<sha256-digest>"
|
||||
${remote_exec} docker compose -f "${COMPOSE_FILE}" config --services 2>/dev/null \
|
||||
| while read -r service; do
|
||||
image=$(${remote_exec} docker compose -f "${COMPOSE_FILE}" \
|
||||
images --format '{{.Repository}}@{{.ID}}' "${service}" 2>/dev/null \
|
||||
| head -n1 || true)
|
||||
[ -n "${image}" ] || continue
|
||||
key=$(echo "PREV_${service}_IMAGE" | tr '[:lower:]-' '[:upper:]_')
|
||||
echo "${key}=${image}"
|
||||
done
|
||||
} > "${PREV_TAGS_FILE}"
|
||||
|
||||
echo "[stop-services] docker compose down (graceful, 30s grace per compose stop_grace_period)"
|
||||
${remote_exec} docker compose -f "${COMPOSE_FILE}" down --remove-orphans
|
||||
|
||||
echo "[stop-services] OK"
|
||||
Reference in New Issue
Block a user