Automate immutable production deployments

deploy/scripts/auto-deploy.sh

@@ -0,0 +1,414 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+WM_DEPLOY_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+# shellcheck source=common.sh
+source "$WM_DEPLOY_DIR/scripts/common.sh"
+
+[[ "$(id -u)" -eq 0 ]] ||
+  wm_die "Dieses Skript muss als root ausgeführt werden."
+wm_load_env
+wm_assert_data_mount
+
+registry="${WATERMAPS_REGISTRY:-}"
+registry_owner="${WATERMAPS_REGISTRY_OWNER:-}"
+repository="${WATERMAPS_GITEA_REPOSITORY:-$registry_owner/watermaps}"
+
+[[ "$registry" =~ ^[a-z0-9][a-z0-9.-]*(:[0-9]{1,5})?$ ]] ||
+  wm_die "WATERMAPS_REGISTRY muss ein Registry-Hostname ohne URL-Schema sein."
+[[ "$registry_owner" =~ ^[A-Za-z0-9_.-]+$ ]] ||
+  wm_die "WATERMAPS_REGISTRY_OWNER enthält keinen gültigen Gitea-Besitzer."
+[[ "$repository" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] ||
+  wm_die "WATERMAPS_GITEA_REPOSITORY muss Besitzer und Repository enthalten."
+
+release_repository="$registry/$registry_owner/watermaps-release"
+release_pointer="$release_repository:main"
+app_repository="$registry/$registry_owner/watermaps"
+route_data_repository="$registry/$registry_owner/watermaps-route-data"
+release_store="$WATERMAPS_DATA_DIR/releases"
+install_root="${WATERMAPS_INSTALL_DIR:-/opt/watermaps}"
+wm_require_safe_absolute_dir "$install_root"
+shared_deploy_dir="$install_root/deploy"
+active_images_file="$shared_deploy_dir/.env.images"
+current_release_link="$install_root/current"
+release_retention="${WATERMAPS_RELEASE_RETENTION:-5}"
+[[ "$release_retention" =~ ^[2-9]$|^1[0-9]$|^20$ ]] ||
+  wm_die "WATERMAPS_RELEASE_RETENTION muss zwischen 2 und 20 liegen."
+
+install -d -m 0755 \
+  "$WATERMAPS_RUNTIME_DIR/locks" \
+  "$release_store" \
+  "$shared_deploy_dir"
+exec 8>"$WATERMAPS_RUNTIME_DIR/locks/auto-deploy.lock"
+if ! flock --nonblock 8; then
+  wm_log "Eine automatische Release-Prüfung läuft bereits."
+  exit 0
+fi
+
+temporary_release_dir=""
+temporary_images_file=""
+temporary_current_link=""
+release_container_id=""
+release_directory=""
+
+cleanup() {
+  local status=$?
+  trap - EXIT HUP INT TERM
+  set +e
+  if [[ -n "$release_container_id" ]]; then
+    docker container rm --force "$release_container_id" >/dev/null 2>&1
+  fi
+  if [[ -n "$temporary_release_dir" && -d "$temporary_release_dir" ]]; then
+    rm -rf -- "$temporary_release_dir"
+  fi
+  if [[ -n "$temporary_images_file" && -f "$temporary_images_file" ]]; then
+    rm -f -- "$temporary_images_file"
+  fi
+  if [[ -n "$temporary_current_link" && -L "$temporary_current_link" ]]; then
+    rm -f -- "$temporary_current_link"
+  fi
+  exit "$status"
+}
+
+trap cleanup EXIT HUP INT TERM
+
+image_revision() {
+  local image_reference="$1"
+  local revision
+
+  revision="$(
+    docker image inspect \
+      --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' \
+      "$image_reference"
+  )"
+  [[ "$revision" =~ ^[0-9a-f]{40}$ ]] ||
+    wm_die "Image $image_reference enthält kein gültiges OCI-Revision-Label."
+  printf '%s\n' "$revision"
+}
+
+image_id() {
+  docker image inspect --format '{{.Id}}' "$1"
+}
+
+gitea_main_revision() {
+  local api_url
+  api_url="https://$registry/api/v1/repos/$repository/branches/main"
+  curl \
+    --fail \
+    --silent \
+    --show-error \
+    --location \
+    --max-time 30 \
+    "$api_url" |
+    jq --exit-status --raw-output '
+      .commit.id
+      | select(type == "string" and test("^[0-9a-f]{40}$"))
+    '
+}
+
+validate_release_bundle() {
+  local directory="$1"
+  local expected_revision="$2"
+  local required_file
+  local manifest_revision manifest_source manifest_app manifest_route
+
+  for required_file in \
+    SHA256SUMS \
+    release.env \
+    deploy/compose.production.yml \
+    deploy/nginx/bootstrap.conf \
+    deploy/nginx/https.conf.template \
+    deploy/scripts/common.sh \
+    deploy/scripts/auto-deploy.sh \
+    deploy/scripts/auto-deploy-entrypoint.sh \
+    deploy/scripts/bootstrap-server.sh \
+    deploy/scripts/deploy.sh \
+    deploy/scripts/renew-certificate.sh \
+    deploy/scripts/rollback.sh \
+    deploy/scripts/update-route-data.sh \
+    deploy/systemd/watermaps-auto-deploy.service \
+    deploy/systemd/watermaps-auto-deploy.timer \
+    deploy/systemd/watermaps-certbot-renew.service \
+    deploy/systemd/watermaps-certbot-renew.timer \
+    deploy/systemd/watermaps-route-update.service \
+    deploy/systemd/watermaps-route-update.timer \
+    database/schema.sql \
+    database/migrations/0001_initial.sql; do
+    [[ -f "$directory/$required_file" ]] ||
+      wm_die "Release-Bundle ist unvollständig: $required_file fehlt."
+  done
+
+  (
+    cd "$directory"
+    sha256sum --check --strict SHA256SUMS >/dev/null
+  ) || wm_die "Release-Bundle hat die Integritätsprüfung nicht bestanden."
+
+  [[ "$(wm_env_value "$directory/release.env" format_version)" == "1" ]] ||
+    wm_die "Release-Manifest hat eine unbekannte Formatversion."
+  manifest_revision="$(wm_env_value "$directory/release.env" revision)"
+  manifest_source="$(wm_env_value "$directory/release.env" source)"
+  manifest_app="$(wm_env_value "$directory/release.env" app_image)"
+  manifest_route="$(wm_env_value "$directory/release.env" route_data_image)"
+
+  [[ "$manifest_revision" == "$expected_revision" ]] ||
+    wm_die "Release-Manifest und erwartete Revision stimmen nicht überein."
+  [[ "$manifest_source" == "https://$registry/$repository" ]] ||
+    wm_die "Release-Manifest verweist auf eine unerwartete Quelle."
+  [[ "$manifest_app" =~ ^${app_repository//./\\.}@sha256:[0-9a-f]{64}$ ]] ||
+    wm_die "Release-Manifest enthält kein gültiges App-Image."
+  [[ "$manifest_route" =~ ^${route_data_repository//./\\.}@sha256:[0-9a-f]{64}$ ]] ||
+    wm_die "Release-Manifest enthält kein gültiges Routingdaten-Image."
+}
+
+ensure_release_bundle() {
+  local revision="$1"
+  local image_reference="$release_repository:$revision"
+  local destination="$release_store/$revision"
+
+  if [[ -d "$destination" ]]; then
+    validate_release_bundle "$destination" "$revision"
+    release_directory="$destination"
+    return
+  fi
+
+  docker pull "$image_reference" >/dev/null
+  [[ "$(image_revision "$image_reference")" == "$revision" ]] ||
+    wm_die "Release-Image und erwartete Revision stimmen nicht überein."
+
+  temporary_release_dir="$(mktemp -d "$release_store/.candidate-$revision.XXXXXX")"
+  release_container_id="$(docker create "$image_reference")"
+  docker cp "$release_container_id:/release/." "$temporary_release_dir/"
+  docker container rm "$release_container_id" >/dev/null
+  release_container_id=""
+  validate_release_bundle "$temporary_release_dir" "$revision"
+  chmod 0755 "$temporary_release_dir/deploy/scripts/"*.sh
+  mv "$temporary_release_dir" "$destination"
+  temporary_release_dir=""
+  release_directory="$destination"
+}
+
+install_release_units() {
+  local source_directory="$1"
+
+  install -m 0755 \
+    "$source_directory/deploy/scripts/auto-deploy-entrypoint.sh" \
+    /usr/local/sbin/watermaps-auto-deploy
+  install -m 0644 \
+    "$source_directory/deploy/systemd/watermaps-auto-deploy.service" \
+    /etc/systemd/system/
+  install -m 0644 \
+    "$source_directory/deploy/systemd/watermaps-auto-deploy.timer" \
+    /etc/systemd/system/
+  install -m 0644 \
+    "$source_directory/deploy/systemd/watermaps-route-update.service" \
+    /etc/systemd/system/
+  install -m 0644 \
+    "$source_directory/deploy/systemd/watermaps-route-update.timer" \
+    /etc/systemd/system/
+  install -m 0644 \
+    "$source_directory/deploy/systemd/watermaps-certbot-renew.service" \
+    /etc/systemd/system/
+  install -m 0644 \
+    "$source_directory/deploy/systemd/watermaps-certbot-renew.timer" \
+    /etc/systemd/system/
+  systemctl daemon-reload
+  systemctl enable --now \
+    watermaps-auto-deploy.timer \
+    watermaps-route-update.timer \
+    watermaps-certbot-renew.timer \
+    >/dev/null
+}
+
+activate_release() {
+  local source_directory="$1"
+
+  temporary_current_link="$install_root/.current.$$.new"
+  rm -f -- "$temporary_current_link"
+  ln --symbolic "$source_directory" "$temporary_current_link"
+  mv --no-target-directory "$temporary_current_link" "$current_release_link"
+  temporary_current_link=""
+}
+
+prune_old_releases() {
+  local candidate old_revision old_directory old_app old_route
+  local retained_revision retained_manifest image_is_retained
+  local -a revisions=()
+  local -A keep=()
+
+  mapfile -t revisions < <(
+    find "$release_store" \
+      -mindepth 1 \
+      -maxdepth 1 \
+      -type d \
+      -printf '%T@ %f\n' |
+      sort --numeric-sort --reverse |
+      awk '$2 ~ /^[0-9a-f]{40}$/ { print $2 }'
+  )
+
+  keep["$revision"]=1
+  if [[ -n "$active_revision" ]]; then
+    keep["$active_revision"]=1
+  fi
+  for candidate in "${revisions[@]:0:release_retention}"; do
+    keep["$candidate"]=1
+  done
+
+  for old_revision in "${revisions[@]}"; do
+    [[ -z "${keep[$old_revision]:-}" ]] || continue
+    [[ "$old_revision" =~ ^[0-9a-f]{40}$ ]] || continue
+    old_directory="$release_store/$old_revision"
+    [[ -d "$old_directory" ]] || continue
+
+    old_app="$(wm_env_value "$old_directory/release.env" app_image)"
+    old_route="$(wm_env_value "$old_directory/release.env" route_data_image)"
+    for candidate in "$old_app" "$old_route"; do
+      image_is_retained=false
+      for retained_revision in "${!keep[@]}"; do
+        retained_manifest="$release_store/$retained_revision/release.env"
+        if [[ -f "$retained_manifest" ]] &&
+          grep --fixed-strings --line-regexp --quiet \
+            "app_image=$candidate" "$retained_manifest" ||
+          [[ -f "$retained_manifest" ]] &&
+          grep --fixed-strings --line-regexp --quiet \
+            "route_data_image=$candidate" "$retained_manifest"; then
+          image_is_retained=true
+          break
+        fi
+      done
+      if [[ "$image_is_retained" == "false" ]]; then
+        docker image rm "$candidate" >/dev/null 2>&1 || true
+      fi
+    done
+
+    docker image rm \
+      "$app_repository:$old_revision" \
+      "$route_data_repository:$old_revision" \
+      "$release_repository:$old_revision" \
+      >/dev/null 2>&1 ||
+      true
+    rm -rf -- "$old_directory"
+    wm_log "Altes lokales Release $old_revision wurde gezielt bereinigt."
+  done
+}
+
+wm_log "Prüfe den atomaren Gitea-Release-Pointer."
+docker pull "$release_pointer" >/dev/null
+revision="$(image_revision "$release_pointer")"
+release_pointer_id="$(image_id "$release_pointer")"
+
+main_revision="$(gitea_main_revision)"
+if [[ "$revision" != "$main_revision" ]]; then
+  wm_log "Release $revision ist nicht der aktuelle main-Commit $main_revision; warte auf den laufenden Build."
+  exit 0
+fi
+
+release_image="$release_repository:$revision"
+docker pull "$release_image" >/dev/null
+[[ "$(image_id "$release_image")" == "$release_pointer_id" ]] ||
+  wm_die "Der bewegliche Release-Pointer stimmt nicht mit dem unveränderlichen Commit-Image überein."
+
+ensure_release_bundle "$revision"
+
+active_revision=""
+if [[ -f "$active_images_file" ]]; then
+  wm_validate_images_env "$active_images_file"
+  active_revision="$(wm_env_value "$active_images_file" WATERMAPS_DEPLOY_REVISION)"
+fi
+if [[ "$active_revision" == "$revision" ]]; then
+  current_release_target="$(
+    readlink --canonicalize "$current_release_link" 2>/dev/null || true
+  )"
+  expected_release_target="$(readlink --canonicalize "$release_directory")"
+  if [[ ! -L "$current_release_link" ||
+        "$current_release_target" != "$expected_release_target" ]]; then
+    activate_release "$release_directory"
+    install_release_units "$release_directory"
+    wm_log "Der atomare Current-Link für Commit $revision wurde repariert."
+  fi
+  wm_log "Commit $revision ist bereits aktiv."
+  exit 0
+fi
+
+app_image="$(wm_env_value "$release_directory/release.env" app_image)"
+route_data_image="$(wm_env_value "$release_directory/release.env" route_data_image)"
+docker pull "$app_image" >/dev/null
+docker pull "$route_data_image" >/dev/null
+[[ "$(image_revision "$app_image")" == "$revision" ]] ||
+  wm_die "App-Image gehört nicht zu Release $revision."
+[[ "$(image_revision "$route_data_image")" == "$revision" ]] ||
+  wm_die "Routingdaten-Image gehört nicht zu Release $revision."
+
+resolved_app_image="$(wm_resolve_image_digest "$app_image")"
+resolved_route_data_image="$(wm_resolve_image_digest "$route_data_image")"
+temporary_images_file="$(
+  mktemp "$WATERMAPS_RUNTIME_DIR/.auto-deploy-images.XXXXXX"
+)"
+{
+  printf 'WATERMAPS_DEPLOY_REVISION=%s\n' "$revision"
+  printf 'WATERMAPS_APP_IMAGE=%s\n' "$resolved_app_image"
+  printf 'WATERMAPS_ROUTE_DATA_IMAGE=%s\n' "$resolved_route_data_image"
+} >"$temporary_images_file"
+chmod 0600 "$temporary_images_file"
+wm_validate_images_env "$temporary_images_file"
+
+# A stale workflow must never activate after a newer push. Pull and compare the
+# single release pointer immediately before changing deployment files.
+docker pull "$release_pointer" >/dev/null
+[[ "$(image_revision "$release_pointer")" == "$revision" ]] ||
+  wm_die "Während der Vorbereitung wurde ein neueres Release veröffentlicht."
+[[ "$(image_id "$release_pointer")" == "$release_pointer_id" ]] ||
+  wm_die "Der Release-Pointer wurde während der Vorbereitung ausgetauscht."
+[[ "$(gitea_main_revision)" == "$revision" ]] ||
+  wm_die "main wurde während der Vorbereitung auf einen neueren Commit verschoben."
+
+previous_release_directory=""
+previous_images_file=""
+if [[ -n "$active_revision" ]]; then
+  previous_release_directory="$release_store/$active_revision"
+  previous_images_file="$WATERMAPS_RUNTIME_DIR/deployments/$active_revision.env"
+  if [[ ! -d "$previous_release_directory" ||
+        ! -f "$previous_images_file" ]]; then
+    previous_release_directory=""
+    previous_images_file=""
+  fi
+fi
+
+set +e
+WATERMAPS_ENV_FILE="$WM_ENV_FILE" \
+WATERMAPS_ACTIVE_IMAGES_FILE="$active_images_file" \
+WATERMAPS_DISABLE_INTERNAL_ROLLBACK=true \
+  "$release_directory/deploy/scripts/deploy.sh" \
+    --images-file "$temporary_images_file"
+deploy_status=$?
+set -e
+
+if [[ "$deploy_status" -ne 0 ]]; then
+  if [[ "$deploy_status" -eq 75 ]]; then
+    wm_log "Ein Datenupdate oder manuelles Deployment läuft; das Release wird später erneut versucht."
+    exit 0
+  fi
+  if [[ -n "$previous_release_directory" &&
+        -n "$previous_images_file" ]]; then
+    wm_log "Das vorherige Release $active_revision wird aus seinem unveränderlichen Bundle wiederhergestellt."
+    set +e
+    WATERMAPS_ENV_FILE="$WM_ENV_FILE" \
+    WATERMAPS_ACTIVE_IMAGES_FILE="$active_images_file" \
+    WATERMAPS_DISABLE_INTERNAL_ROLLBACK=true \
+      "$previous_release_directory/deploy/scripts/deploy.sh" \
+        --images-file "$previous_images_file"
+    rollback_status=$?
+    set -e
+    if [[ "$rollback_status" -ne 0 ]]; then
+      wm_die "Release $revision und die Wiederherstellung von $active_revision sind fehlgeschlagen."
+    fi
+    activate_release "$previous_release_directory"
+    wm_log "Vorheriges Release $active_revision wurde erneut geprüft und ist aktiv."
+  fi
+  exit "$deploy_status"
+fi
+
+activate_release "$release_directory"
+install_release_units "$release_directory"
+prune_old_releases
+wm_log "Automatisches Deployment von Commit $revision abgeschlossen."