From 99ba1a237344c27600e2156b2d0b4a76cc73bdca Mon Sep 17 00:00:00 2001
From: BuTzZ <kevin.janssen@ease-technologies.de>
Date: Wed, 29 Jul 2026 13:18:44 +0200
Subject: [PATCH] Make first deployment race-safe

---
 deploy/README.md                              |   4 +
 deploy/scripts/remote-common.sh               |   8 +-
 .../scripts/tests/upload-and-deploy.test.sh   | 120 ++++++++++++++++++
 deploy/scripts/upload-and-deploy.sh           |  50 +++++++-
 package.json                                  |   2 +-
 5 files changed, 179 insertions(+), 5 deletions(-)
 create mode 100644 deploy/scripts/tests/upload-and-deploy.test.sh

diff --git a/deploy/README.md b/deploy/README.md
index 2acd787..b1425cb 100644
--- a/deploy/README.md
+++ b/deploy/README.md
@@ -134,6 +134,10 @@ weder Git noch Node/npm oder einen Repository-Checkout.
 Der erste Datenaufbau lädt die Geofabrik-Extrakte für Deutschland und die
 Niederlande, baut den Fahrroutenindex und importiert daraus die
 `marine_features`. Je nach Serverleistung kann dies längere Zeit dauern.
+Der Erstlauf wird dafür als begrenzter systemd-Job gestartet; das lokale
+Upload-Skript wartet über wiederverbindbare SSH-Prüfungen auf exakt den
+angeforderten Commit. So konkurriert der initiale Aufruf nicht mit dem
+Auto-Deploy-Timer und ein kurzzeitiger SSH-Abbruch beendet den Import nicht.
 App und Nginx werden beim ersten Release erst gestartet, wenn sowohl der
 Routingindex als auch ein nicht leerer, zur PBF-Prüfsumme passender Bestand an
 Häfen, Schleusen und Brücken geprüft wurde. Ein HTTP-Healthcheck allein kann
diff --git a/deploy/scripts/remote-common.sh b/deploy/scripts/remote-common.sh
index f15d620..d710327 100755
--- a/deploy/scripts/remote-common.sh
+++ b/deploy/scripts/remote-common.sh
@@ -64,7 +64,13 @@ wm_local_resolve_ssh() {
   esac
 
   WM_SSH_TARGET="$WM_SSH_USER@$WM_SERVER_IPV4"
-  WM_SSH_OPTIONS=(-o BatchMode=yes -o StrictHostKeyChecking=accept-new)
+  WM_SSH_OPTIONS=(
+    -o BatchMode=yes
+    -o StrictHostKeyChecking=accept-new
+    -o ServerAliveInterval=30
+    -o ServerAliveCountMax=20
+    -o TCPKeepAlive=yes
+  )
   if [[ -n "$WM_SSH_IDENTITY" ]]; then
     WM_SSH_OPTIONS+=(-i "$WM_SSH_IDENTITY" -o IdentitiesOnly=yes)
   fi
diff --git a/deploy/scripts/tests/upload-and-deploy.test.sh b/deploy/scripts/tests/upload-and-deploy.test.sh
new file mode 100644
index 0000000..b76e307
--- /dev/null
+++ b/deploy/scripts/tests/upload-and-deploy.test.sh
@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
+UPLOAD_SCRIPT="$ROOT_DIR/deploy/scripts/upload-and-deploy.sh"
+TEST_ROOT="$(mktemp -d)"
+FAKE_BIN="$TEST_ROOT/bin"
+FAKE_LOG="$TEST_ROOT/calls.log"
+CHECK_COUNT="$TEST_ROOT/release-check-count"
+TEST_IDENTITY="$TEST_ROOT/deploy-key"
+REVISION="3333333333333333333333333333333333333333"
+LOCAL_ENV="$ROOT_DIR/deploy/.env.production"
+CREATED_LOCAL_ENV=false
+
+cleanup() {
+  if [[ "$CREATED_LOCAL_ENV" == "true" && -f "$LOCAL_ENV" ]]; then
+    rm -f -- "$LOCAL_ENV"
+  fi
+  rm -rf -- "$TEST_ROOT"
+}
+trap cleanup EXIT
+
+mkdir -p "$FAKE_BIN"
+touch "$FAKE_LOG" "$TEST_IDENTITY"
+printf '0\n' >"$CHECK_COUNT"
+
+if [[ ! -f "$LOCAL_ENV" ]]; then
+  cp "$ROOT_DIR/deploy/.env.production.example" "$LOCAL_ENV"
+  CREATED_LOCAL_ENV=true
+fi
+
+fail() {
+  printf 'Fehler: %s\n' "$*" >&2
+  exit 1
+}
+
+assert_contains() {
+  local expected="$1"
+  local actual="$2"
+  grep -Fq -- "$expected" <<<"$actual" ||
+    fail "Erwarteter Text fehlt: $expected"
+}
+
+cat >"$FAKE_BIN/git" <<'SH'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+case "$*" in
+  *"status --porcelain --untracked-files=normal"*)
+    exit 0
+    ;;
+  *"rev-parse HEAD"*)
+    printf '%s\n' "$FAKE_REVISION"
+    ;;
+  *)
+    printf 'Unerwarteter Git-Aufruf: %s\n' "$*" >&2
+    exit 90
+    ;;
+esac
+SH
+
+cat >"$FAKE_BIN/rsync" <<'SH'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+printf 'rsync|%s\n' "$*" >>"$FAKE_CALL_LOG"
+SH
+
+cat >"$FAKE_BIN/ssh" <<'SH'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+printf 'ssh|%s\n' "$*" >>"$FAKE_CALL_LOG"
+
+if [[ "$*" == *"WATERMAPS_DEPLOY_REVISION=$FAKE_REVISION"* ]]; then
+  count="$(<"$FAKE_CHECK_COUNT")"
+  count=$((count + 1))
+  printf '%s\n' "$count" >"$FAKE_CHECK_COUNT"
+  ((count >= 2))
+fi
+SH
+
+cat >"$FAKE_BIN/sleep" <<'SH'
+#!/usr/bin/env bash
+set -Eeuo pipefail
+printf 'sleep|%s\n' "$*" >>"$FAKE_CALL_LOG"
+SH
+
+chmod 0755 "$FAKE_BIN/"*
+
+set +e
+output="$(
+  PATH="$FAKE_BIN:$PATH" \
+  FAKE_CALL_LOG="$FAKE_LOG" \
+  FAKE_CHECK_COUNT="$CHECK_COUNT" \
+  FAKE_REVISION="$REVISION" \
+  WATERMAPS_RELEASE_WAIT_SECONDS=60 \
+  WATERMAPS_RELEASE_POLL_SECONDS=5 \
+    "$UPLOAD_SCRIPT" \
+      --host 203.0.113.10 \
+      --identity "$TEST_IDENTITY" \
+      2>&1
+)"
+status=$?
+set -e
+
+[[ "$status" -eq 0 ]] ||
+  fail "Upload-Simulation ist fehlgeschlagen: $output"
+[[ "$(<"$CHECK_COUNT")" == "2" ]] ||
+  fail "Der revisionsgenaue Status wurde nicht bis zur Aktivierung erneut geprüft."
+assert_contains \
+  "systemctl start --no-block watermaps-auto-deploy.service" \
+  "$(<"$FAKE_LOG")"
+assert_contains \
+  "Das revisionsgenaue Erstdeployment läuft unter systemd" \
+  "$output"
+assert_contains \
+  "Commit $REVISION ist auf dem Server aktiv." \
+  "$output"
+assert_contains "sleep|5" "$(<"$FAKE_LOG")"
+
+printf 'Upload/Erstdeployment: systemd-Start und revisionsgenaues Warten: OK\n'
diff --git a/deploy/scripts/upload-and-deploy.sh b/deploy/scripts/upload-and-deploy.sh
index eb5e9be..c573c3f 100755
--- a/deploy/scripts/upload-and-deploy.sh
+++ b/deploy/scripts/upload-and-deploy.sh
@@ -109,6 +109,51 @@ if [[ -n "$WM_REMOTE_SUDO" ]]; then
   rsync_path="$WM_REMOTE_SUDO rsync"
 fi
 
+wait_for_remote_release() {
+  local expected_revision="$1"
+  local timeout_seconds="${WATERMAPS_RELEASE_WAIT_SECONDS:-43200}"
+  local poll_seconds="${WATERMAPS_RELEASE_POLL_SECONDS:-15}"
+  local deadline
+  local attempts=0
+  local remote_check
+
+  [[ "$timeout_seconds" =~ ^[1-9][0-9]*$ ]] &&
+    ((timeout_seconds >= 60 && timeout_seconds <= 46800)) ||
+    wm_local_die "WATERMAPS_RELEASE_WAIT_SECONDS muss zwischen 60 und 46800 liegen."
+  [[ "$poll_seconds" =~ ^[1-9][0-9]*$ ]] &&
+    ((poll_seconds >= 5 && poll_seconds <= 60)) ||
+    wm_local_die "WATERMAPS_RELEASE_POLL_SECONDS muss zwischen 5 und 60 liegen."
+  deadline=$((SECONDS + timeout_seconds))
+
+  remote_check="${remote_prefix}test -L /opt/watermaps/current"
+  remote_check+=" && ${remote_prefix}grep -Fxq"
+  remote_check+=" 'WATERMAPS_DEPLOY_REVISION=$expected_revision'"
+  remote_check+=" /opt/watermaps/deploy/.env.images"
+  remote_check+=" && ${remote_prefix}readlink --canonicalize /opt/watermaps/current"
+  remote_check+=" | grep -Eq '/$expected_revision$'"
+
+  while ((SECONDS < deadline)); do
+    if ssh "${WM_SSH_OPTIONS[@]}" "$WM_SSH_TARGET" "$remote_check" \
+      >/dev/null 2>&1; then
+      printf '[watermaps] Commit %s ist auf dem Server aktiv.\n' \
+        "$expected_revision"
+      return 0
+    fi
+
+    attempts=$((attempts + 1))
+    if ((attempts == 1 || attempts % 4 == 0)); then
+      printf '[watermaps] Das revisionsgenaue Erstdeployment läuft unter systemd; warte auf Commit %s …\n' \
+        "$expected_revision"
+    fi
+    sleep "$poll_seconds"
+  done
+
+  ssh "${WM_SSH_OPTIONS[@]}" "$WM_SSH_TARGET" \
+    "${remote_prefix}systemctl --no-pager --full status watermaps-auto-deploy.service" \
+    || true
+  wm_local_die "Commit $expected_revision wurde nicht innerhalb von ${timeout_seconds}s aktiviert."
+}
+
 printf '[watermaps] Warte auf Cloud-init und das persistente Hetzner-Volume.\n'
 ssh "${WM_SSH_OPTIONS[@]}" "$WM_SSH_TARGET" \
   "cloud-init status --wait && ${remote_prefix}systemctl start watermaps-volume-setup.service && mountpoint --quiet /srv/watermaps-data"
@@ -148,10 +193,9 @@ fi
 
 remote_command="${remote_prefix}chmod +x /opt/watermaps/deploy/scripts/*.sh"
 remote_command+=" && ${remote_prefix}/opt/watermaps/deploy/scripts/bootstrap-server.sh"
-remote_command+=" && ${remote_prefix}WATERMAPS_ENV_FILE=/opt/watermaps/deploy/.env.production /opt/watermaps/deploy/scripts/auto-deploy.sh"
-remote_command+=" && ${remote_prefix}test -L /opt/watermaps/current"
-remote_command+=" && ${remote_prefix}grep -Fxq 'WATERMAPS_DEPLOY_REVISION=$revision' /opt/watermaps/deploy/.env.images"
+remote_command+=" && ${remote_prefix}systemctl start --no-block watermaps-auto-deploy.service"
 ssh "${WM_SSH_OPTIONS[@]}" "$WM_SSH_TARGET" "$remote_command"
+wait_for_remote_release "$revision"
 
 if [[ "$run_go_live" == "true" ]]; then
   ssh "${WM_SSH_OPTIONS[@]}" "$WM_SSH_TARGET" \
diff --git a/package.json b/package.json
index 93d7fca..c0ca3d8 100644
--- a/package.json
+++ b/package.json
@@ -22,7 +22,7 @@
     "setup:local-routing": "./scripts/setup-local-routing.sh",
     "sync:euris-locks": "node scripts/sync-euris-locks.mjs",
     "test": "npm run build --workspace @watermaps/shared && npm run test --workspace @watermaps/shared && npm run test --workspace @watermaps/api && npm run test --workspace @watermaps/web && npm run test:local-routing && npm run test:deployment",
-    "test:deployment": "bash deploy/scripts/tests/image-references.test.sh && bash deploy/scripts/tests/auto-deploy.test.sh && bash deploy/scripts/tests/route-data-helpers.test.sh && bash deploy/scripts/tests/update-route-data.test.sh",
+    "test:deployment": "bash deploy/scripts/tests/image-references.test.sh && bash deploy/scripts/tests/auto-deploy.test.sh && bash deploy/scripts/tests/upload-and-deploy.test.sh && bash deploy/scripts/tests/route-data-helpers.test.sh && bash deploy/scripts/tests/update-route-data.test.sh",
     "test:e2e": "npm run test:e2e --workspace @watermaps/web",
     "test:local-routing": "PYTHONPATH=.tools/python python3 -m unittest discover -s scripts/tests -p 'test_*.py'",
     "typecheck": "npm run build --workspace @watermaps/shared && npm run typecheck --workspace @watermaps/shared && npm run typecheck --workspace @watermaps/api && npm run typecheck --workspace @watermaps/web"