Make first deployment race-safe
This commit is contained in:
BuTzZ
@@ -134,6 +134,10 @@ weder Git noch Node/npm oder einen Repository-Checkout.
|
||||
Der erste Datenaufbau lädt die Geofabrik-Extrakte für Deutschland und die
|
||||
Niederlande, baut den Fahrroutenindex und importiert daraus die
|
||||
`marine_features`. Je nach Serverleistung kann dies längere Zeit dauern.
|
||||
Der Erstlauf wird dafür als begrenzter systemd-Job gestartet; das lokale
|
||||
Upload-Skript wartet über wiederverbindbare SSH-Prüfungen auf exakt den
|
||||
angeforderten Commit. So konkurriert der initiale Aufruf nicht mit dem
|
||||
Auto-Deploy-Timer und ein kurzzeitiger SSH-Abbruch beendet den Import nicht.
|
||||
App und Nginx werden beim ersten Release erst gestartet, wenn sowohl der
|
||||
Routingindex als auch ein nicht leerer, zur PBF-Prüfsumme passender Bestand an
|
||||
Häfen, Schleusen und Brücken geprüft wurde. Ein HTTP-Healthcheck allein kann
|
||||
|
||||
@@ -64,7 +64,13 @@ wm_local_resolve_ssh() {
|
||||
esac
|
||||
|
||||
WM_SSH_TARGET="$WM_SSH_USER@$WM_SERVER_IPV4"
|
||||
WM_SSH_OPTIONS=(-o BatchMode=yes -o StrictHostKeyChecking=accept-new)
|
||||
WM_SSH_OPTIONS=(
|
||||
-o BatchMode=yes
|
||||
-o StrictHostKeyChecking=accept-new
|
||||
-o ServerAliveInterval=30
|
||||
-o ServerAliveCountMax=20
|
||||
-o TCPKeepAlive=yes
|
||||
)
|
||||
if [[ -n "$WM_SSH_IDENTITY" ]]; then
|
||||
WM_SSH_OPTIONS+=(-i "$WM_SSH_IDENTITY" -o IdentitiesOnly=yes)
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
|
||||
UPLOAD_SCRIPT="$ROOT_DIR/deploy/scripts/upload-and-deploy.sh"
|
||||
TEST_ROOT="$(mktemp -d)"
|
||||
FAKE_BIN="$TEST_ROOT/bin"
|
||||
FAKE_LOG="$TEST_ROOT/calls.log"
|
||||
CHECK_COUNT="$TEST_ROOT/release-check-count"
|
||||
TEST_IDENTITY="$TEST_ROOT/deploy-key"
|
||||
REVISION="3333333333333333333333333333333333333333"
|
||||
LOCAL_ENV="$ROOT_DIR/deploy/.env.production"
|
||||
CREATED_LOCAL_ENV=false
|
||||
|
||||
cleanup() {
|
||||
if [[ "$CREATED_LOCAL_ENV" == "true" && -f "$LOCAL_ENV" ]]; then
|
||||
rm -f -- "$LOCAL_ENV"
|
||||
fi
|
||||
rm -rf -- "$TEST_ROOT"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
mkdir -p "$FAKE_BIN"
|
||||
touch "$FAKE_LOG" "$TEST_IDENTITY"
|
||||
printf '0\n' >"$CHECK_COUNT"
|
||||
|
||||
if [[ ! -f "$LOCAL_ENV" ]]; then
|
||||
cp "$ROOT_DIR/deploy/.env.production.example" "$LOCAL_ENV"
|
||||
CREATED_LOCAL_ENV=true
|
||||
fi
|
||||
|
||||
fail() {
|
||||
printf 'Fehler: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
assert_contains() {
|
||||
local expected="$1"
|
||||
local actual="$2"
|
||||
grep -Fq -- "$expected" <<<"$actual" ||
|
||||
fail "Erwarteter Text fehlt: $expected"
|
||||
}
|
||||
|
||||
cat >"$FAKE_BIN/git" <<'SH'
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
case "$*" in
|
||||
*"status --porcelain --untracked-files=normal"*)
|
||||
exit 0
|
||||
;;
|
||||
*"rev-parse HEAD"*)
|
||||
printf '%s\n' "$FAKE_REVISION"
|
||||
;;
|
||||
*)
|
||||
printf 'Unerwarteter Git-Aufruf: %s\n' "$*" >&2
|
||||
exit 90
|
||||
;;
|
||||
esac
|
||||
SH
|
||||
|
||||
cat >"$FAKE_BIN/rsync" <<'SH'
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
printf 'rsync|%s\n' "$*" >>"$FAKE_CALL_LOG"
|
||||
SH
|
||||
|
||||
cat >"$FAKE_BIN/ssh" <<'SH'
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
printf 'ssh|%s\n' "$*" >>"$FAKE_CALL_LOG"
|
||||
|
||||
if [[ "$*" == *"WATERMAPS_DEPLOY_REVISION=$FAKE_REVISION"* ]]; then
|
||||
count="$(<"$FAKE_CHECK_COUNT")"
|
||||
count=$((count + 1))
|
||||
printf '%s\n' "$count" >"$FAKE_CHECK_COUNT"
|
||||
((count >= 2))
|
||||
fi
|
||||
SH
|
||||
|
||||
cat >"$FAKE_BIN/sleep" <<'SH'
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
printf 'sleep|%s\n' "$*" >>"$FAKE_CALL_LOG"
|
||||
SH
|
||||
|
||||
chmod 0755 "$FAKE_BIN/"*
|
||||
|
||||
set +e
|
||||
output="$(
|
||||
PATH="$FAKE_BIN:$PATH" \
|
||||
FAKE_CALL_LOG="$FAKE_LOG" \
|
||||
FAKE_CHECK_COUNT="$CHECK_COUNT" \
|
||||
FAKE_REVISION="$REVISION" \
|
||||
WATERMAPS_RELEASE_WAIT_SECONDS=60 \
|
||||
WATERMAPS_RELEASE_POLL_SECONDS=5 \
|
||||
"$UPLOAD_SCRIPT" \
|
||||
--host 203.0.113.10 \
|
||||
--identity "$TEST_IDENTITY" \
|
||||
2>&1
|
||||
)"
|
||||
status=$?
|
||||
set -e
|
||||
|
||||
[[ "$status" -eq 0 ]] ||
|
||||
fail "Upload-Simulation ist fehlgeschlagen: $output"
|
||||
[[ "$(<"$CHECK_COUNT")" == "2" ]] ||
|
||||
fail "Der revisionsgenaue Status wurde nicht bis zur Aktivierung erneut geprüft."
|
||||
assert_contains \
|
||||
"systemctl start --no-block watermaps-auto-deploy.service" \
|
||||
"$(<"$FAKE_LOG")"
|
||||
assert_contains \
|
||||
"Das revisionsgenaue Erstdeployment läuft unter systemd" \
|
||||
"$output"
|
||||
assert_contains \
|
||||
"Commit $REVISION ist auf dem Server aktiv." \
|
||||
"$output"
|
||||
assert_contains "sleep|5" "$(<"$FAKE_LOG")"
|
||||
|
||||
printf 'Upload/Erstdeployment: systemd-Start und revisionsgenaues Warten: OK\n'
|
||||
@@ -109,6 +109,51 @@ if [[ -n "$WM_REMOTE_SUDO" ]]; then
|
||||
rsync_path="$WM_REMOTE_SUDO rsync"
|
||||
fi
|
||||
|
||||
wait_for_remote_release() {
|
||||
local expected_revision="$1"
|
||||
local timeout_seconds="${WATERMAPS_RELEASE_WAIT_SECONDS:-43200}"
|
||||
local poll_seconds="${WATERMAPS_RELEASE_POLL_SECONDS:-15}"
|
||||
local deadline
|
||||
local attempts=0
|
||||
local remote_check
|
||||
|
||||
[[ "$timeout_seconds" =~ ^[1-9][0-9]*$ ]] &&
|
||||
((timeout_seconds >= 60 && timeout_seconds <= 46800)) ||
|
||||
wm_local_die "WATERMAPS_RELEASE_WAIT_SECONDS muss zwischen 60 und 46800 liegen."
|
||||
[[ "$poll_seconds" =~ ^[1-9][0-9]*$ ]] &&
|
||||
((poll_seconds >= 5 && poll_seconds <= 60)) ||
|
||||
wm_local_die "WATERMAPS_RELEASE_POLL_SECONDS muss zwischen 5 und 60 liegen."
|
||||
deadline=$((SECONDS + timeout_seconds))
|
||||
|
||||
remote_check="${remote_prefix}test -L /opt/watermaps/current"
|
||||
remote_check+=" && ${remote_prefix}grep -Fxq"
|
||||
remote_check+=" 'WATERMAPS_DEPLOY_REVISION=$expected_revision'"
|
||||
remote_check+=" /opt/watermaps/deploy/.env.images"
|
||||
remote_check+=" && ${remote_prefix}readlink --canonicalize /opt/watermaps/current"
|
||||
remote_check+=" | grep -Eq '/$expected_revision$'"
|
||||
|
||||
while ((SECONDS < deadline)); do
|
||||
if ssh "${WM_SSH_OPTIONS[@]}" "$WM_SSH_TARGET" "$remote_check" \
|
||||
>/dev/null 2>&1; then
|
||||
printf '[watermaps] Commit %s ist auf dem Server aktiv.\n' \
|
||||
"$expected_revision"
|
||||
return 0
|
||||
fi
|
||||
|
||||
attempts=$((attempts + 1))
|
||||
if ((attempts == 1 || attempts % 4 == 0)); then
|
||||
printf '[watermaps] Das revisionsgenaue Erstdeployment läuft unter systemd; warte auf Commit %s …\n' \
|
||||
"$expected_revision"
|
||||
fi
|
||||
sleep "$poll_seconds"
|
||||
done
|
||||
|
||||
ssh "${WM_SSH_OPTIONS[@]}" "$WM_SSH_TARGET" \
|
||||
"${remote_prefix}systemctl --no-pager --full status watermaps-auto-deploy.service" \
|
||||
|| true
|
||||
wm_local_die "Commit $expected_revision wurde nicht innerhalb von ${timeout_seconds}s aktiviert."
|
||||
}
|
||||
|
||||
printf '[watermaps] Warte auf Cloud-init und das persistente Hetzner-Volume.\n'
|
||||
ssh "${WM_SSH_OPTIONS[@]}" "$WM_SSH_TARGET" \
|
||||
"cloud-init status --wait && ${remote_prefix}systemctl start watermaps-volume-setup.service && mountpoint --quiet /srv/watermaps-data"
|
||||
@@ -148,10 +193,9 @@ fi
|
||||
|
||||
remote_command="${remote_prefix}chmod +x /opt/watermaps/deploy/scripts/*.sh"
|
||||
remote_command+=" && ${remote_prefix}/opt/watermaps/deploy/scripts/bootstrap-server.sh"
|
||||
remote_command+=" && ${remote_prefix}WATERMAPS_ENV_FILE=/opt/watermaps/deploy/.env.production /opt/watermaps/deploy/scripts/auto-deploy.sh"
|
||||
remote_command+=" && ${remote_prefix}test -L /opt/watermaps/current"
|
||||
remote_command+=" && ${remote_prefix}grep -Fxq 'WATERMAPS_DEPLOY_REVISION=$revision' /opt/watermaps/deploy/.env.images"
|
||||
remote_command+=" && ${remote_prefix}systemctl start --no-block watermaps-auto-deploy.service"
|
||||
ssh "${WM_SSH_OPTIONS[@]}" "$WM_SSH_TARGET" "$remote_command"
|
||||
wait_for_remote_release "$revision"
|
||||
|
||||
if [[ "$run_go_live" == "true" ]]; then
|
||||
ssh "${WM_SSH_OPTIONS[@]}" "$WM_SSH_TARGET" \
|
||||
|
||||
+1
-1
@@ -22,7 +22,7 @@
|
||||
"setup:local-routing": "./scripts/setup-local-routing.sh",
|
||||
"sync:euris-locks": "node scripts/sync-euris-locks.mjs",
|
||||
"test": "npm run build --workspace @watermaps/shared && npm run test --workspace @watermaps/shared && npm run test --workspace @watermaps/api && npm run test --workspace @watermaps/web && npm run test:local-routing && npm run test:deployment",
|
||||
"test:deployment": "bash deploy/scripts/tests/image-references.test.sh && bash deploy/scripts/tests/auto-deploy.test.sh && bash deploy/scripts/tests/route-data-helpers.test.sh && bash deploy/scripts/tests/update-route-data.test.sh",
|
||||
"test:deployment": "bash deploy/scripts/tests/image-references.test.sh && bash deploy/scripts/tests/auto-deploy.test.sh && bash deploy/scripts/tests/upload-and-deploy.test.sh && bash deploy/scripts/tests/route-data-helpers.test.sh && bash deploy/scripts/tests/update-route-data.test.sh",
|
||||
"test:e2e": "npm run test:e2e --workspace @watermaps/web",
|
||||
"test:local-routing": "PYTHONPATH=.tools/python python3 -m unittest discover -s scripts/tests -p 'test_*.py'",
|
||||
"typecheck": "npm run build --workspace @watermaps/shared && npm run typecheck --workspace @watermaps/shared && npm run typecheck --workspace @watermaps/api && npm run typecheck --workspace @watermaps/web"
|
||||
|
||||
Reference in New Issue
Block a user