diff --git a/.dockerignore b/.dockerignore index 837019c..e541f13 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,16 @@ node_modules .tools +.git +**/.terraform +**/*.tfstate +**/*.tfstate.* +**/*.tfplan +**/*.tfvars +**/*.tfvars.json +**/.env +**/.env.* +**/.gitea-token +**/.gitea-*-token dist .vite coverage @@ -12,6 +23,4 @@ data/geofabrik/*.osm.pbf.md5 data/geofabrik/*.part data/geofabrik/*.expected-md5 data/local/*.json -.env -.env.* .DS_Store diff --git a/.gitea/workflows/container-images.yml b/.gitea/workflows/container-images.yml index 55c1316..b375db9 100644 --- a/.gitea/workflows/container-images.yml +++ b/.gitea/workflows/container-images.yml @@ -18,10 +18,10 @@ jobs: test: runs-on: ubuntu-latest container: - image: node:22-bookworm + image: node:22-bookworm@sha256:5647be709086c696ff32edaaf1c70cd26d1da6ab2b39c32f3c7b4c4a31957e37 steps: - name: Check out source - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - name: Install test dependencies run: | @@ -45,10 +45,13 @@ jobs: if: ${{ gitea.event_name == 'push' }} needs: - test + concurrency: + group: watermaps-production-publish + cancel-in-progress: false runs-on: ubuntu-latest steps: - name: Check out source - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - name: Log in to the Gitea Container Registry env: @@ -60,11 +63,11 @@ jobs: printf '%s\n' "$REGISTRY_TOKEN" | docker login "$REGISTRY" --username "$REGISTRY_USERNAME" --password-stdin - - name: Build images for the triggering commit + - name: Build application images for the triggering commit env: REVISION: ${{ gitea.sha }} run: | - test "$(printf '%s' "$REVISION" | wc -c)" -eq 40 + [[ "$REVISION" =~ ^[0-9a-f]{40}$ ]] docker build \ --label "org.opencontainers.image.revision=$REVISION" \ --label "org.opencontainers.image.source=${{ gitea.server_url }}/${{ gitea.repository }}" \ @@ -77,12 +80,68 @@ jobs: --tag "$REGISTRY/$REGISTRY_OWNER/watermaps-route-data:$REVISION" \ . - - name: Push immutable commit images + - name: Push immutable images and build their release manifest env: REVISION: ${{ gitea.sha }} run: | - docker push "$REGISTRY/$REGISTRY_OWNER/watermaps:$REVISION" - docker push "$REGISTRY/$REGISTRY_OWNER/watermaps-route-data:$REVISION" + set -Eeuo pipefail + + push_and_resolve() { + local tagged_image="$1" + local push_output digest repository + + if ! push_output="$(docker push "$tagged_image" 2>&1)"; then + printf '%s\n' "$push_output" >&2 + return 1 + fi + printf '%s\n' "$push_output" >&2 + digest="$( + printf '%s\n' "$push_output" | + sed -nE 's/.*digest: (sha256:[0-9a-f]{64}).*/\1/p' | + tail -n 1 + )" + [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] + repository="${tagged_image%:*}" + printf '%s@%s\n' "$repository" "$digest" + } + + app_image="$( + push_and_resolve \ + "$REGISTRY/$REGISTRY_OWNER/watermaps:$REVISION" + )" + route_data_image="$( + push_and_resolve \ + "$REGISTRY/$REGISTRY_OWNER/watermaps-route-data:$REVISION" + )" + + docker build \ + --file deploy/release.Dockerfile \ + --build-arg "REVISION=$REVISION" \ + --build-arg "SOURCE=${{ gitea.server_url }}/${{ gitea.repository }}" \ + --build-arg "APP_IMAGE=$app_image" \ + --build-arg "ROUTE_DATA_IMAGE=$route_data_image" \ + --tag "$REGISTRY/$REGISTRY_OWNER/watermaps-release:$REVISION" \ + . + docker push "$REGISTRY/$REGISTRY_OWNER/watermaps-release:$REVISION" + + - name: Publish the atomic main release pointer + env: + REVISION: ${{ gitea.sha }} + run: | + current_main_revision="$( + git ls-remote --exit-code origin refs/heads/main | + awk 'NR == 1 { print $1 }' + )" + if [[ "$current_main_revision" != "$REVISION" ]]; then + printf 'Commit %s is no longer the main tip; immutable images stay published without moving main.\n' \ + "$REVISION" + exit 0 + fi + + docker tag \ + "$REGISTRY/$REGISTRY_OWNER/watermaps-release:$REVISION" \ + "$REGISTRY/$REGISTRY_OWNER/watermaps-release:main" + docker push "$REGISTRY/$REGISTRY_OWNER/watermaps-release:main" - name: Log out from the registry if: always() diff --git a/Dockerfile b/Dockerfile index 36efd24..0be0ed6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:22-bookworm-slim AS build +FROM node:22-bookworm-slim@sha256:6c74791e557ce11fc957704f6d4fe134a7bc8d6f5ca4403205b2966bd488f6b3 AS build WORKDIR /app @@ -15,7 +15,7 @@ COPY packages ./packages RUN npm run build \ && npm prune --omit=dev -FROM node:22-bookworm-slim AS runtime +FROM node:22-bookworm-slim@sha256:6c74791e557ce11fc957704f6d4fe134a7bc8d6f5ca4403205b2966bd488f6b3 AS runtime ENV NODE_ENV=production \ HOST=0.0.0.0 \ diff --git a/database/migrations/0001_initial.sql b/database/migrations/0001_initial.sql new file mode 100644 index 0000000..2f61f12 --- /dev/null +++ b/database/migrations/0001_initial.sql @@ -0,0 +1,70 @@ +-- Initiales, wiederholbar ausführbares Watermaps-Schema. +CREATE EXTENSION IF NOT EXISTS postgis; + +CREATE TABLE IF NOT EXISTS marine_features ( + id bigserial PRIMARY KEY, + layer text NOT NULL CHECK (layer IN ('seamarks', 'bridges', 'locks', 'harbours', 'waterways', 'fairways')), + source text NOT NULL DEFAULT 'manual', + source_id text, + name text, + properties jsonb NOT NULL DEFAULT '{}'::jsonb, + geom geometry(Geometry, 4326) NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS marine_features_geom_idx ON marine_features USING gist (geom); +CREATE INDEX IF NOT EXISTS marine_features_layer_idx ON marine_features (layer); +CREATE INDEX IF NOT EXISTS marine_features_properties_idx ON marine_features USING gin (properties); +CREATE UNIQUE INDEX IF NOT EXISTS marine_features_source_layer_uidx + ON marine_features (source, source_id, layer) + WHERE source_id IS NOT NULL; + +CREATE TABLE IF NOT EXISTS marine_enrichment_attempts ( + id bigserial PRIMARY KEY, + layer text NOT NULL CHECK (layer IN ('locks', 'harbours')), + original_source text NOT NULL, + original_source_id text NOT NULL, + provider text NOT NULL, + query_fingerprint text NOT NULL, + query text, + status text NOT NULL CHECK ( + status IN ( + 'success', + 'no_match', + 'ambiguous', + 'no_contacts', + 'fetch_failed', + 'provider_blocked', + 'invalid_candidate' + ) + ), + result_url text, + details jsonb NOT NULL DEFAULT '{}'::jsonb, + attempted_at timestamptz NOT NULL DEFAULT now(), + retry_after timestamptz, + UNIQUE (layer, original_source, original_source_id, provider, query_fingerprint) +); + +CREATE INDEX IF NOT EXISTS marine_enrichment_attempts_retry_idx + ON marine_enrichment_attempts (provider, retry_after); + +CREATE TABLE IF NOT EXISTS marine_fairway_edges ( + id bigserial PRIMARY KEY, + source text NOT NULL DEFAULT 'osm', + source_id text, + name text, + min_depth_m numeric, + properties jsonb NOT NULL DEFAULT '{}'::jsonb, + geom geometry(LineString, 4326) NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS marine_fairway_edges_geom_idx ON marine_fairway_edges USING gist (geom); +CREATE INDEX IF NOT EXISTS marine_fairway_edges_source_idx ON marine_fairway_edges (source, source_id); +CREATE UNIQUE INDEX IF NOT EXISTS marine_fairway_edges_source_uidx + ON marine_fairway_edges (source, source_id) + WHERE source_id IS NOT NULL; + +CREATE OR REPLACE VIEW marine_features_mvt AS +SELECT id, layer, source, name, properties, geom +FROM marine_features; diff --git a/database/schema.sql b/database/schema.sql index 652e975..9ce4a63 100644 --- a/database/schema.sql +++ b/database/schema.sql @@ -1,69 +1,2 @@ -CREATE EXTENSION IF NOT EXISTS postgis; - -CREATE TABLE IF NOT EXISTS marine_features ( - id bigserial PRIMARY KEY, - layer text NOT NULL CHECK (layer IN ('seamarks', 'bridges', 'locks', 'harbours', 'waterways', 'fairways')), - source text NOT NULL DEFAULT 'manual', - source_id text, - name text, - properties jsonb NOT NULL DEFAULT '{}'::jsonb, - geom geometry(Geometry, 4326) NOT NULL, - updated_at timestamptz NOT NULL DEFAULT now() -); - -CREATE INDEX IF NOT EXISTS marine_features_geom_idx ON marine_features USING gist (geom); -CREATE INDEX IF NOT EXISTS marine_features_layer_idx ON marine_features (layer); -CREATE INDEX IF NOT EXISTS marine_features_properties_idx ON marine_features USING gin (properties); -CREATE UNIQUE INDEX IF NOT EXISTS marine_features_source_layer_uidx - ON marine_features (source, source_id, layer) - WHERE source_id IS NOT NULL; - -CREATE TABLE IF NOT EXISTS marine_enrichment_attempts ( - id bigserial PRIMARY KEY, - layer text NOT NULL CHECK (layer IN ('locks', 'harbours')), - original_source text NOT NULL, - original_source_id text NOT NULL, - provider text NOT NULL, - query_fingerprint text NOT NULL, - query text, - status text NOT NULL CHECK ( - status IN ( - 'success', - 'no_match', - 'ambiguous', - 'no_contacts', - 'fetch_failed', - 'provider_blocked', - 'invalid_candidate' - ) - ), - result_url text, - details jsonb NOT NULL DEFAULT '{}'::jsonb, - attempted_at timestamptz NOT NULL DEFAULT now(), - retry_after timestamptz, - UNIQUE (layer, original_source, original_source_id, provider, query_fingerprint) -); - -CREATE INDEX IF NOT EXISTS marine_enrichment_attempts_retry_idx - ON marine_enrichment_attempts (provider, retry_after); - -CREATE TABLE IF NOT EXISTS marine_fairway_edges ( - id bigserial PRIMARY KEY, - source text NOT NULL DEFAULT 'osm', - source_id text, - name text, - min_depth_m numeric, - properties jsonb NOT NULL DEFAULT '{}'::jsonb, - geom geometry(LineString, 4326) NOT NULL, - updated_at timestamptz NOT NULL DEFAULT now() -); - -CREATE INDEX IF NOT EXISTS marine_fairway_edges_geom_idx ON marine_fairway_edges USING gist (geom); -CREATE INDEX IF NOT EXISTS marine_fairway_edges_source_idx ON marine_fairway_edges (source, source_id); -CREATE UNIQUE INDEX IF NOT EXISTS marine_fairway_edges_source_uidx - ON marine_fairway_edges (source, source_id) - WHERE source_id IS NOT NULL; - -CREATE OR REPLACE VIEW marine_features_mvt AS -SELECT id, layer, source, name, properties, geom -FROM marine_features; +\set ON_ERROR_STOP on +\ir migrations/0001_initial.sql diff --git a/deploy/.env.production.example b/deploy/.env.production.example index 07596a9..a01cb98 100644 --- a/deploy/.env.production.example +++ b/deploy/.env.production.example @@ -16,20 +16,22 @@ WATERMAPS_RUNTIME_DIR=/srv/watermaps-runtime # Verhindert Speicherabbrüche beim vollständigen DE/NL-Indexaufbau auf 4-GB-Servern. WATERMAPS_SWAP_SIZE_GB=4 +WATERMAPS_RELEASE_RETENTION=5 # Persistente PostGIS-Datenbank für Häfen, Schleusen, Brücken und Kontaktdaten. # Vor dem ersten Deployment beispielsweise mit `openssl rand -hex 32` # erzeugen. Das Secret muss mindestens 24 Zeichen lang sein. WATERMAPS_POSTGRES_PASSWORD=REPLACE_WITH_RANDOM_32_BYTE_HEX_SECRET -WATERMAPS_POSTGRES_IMAGE=postgis/postgis:16-3.4 +WATERMAPS_POSTGRES_IMAGE=postgis/postgis@sha256:44126d872ac91993766c341e369c539e8196614321765d36a6f1bab0419a5fa5 # Gitea Container Registry. Die commitgenauen App-Referenzen werden beim # Deployment separat erzeugt und niemals hier von Hand auf `latest` gesetzt. WATERMAPS_REGISTRY=gitea.incoso.eu WATERMAPS_REGISTRY_OWNER=kevin_janssen +WATERMAPS_GITEA_REPOSITORY=kevin_janssen/watermaps -WATERMAPS_NGINX_IMAGE=nginx:1.30.4-alpine -WATERMAPS_CERTBOT_IMAGE=certbot/certbot:v5.7.0 +WATERMAPS_NGINX_IMAGE=nginx@sha256:97d490c12ba55b4946b01546d1c3ed324e8d41ab1c9fcb2a616aa470620e5b46 +WATERMAPS_CERTBOT_IMAGE=certbot/certbot@sha256:34ee91d2f43008eb78a007d22f23ed4b2eaa9a454cb27ca2c042b49527a695b4 # Nur auf true setzen, wenn bei jedem Deployment Deutschland und die # Niederlande erneut geprüft und der Routingindex neu gebaut werden sollen. diff --git a/deploy/.gitignore b/deploy/.gitignore index f1a9738..ad74a3b 100644 --- a/deploy/.gitignore +++ b/deploy/.gitignore @@ -1,4 +1,7 @@ .env.production .env.images .env.images.candidate +.gitea-actions-config-token +.gitea-registry-publish-token +.gitea-token !.env.production.example diff --git a/deploy/README.md b/deploy/README.md index 7ce1dec..2acd787 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -27,52 +27,87 @@ diese Datei. Er bleibt lokal in der ignorierten Datei `infra/opentofu/terraform.tfvars` (alternativ kann der Provider `TF_VAR_hcloud_token` lesen). -Die private SSH-Keydatei wird ebenfalls nicht gespeichert. Sie kann beim -Deployment mit `--identity` oder über `WATERMAPS_SSH_KEY` angegeben werden. -Falls OpenTofu keinen Output `ssh_private_key_path` bereitstellt und der Key -nicht bereits über den SSH-Agenten verfügbar ist, ist eine dieser beiden -Angaben erforderlich. Der vorbereitete SSH-Benutzer heißt standardmäßig -`deploy`; die privilegierten Installationsschritte laufen über dessen -passwortloses `sudo`. +Die private SSH-Keydatei wird ebenfalls nicht gespeichert. Standardmäßig +verwenden die lokalen Deployment-Skripte +`~/.ssh/watermaps_deploy_ed25519`; alternativ kann sie mit `--identity` oder +über `WATERMAPS_SSH_KEY` angegeben werden. Der vorbereitete SSH-Benutzer heißt +standardmäßig `deploy`; die privilegierten Installationsschritte laufen über +dessen passwortloses `sudo`. ## Gitea Actions und Container Registry Der Workflow `.gitea/workflows/container-images.yml` führt Typechecks und Tests -aus. Nach einem erfolgreichen Push auf `main` baut und veröffentlicht er zwei +aus. Nach einem erfolgreichen Push auf `main` baut und veröffentlicht er drei OCI-Images unter der vollständigen Git-Commit-SHA: ```text gitea.incoso.eu/kevin_janssen/watermaps: gitea.incoso.eu/kevin_janssen/watermaps-route-data: +gitea.incoso.eu/kevin_janssen/watermaps-release: ``` +Das dritte Image ist ein nicht ausführbares Release-Bundle mit Compose-Datei, +Nginx-Konfiguration, Systemd-Units, Deployment-Skripten und versionierten +Datenbankmigrationen. Sein geprüftes Manifest bindet die exakten +Registry-Digests von App und Routingdaten; `SHA256SUMS` schützt alle Dateien des +Bundles. Erst nachdem alle drei Commit-Images veröffentlicht wurden, wird +`watermaps-release:main` als einzelner atomarer Release-Pointer verschoben. +App- und Routingdaten-Images werden nie über bewegliche Tags gestartet. + In Gitea müssen Repository Actions aktiviert und ein Docker-fähiger `ubuntu-latest`-Runner registriert sein. Unter `Repository → Settings → Actions → Secrets` werden benötigt: - `REGISTRY_USERNAME`: Gitea-Benutzer, dem die Packages gehören -- `REGISTRY_TOKEN`: Personal Access Token mit `package: Read and Write` +- `REGISTRY_TOKEN`: eigener CI-Personal-Access-Token ausschließlich mit + `write:package` -Der separate PAT ist derzeit nötig, weil Giteas eingebauter Job-Token -OCI-Pakete noch nicht zuverlässig veröffentlichen kann. Für Pull Requests -werden nur Tests ausgeführt; Registry-Secrets werden dabei nicht verwendet. +Der separate PAT ist nötig, weil Gitea 1.26 mit dem eingebauten Job-Token noch +keine OCI-Pakete veröffentlichen kann. Für Pull Requests werden nur Tests +ausgeführt; Registry-Secrets werden dabei nicht verwendet. + +Die Secrets lassen sich mit zwei getrennten, lokal ignorierten Token-Dateien +konfigurieren: + +- `deploy/.gitea-actions-config-token`: kurzlebiger Konfigurations-PAT nur mit + `write:repository`, um die Actions-Secrets über die Repository-API zu setzen +- `deploy/.gitea-registry-publish-token`: CI-PAT nur mit `write:package`, der + anschließend als `REGISTRY_TOKEN` in Gitea Actions hinterlegt wird + +Beide Dateien enthalten jeweils nur den Token in einer einzelnen Zeile und +müssen exakt Dateimodus 600 haben: + +```bash +chmod 600 \ + deploy/.gitea-actions-config-token \ + deploy/.gitea-registry-publish-token +./deploy/scripts/configure-gitea-actions.sh +``` + +Das Skript nutzt den Konfigurations-PAT ausschließlich lokal für den API-Aufruf +und speichert ihn ausdrücklich nicht als Actions-Secret. Anschließend sollten +beide lokalen Token-Dateien gelöscht werden; Gitea zeigt den gespeicherten +Registry-Secret-Wert nicht wieder an. ## Deployment Nach einem erfolgreichen Image-Build liest das Skript standardmäßig den OpenTofu-Output `server_ipv4`. Es erlaubt ausschließlich einen sauberen, -vollständig committeten Git-Stand und überträgt nur den kleinen Ordner -`deploy/` sowie das kanonische `database/schema.sql`, nicht den -Anwendungsquellcode. Die Image-Tags entsprechen exakt `git rev-parse HEAD`. +vollständig committeten Git-Stand und überträgt für die Ersteinrichtung nur +Bootstrap-Dateien und die getrennte Produktionskonfiguration, nicht den +Anwendungsquellcode. Der Server bezieht das kanonische Release anschließend +selbst aus der Gitea Registry und prüft, dass es exakt `git rev-parse HEAD` +entspricht. -Ist die Registry privat, werden einmalig beziehungsweise nach Tokenwechsel -lokale Pull-Zugangsdaten mitgegeben: +Pakete des öffentlichen Gitea-Benutzers können ohne Server-Credential gelesen +werden. Wird die Registry später privat geschaltet, werden einmalig +beziehungsweise nach Tokenwechsel lokale Pull-Zugangsdaten mitgegeben: ```bash WATERMAPS_REGISTRY_USERNAME=kevin_janssen \ WATERMAPS_REGISTRY_TOKEN='' \ ./deploy/scripts/upload-and-deploy.sh \ - --identity ~/.ssh/watermaps_hetzner_ed25519 + --identity ~/.ssh/watermaps_deploy_ed25519 ``` Der Token benötigt auf dem Produktionsserver nur `package: Read`. Er wird per @@ -84,14 +119,17 @@ Anmeldung gültig ist: ```bash ./deploy/scripts/upload-and-deploy.sh \ - --identity ~/.ssh/watermaps_hetzner_ed25519 + --identity ~/.ssh/watermaps_deploy_ed25519 ``` -Das Serverskript lädt App, den kombinierten Routing-/Feature-Daten-Builder, -PostGIS, Nginx und Certbot mit `docker compose pull`. Die beiden commitgenauen -Gitea-Tags werden anschließend in ihre unveränderlichen Registry-Digests -aufgelöst. Erst danach werden die Container mit `--no-build` gestartet. Der -Produktionsserver benötigt deshalb weder Git noch Node/npm oder den Quellcode. +Das Serverskript lädt das Release-Manifest, App, den kombinierten +Routing-/Feature-Daten-Builder, PostGIS, Nginx und Certbot. App und Builder +werden ausschließlich über die im Release gebundenen Digests gestartet. +Geordnete SQL-Dateien unter `database/migrations/` werden mit +`watermaps_schema_migrations` protokolliert und jeweils in einer Transaktion +ausgeführt. Neue Migrationen müssen nach dem Expand/Contract-Prinzip mit dem +vorherigen App-Release kompatibel bleiben. Der Produktionsserver benötigt +weder Git noch Node/npm oder einen Repository-Checkout. Der erste Datenaufbau lädt die Geofabrik-Extrakte für Deutschland und die Niederlande, baut den Fahrroutenindex und importiert daraus die @@ -131,18 +169,19 @@ HTTP-Anfragen erhalten 404. ## Rollback Jedes erfolgreiche Release wird mit Commit-SHA und den aufgelösten -Image-Digests unter `/srv/watermaps-runtime/deployments/` gespeichert. Scheitert -ein Deployment nach dem Containerwechsel, startet `deploy.sh` automatisch das -vorherige Release und führt die Health-, Routen- und Featuretests erneut aus. -Ein Rollback wechselt nur die unveränderlichen App-/Builder-Images. Die -persistente PostGIS-Datenbank bleibt erhalten; das Schema und der Import sind -aufwärtskompatibel und idempotent. +Image-Digests unter `/srv/watermaps-runtime/deployments/` gespeichert; sein +vollständiges Bundle liegt unter `/srv/watermaps-data/releases/`. +`/opt/watermaps/current` wird erst nach Health-, Routen- und Featuretests +atomar auf ein neues Bundle umgeschaltet. Scheitert ein Deployment, wird das +vorherige unveränderliche Bundle erneut ausgeführt und vollständig geprüft. +Die persistente PostGIS-Datenbank bleibt erhalten; Migrationen sind deshalb +bewusst vorwärtskompatibel zu gestalten. Das unmittelbar vorherige Release lässt sich auch manuell aktivieren: ```bash ./deploy/scripts/remote-rollback.sh \ - --identity ~/.ssh/watermaps_hetzner_ed25519 + --identity ~/.ssh/watermaps_deploy_ed25519 ``` Oder ein bestimmter, bereits erfolgreich deployter Commit: @@ -150,7 +189,7 @@ Oder ein bestimmter, bereits erfolgreich deployter Commit: ```bash ./deploy/scripts/remote-rollback.sh \ --revision 0123456789abcdef0123456789abcdef01234567 \ - --identity ~/.ssh/watermaps_hetzner_ed25519 + --identity ~/.ssh/watermaps_deploy_ed25519 ``` Der Rollback verwendet gespeicherte Digests, nicht einen beweglichen Tag. @@ -167,7 +206,7 @@ A watermaps.incoso.eu Nach der DNS-Propagation wird der Livegang lokal ausgelöst: ```bash -./deploy/scripts/remote-go-live.sh --identity ~/.ssh/watermaps_hetzner_ed25519 +./deploy/scripts/remote-go-live.sh --identity ~/.ssh/watermaps_deploy_ed25519 ``` Das Serverskript prüft, dass sämtliche A-Records ausschließlich auf die @@ -178,7 +217,19 @@ HTTPS um. ## Automatik -`bootstrap-server.sh` installiert zwei systemd-Timer: +`bootstrap-server.sh` installiert drei systemd-Timer: + +- `watermaps-auto-deploy.timer`: prüft ausgehend jede Minute + `watermaps-release:main`. Das Revision-Label muss exakt dem aktuellen + Gitea-`main`-Commit entsprechen. Der Server prüft Manifest, Dateiprüfsummen, + gebundene Image-Digests und OCI-Revisionen und deployt erst nach einer + zweiten Pointer-/Branch-Prüfung. Es ist weder ein eingehender Gitea-Webhook + noch ein SSH-Key im Actions-Runner nötig. Aktives, vorheriges und die über + `WATERMAPS_RELEASE_RETENTION` festgelegte Zahl lokaler Releases bleiben + erhalten; ältere Watermaps-Artefakte werden gezielt bereinigt. Vor dem ersten + Release verwendet ein stabiler Systemd-Einstiegspunkt das Bootstrap-Skript, + danach automatisch das atomar aktivierte `current`. Noch nicht fertige erste + Registry-Builds werden dadurch zuverlässig erneut versucht. - `watermaps-route-update.timer`: täglich neue Deutschland- und Niederlande-Daten. Der Fahrroutenindex wird atomar aktiviert. Der @@ -193,7 +244,7 @@ Ein vollständiger manueller Feature-Neuimport lässt sich über die Option `--force-marine` erzwingen: ```bash -/opt/watermaps/deploy/scripts/update-route-data.sh --force-marine +/opt/watermaps/current/deploy/scripts/update-route-data.sh --force-marine ``` `WATERMAPS_REBUILD_MARINE_DATA=true` in `.env.production` erzwingt den @@ -205,11 +256,12 @@ Status und Logs: ```bash systemctl list-timers 'watermaps-*' +journalctl -u watermaps-auto-deploy.service journalctl -u watermaps-route-update.service journalctl -u watermaps-certbot-renew.service docker compose \ - --project-directory /opt/watermaps \ + --project-directory /opt/watermaps/current \ --env-file /opt/watermaps/deploy/.env.production \ --env-file /opt/watermaps/deploy/.env.images \ - -f /opt/watermaps/deploy/compose.production.yml ps + -f /opt/watermaps/current/deploy/compose.production.yml ps ``` diff --git a/deploy/compose.production.yml b/deploy/compose.production.yml index 67f4d9b..48b4e23 100644 --- a/deploy/compose.production.yml +++ b/deploy/compose.production.yml @@ -62,6 +62,10 @@ services: source: ./database/schema.sql target: /docker-entrypoint-initdb.d/01-schema.sql read_only: true + - type: bind + source: ./database/migrations + target: /docker-entrypoint-initdb.d/migrations + read_only: true restart: unless-stopped init: true security_opt: diff --git a/deploy/release.Dockerfile b/deploy/release.Dockerfile new file mode 100644 index 0000000..43996e1 --- /dev/null +++ b/deploy/release.Dockerfile @@ -0,0 +1,57 @@ +FROM alpine@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce AS bundle + +ARG REVISION +ARG SOURCE +ARG APP_IMAGE +ARG ROUTE_DATA_IMAGE + +WORKDIR /release + +COPY deploy/compose.production.yml /release/deploy/compose.production.yml +COPY deploy/nginx /release/deploy/nginx +COPY deploy/scripts /release/deploy/scripts +COPY deploy/systemd /release/deploy/systemd +COPY database /release/database + +RUN set -eu; \ + case "$REVISION" in \ + *[!0-9a-f]*|'') exit 1 ;; \ + esac; \ + test "${#REVISION}" -eq 40; \ + app_digest="${APP_IMAGE##*@sha256:}"; \ + route_digest="${ROUTE_DATA_IMAGE##*@sha256:}"; \ + test "$app_digest" != "$APP_IMAGE"; \ + test "$route_digest" != "$ROUTE_DATA_IMAGE"; \ + case "$app_digest" in \ + *[!0-9a-f]*|'') exit 1 ;; \ + esac; \ + case "$route_digest" in \ + *[!0-9a-f]*|'') exit 1 ;; \ + esac; \ + test "${#app_digest}" -eq 64; \ + test "${#route_digest}" -eq 64; \ + printf '%s\n' \ + "format_version=1" \ + "revision=$REVISION" \ + "source=$SOURCE" \ + "app_image=$APP_IMAGE" \ + "route_data_image=$ROUTE_DATA_IMAGE" \ + > release.env; \ + { \ + sha256sum release.env; \ + find deploy database -type f -print | LC_ALL=C sort | xargs sha256sum; \ + } > SHA256SUMS + +FROM scratch + +ARG REVISION +ARG SOURCE + +LABEL org.opencontainers.image.title="Watermaps production release bundle" \ + org.opencontainers.image.description="Versioned deployment files, exact image digests and database migrations for Watermaps" \ + org.opencontainers.image.revision="${REVISION}" \ + org.opencontainers.image.source="${SOURCE}" + +COPY --from=bundle /release /release + +CMD ["/release/deploy/scripts/deploy.sh"] diff --git a/deploy/route-data.Dockerfile b/deploy/route-data.Dockerfile index 4f9d1b9..37c2e7b 100644 --- a/deploy/route-data.Dockerfile +++ b/deploy/route-data.Dockerfile @@ -1,4 +1,4 @@ -FROM node:22-bookworm-slim +FROM node:22-bookworm-slim@sha256:6c74791e557ce11fc957704f6d4fe134a7bc8d6f5ca4403205b2966bd488f6b3 RUN apt-get update \ && apt-get install --yes --no-install-recommends \ diff --git a/deploy/scripts/auto-deploy-entrypoint.sh b/deploy/scripts/auto-deploy-entrypoint.sh new file mode 100755 index 0000000..c3321a0 --- /dev/null +++ b/deploy/scripts/auto-deploy-entrypoint.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +install_root="${WATERMAPS_INSTALL_DIR:-/opt/watermaps}" +[[ "$install_root" == /* && "$install_root" != "/" ]] || { + printf 'Ungültiges Watermaps-Installationsverzeichnis: %s\n' "$install_root" >&2 + exit 1 +} + +current_script="$install_root/current/deploy/scripts/auto-deploy.sh" +bootstrap_script="$install_root/deploy/scripts/auto-deploy.sh" + +if [[ -x "$current_script" ]]; then + exec "$current_script" +fi +if [[ -x "$bootstrap_script" ]]; then + exec "$bootstrap_script" +fi + +printf 'Kein Watermaps-Auto-Deploy-Skript gefunden.\n' >&2 +exit 1 diff --git a/deploy/scripts/auto-deploy.sh b/deploy/scripts/auto-deploy.sh new file mode 100755 index 0000000..83dafd8 --- /dev/null +++ b/deploy/scripts/auto-deploy.sh @@ -0,0 +1,414 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +WM_DEPLOY_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=common.sh +source "$WM_DEPLOY_DIR/scripts/common.sh" + +[[ "$(id -u)" -eq 0 ]] || + wm_die "Dieses Skript muss als root ausgeführt werden." +wm_load_env +wm_assert_data_mount + +registry="${WATERMAPS_REGISTRY:-}" +registry_owner="${WATERMAPS_REGISTRY_OWNER:-}" +repository="${WATERMAPS_GITEA_REPOSITORY:-$registry_owner/watermaps}" + +[[ "$registry" =~ ^[a-z0-9][a-z0-9.-]*(:[0-9]{1,5})?$ ]] || + wm_die "WATERMAPS_REGISTRY muss ein Registry-Hostname ohne URL-Schema sein." +[[ "$registry_owner" =~ ^[A-Za-z0-9_.-]+$ ]] || + wm_die "WATERMAPS_REGISTRY_OWNER enthält keinen gültigen Gitea-Besitzer." +[[ "$repository" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || + wm_die "WATERMAPS_GITEA_REPOSITORY muss Besitzer und Repository enthalten." + +release_repository="$registry/$registry_owner/watermaps-release" +release_pointer="$release_repository:main" +app_repository="$registry/$registry_owner/watermaps" +route_data_repository="$registry/$registry_owner/watermaps-route-data" +release_store="$WATERMAPS_DATA_DIR/releases" +install_root="${WATERMAPS_INSTALL_DIR:-/opt/watermaps}" +wm_require_safe_absolute_dir "$install_root" +shared_deploy_dir="$install_root/deploy" +active_images_file="$shared_deploy_dir/.env.images" +current_release_link="$install_root/current" +release_retention="${WATERMAPS_RELEASE_RETENTION:-5}" +[[ "$release_retention" =~ ^[2-9]$|^1[0-9]$|^20$ ]] || + wm_die "WATERMAPS_RELEASE_RETENTION muss zwischen 2 und 20 liegen." + +install -d -m 0755 \ + "$WATERMAPS_RUNTIME_DIR/locks" \ + "$release_store" \ + "$shared_deploy_dir" +exec 8>"$WATERMAPS_RUNTIME_DIR/locks/auto-deploy.lock" +if ! flock --nonblock 8; then + wm_log "Eine automatische Release-Prüfung läuft bereits." + exit 0 +fi + +temporary_release_dir="" +temporary_images_file="" +temporary_current_link="" +release_container_id="" +release_directory="" + +cleanup() { + local status=$? + trap - EXIT HUP INT TERM + set +e + if [[ -n "$release_container_id" ]]; then + docker container rm --force "$release_container_id" >/dev/null 2>&1 + fi + if [[ -n "$temporary_release_dir" && -d "$temporary_release_dir" ]]; then + rm -rf -- "$temporary_release_dir" + fi + if [[ -n "$temporary_images_file" && -f "$temporary_images_file" ]]; then + rm -f -- "$temporary_images_file" + fi + if [[ -n "$temporary_current_link" && -L "$temporary_current_link" ]]; then + rm -f -- "$temporary_current_link" + fi + exit "$status" +} + +trap cleanup EXIT HUP INT TERM + +image_revision() { + local image_reference="$1" + local revision + + revision="$( + docker image inspect \ + --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' \ + "$image_reference" + )" + [[ "$revision" =~ ^[0-9a-f]{40}$ ]] || + wm_die "Image $image_reference enthält kein gültiges OCI-Revision-Label." + printf '%s\n' "$revision" +} + +image_id() { + docker image inspect --format '{{.Id}}' "$1" +} + +gitea_main_revision() { + local api_url + api_url="https://$registry/api/v1/repos/$repository/branches/main" + curl \ + --fail \ + --silent \ + --show-error \ + --location \ + --max-time 30 \ + "$api_url" | + jq --exit-status --raw-output ' + .commit.id + | select(type == "string" and test("^[0-9a-f]{40}$")) + ' +} + +validate_release_bundle() { + local directory="$1" + local expected_revision="$2" + local required_file + local manifest_revision manifest_source manifest_app manifest_route + + for required_file in \ + SHA256SUMS \ + release.env \ + deploy/compose.production.yml \ + deploy/nginx/bootstrap.conf \ + deploy/nginx/https.conf.template \ + deploy/scripts/common.sh \ + deploy/scripts/auto-deploy.sh \ + deploy/scripts/auto-deploy-entrypoint.sh \ + deploy/scripts/bootstrap-server.sh \ + deploy/scripts/deploy.sh \ + deploy/scripts/renew-certificate.sh \ + deploy/scripts/rollback.sh \ + deploy/scripts/update-route-data.sh \ + deploy/systemd/watermaps-auto-deploy.service \ + deploy/systemd/watermaps-auto-deploy.timer \ + deploy/systemd/watermaps-certbot-renew.service \ + deploy/systemd/watermaps-certbot-renew.timer \ + deploy/systemd/watermaps-route-update.service \ + deploy/systemd/watermaps-route-update.timer \ + database/schema.sql \ + database/migrations/0001_initial.sql; do + [[ -f "$directory/$required_file" ]] || + wm_die "Release-Bundle ist unvollständig: $required_file fehlt." + done + + ( + cd "$directory" + sha256sum --check --strict SHA256SUMS >/dev/null + ) || wm_die "Release-Bundle hat die Integritätsprüfung nicht bestanden." + + [[ "$(wm_env_value "$directory/release.env" format_version)" == "1" ]] || + wm_die "Release-Manifest hat eine unbekannte Formatversion." + manifest_revision="$(wm_env_value "$directory/release.env" revision)" + manifest_source="$(wm_env_value "$directory/release.env" source)" + manifest_app="$(wm_env_value "$directory/release.env" app_image)" + manifest_route="$(wm_env_value "$directory/release.env" route_data_image)" + + [[ "$manifest_revision" == "$expected_revision" ]] || + wm_die "Release-Manifest und erwartete Revision stimmen nicht überein." + [[ "$manifest_source" == "https://$registry/$repository" ]] || + wm_die "Release-Manifest verweist auf eine unerwartete Quelle." + [[ "$manifest_app" =~ ^${app_repository//./\\.}@sha256:[0-9a-f]{64}$ ]] || + wm_die "Release-Manifest enthält kein gültiges App-Image." + [[ "$manifest_route" =~ ^${route_data_repository//./\\.}@sha256:[0-9a-f]{64}$ ]] || + wm_die "Release-Manifest enthält kein gültiges Routingdaten-Image." +} + +ensure_release_bundle() { + local revision="$1" + local image_reference="$release_repository:$revision" + local destination="$release_store/$revision" + + if [[ -d "$destination" ]]; then + validate_release_bundle "$destination" "$revision" + release_directory="$destination" + return + fi + + docker pull "$image_reference" >/dev/null + [[ "$(image_revision "$image_reference")" == "$revision" ]] || + wm_die "Release-Image und erwartete Revision stimmen nicht überein." + + temporary_release_dir="$(mktemp -d "$release_store/.candidate-$revision.XXXXXX")" + release_container_id="$(docker create "$image_reference")" + docker cp "$release_container_id:/release/." "$temporary_release_dir/" + docker container rm "$release_container_id" >/dev/null + release_container_id="" + validate_release_bundle "$temporary_release_dir" "$revision" + chmod 0755 "$temporary_release_dir/deploy/scripts/"*.sh + mv "$temporary_release_dir" "$destination" + temporary_release_dir="" + release_directory="$destination" +} + +install_release_units() { + local source_directory="$1" + + install -m 0755 \ + "$source_directory/deploy/scripts/auto-deploy-entrypoint.sh" \ + /usr/local/sbin/watermaps-auto-deploy + install -m 0644 \ + "$source_directory/deploy/systemd/watermaps-auto-deploy.service" \ + /etc/systemd/system/ + install -m 0644 \ + "$source_directory/deploy/systemd/watermaps-auto-deploy.timer" \ + /etc/systemd/system/ + install -m 0644 \ + "$source_directory/deploy/systemd/watermaps-route-update.service" \ + /etc/systemd/system/ + install -m 0644 \ + "$source_directory/deploy/systemd/watermaps-route-update.timer" \ + /etc/systemd/system/ + install -m 0644 \ + "$source_directory/deploy/systemd/watermaps-certbot-renew.service" \ + /etc/systemd/system/ + install -m 0644 \ + "$source_directory/deploy/systemd/watermaps-certbot-renew.timer" \ + /etc/systemd/system/ + systemctl daemon-reload + systemctl enable --now \ + watermaps-auto-deploy.timer \ + watermaps-route-update.timer \ + watermaps-certbot-renew.timer \ + >/dev/null +} + +activate_release() { + local source_directory="$1" + + temporary_current_link="$install_root/.current.$$.new" + rm -f -- "$temporary_current_link" + ln --symbolic "$source_directory" "$temporary_current_link" + mv --no-target-directory "$temporary_current_link" "$current_release_link" + temporary_current_link="" +} + +prune_old_releases() { + local candidate old_revision old_directory old_app old_route + local retained_revision retained_manifest image_is_retained + local -a revisions=() + local -A keep=() + + mapfile -t revisions < <( + find "$release_store" \ + -mindepth 1 \ + -maxdepth 1 \ + -type d \ + -printf '%T@ %f\n' | + sort --numeric-sort --reverse | + awk '$2 ~ /^[0-9a-f]{40}$/ { print $2 }' + ) + + keep["$revision"]=1 + if [[ -n "$active_revision" ]]; then + keep["$active_revision"]=1 + fi + for candidate in "${revisions[@]:0:release_retention}"; do + keep["$candidate"]=1 + done + + for old_revision in "${revisions[@]}"; do + [[ -z "${keep[$old_revision]:-}" ]] || continue + [[ "$old_revision" =~ ^[0-9a-f]{40}$ ]] || continue + old_directory="$release_store/$old_revision" + [[ -d "$old_directory" ]] || continue + + old_app="$(wm_env_value "$old_directory/release.env" app_image)" + old_route="$(wm_env_value "$old_directory/release.env" route_data_image)" + for candidate in "$old_app" "$old_route"; do + image_is_retained=false + for retained_revision in "${!keep[@]}"; do + retained_manifest="$release_store/$retained_revision/release.env" + if [[ -f "$retained_manifest" ]] && + grep --fixed-strings --line-regexp --quiet \ + "app_image=$candidate" "$retained_manifest" || + [[ -f "$retained_manifest" ]] && + grep --fixed-strings --line-regexp --quiet \ + "route_data_image=$candidate" "$retained_manifest"; then + image_is_retained=true + break + fi + done + if [[ "$image_is_retained" == "false" ]]; then + docker image rm "$candidate" >/dev/null 2>&1 || true + fi + done + + docker image rm \ + "$app_repository:$old_revision" \ + "$route_data_repository:$old_revision" \ + "$release_repository:$old_revision" \ + >/dev/null 2>&1 || + true + rm -rf -- "$old_directory" + wm_log "Altes lokales Release $old_revision wurde gezielt bereinigt." + done +} + +wm_log "Prüfe den atomaren Gitea-Release-Pointer." +docker pull "$release_pointer" >/dev/null +revision="$(image_revision "$release_pointer")" +release_pointer_id="$(image_id "$release_pointer")" + +main_revision="$(gitea_main_revision)" +if [[ "$revision" != "$main_revision" ]]; then + wm_log "Release $revision ist nicht der aktuelle main-Commit $main_revision; warte auf den laufenden Build." + exit 0 +fi + +release_image="$release_repository:$revision" +docker pull "$release_image" >/dev/null +[[ "$(image_id "$release_image")" == "$release_pointer_id" ]] || + wm_die "Der bewegliche Release-Pointer stimmt nicht mit dem unveränderlichen Commit-Image überein." + +ensure_release_bundle "$revision" + +active_revision="" +if [[ -f "$active_images_file" ]]; then + wm_validate_images_env "$active_images_file" + active_revision="$(wm_env_value "$active_images_file" WATERMAPS_DEPLOY_REVISION)" +fi +if [[ "$active_revision" == "$revision" ]]; then + current_release_target="$( + readlink --canonicalize "$current_release_link" 2>/dev/null || true + )" + expected_release_target="$(readlink --canonicalize "$release_directory")" + if [[ ! -L "$current_release_link" || + "$current_release_target" != "$expected_release_target" ]]; then + activate_release "$release_directory" + install_release_units "$release_directory" + wm_log "Der atomare Current-Link für Commit $revision wurde repariert." + fi + wm_log "Commit $revision ist bereits aktiv." + exit 0 +fi + +app_image="$(wm_env_value "$release_directory/release.env" app_image)" +route_data_image="$(wm_env_value "$release_directory/release.env" route_data_image)" +docker pull "$app_image" >/dev/null +docker pull "$route_data_image" >/dev/null +[[ "$(image_revision "$app_image")" == "$revision" ]] || + wm_die "App-Image gehört nicht zu Release $revision." +[[ "$(image_revision "$route_data_image")" == "$revision" ]] || + wm_die "Routingdaten-Image gehört nicht zu Release $revision." + +resolved_app_image="$(wm_resolve_image_digest "$app_image")" +resolved_route_data_image="$(wm_resolve_image_digest "$route_data_image")" +temporary_images_file="$( + mktemp "$WATERMAPS_RUNTIME_DIR/.auto-deploy-images.XXXXXX" +)" +{ + printf 'WATERMAPS_DEPLOY_REVISION=%s\n' "$revision" + printf 'WATERMAPS_APP_IMAGE=%s\n' "$resolved_app_image" + printf 'WATERMAPS_ROUTE_DATA_IMAGE=%s\n' "$resolved_route_data_image" +} >"$temporary_images_file" +chmod 0600 "$temporary_images_file" +wm_validate_images_env "$temporary_images_file" + +# A stale workflow must never activate after a newer push. Pull and compare the +# single release pointer immediately before changing deployment files. +docker pull "$release_pointer" >/dev/null +[[ "$(image_revision "$release_pointer")" == "$revision" ]] || + wm_die "Während der Vorbereitung wurde ein neueres Release veröffentlicht." +[[ "$(image_id "$release_pointer")" == "$release_pointer_id" ]] || + wm_die "Der Release-Pointer wurde während der Vorbereitung ausgetauscht." +[[ "$(gitea_main_revision)" == "$revision" ]] || + wm_die "main wurde während der Vorbereitung auf einen neueren Commit verschoben." + +previous_release_directory="" +previous_images_file="" +if [[ -n "$active_revision" ]]; then + previous_release_directory="$release_store/$active_revision" + previous_images_file="$WATERMAPS_RUNTIME_DIR/deployments/$active_revision.env" + if [[ ! -d "$previous_release_directory" || + ! -f "$previous_images_file" ]]; then + previous_release_directory="" + previous_images_file="" + fi +fi + +set +e +WATERMAPS_ENV_FILE="$WM_ENV_FILE" \ +WATERMAPS_ACTIVE_IMAGES_FILE="$active_images_file" \ +WATERMAPS_DISABLE_INTERNAL_ROLLBACK=true \ + "$release_directory/deploy/scripts/deploy.sh" \ + --images-file "$temporary_images_file" +deploy_status=$? +set -e + +if [[ "$deploy_status" -ne 0 ]]; then + if [[ "$deploy_status" -eq 75 ]]; then + wm_log "Ein Datenupdate oder manuelles Deployment läuft; das Release wird später erneut versucht." + exit 0 + fi + if [[ -n "$previous_release_directory" && + -n "$previous_images_file" ]]; then + wm_log "Das vorherige Release $active_revision wird aus seinem unveränderlichen Bundle wiederhergestellt." + set +e + WATERMAPS_ENV_FILE="$WM_ENV_FILE" \ + WATERMAPS_ACTIVE_IMAGES_FILE="$active_images_file" \ + WATERMAPS_DISABLE_INTERNAL_ROLLBACK=true \ + "$previous_release_directory/deploy/scripts/deploy.sh" \ + --images-file "$previous_images_file" + rollback_status=$? + set -e + if [[ "$rollback_status" -ne 0 ]]; then + wm_die "Release $revision und die Wiederherstellung von $active_revision sind fehlgeschlagen." + fi + activate_release "$previous_release_directory" + wm_log "Vorheriges Release $active_revision wurde erneut geprüft und ist aktiv." + fi + exit "$deploy_status" +fi + +activate_release "$release_directory" +install_release_units "$release_directory" +prune_old_releases +wm_log "Automatisches Deployment von Commit $revision abgeschlossen." diff --git a/deploy/scripts/bootstrap-server.sh b/deploy/scripts/bootstrap-server.sh index 6bcf7e8..83b4150 100755 --- a/deploy/scripts/bootstrap-server.sh +++ b/deploy/scripts/bootstrap-server.sh @@ -113,7 +113,15 @@ install -m 0644 "$WM_DEPLOY_DIR/systemd/watermaps-route-update.service" /etc/sys install -m 0644 "$WM_DEPLOY_DIR/systemd/watermaps-route-update.timer" /etc/systemd/system/ install -m 0644 "$WM_DEPLOY_DIR/systemd/watermaps-certbot-renew.service" /etc/systemd/system/ install -m 0644 "$WM_DEPLOY_DIR/systemd/watermaps-certbot-renew.timer" /etc/systemd/system/ +install -m 0644 "$WM_DEPLOY_DIR/systemd/watermaps-auto-deploy.service" /etc/systemd/system/ +install -m 0644 "$WM_DEPLOY_DIR/systemd/watermaps-auto-deploy.timer" /etc/systemd/system/ +install -m 0755 \ + "$WM_DEPLOY_DIR/scripts/auto-deploy-entrypoint.sh" \ + /usr/local/sbin/watermaps-auto-deploy systemctl daemon-reload -systemctl enable --now watermaps-route-update.timer watermaps-certbot-renew.timer +systemctl enable \ + watermaps-route-update.timer \ + watermaps-certbot-renew.timer +systemctl enable --now watermaps-auto-deploy.timer wm_log "Server-Bootstrap abgeschlossen." diff --git a/deploy/scripts/configure-gitea-actions.sh b/deploy/scripts/configure-gitea-actions.sh new file mode 100755 index 0000000..bb45b2c --- /dev/null +++ b/deploy/scripts/configure-gitea-actions.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash + +set +x +set -Eeuo pipefail + +WM_DEPLOY_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +WM_ROOT_DIR="$(cd "$WM_DEPLOY_DIR/.." && pwd)" +config_token_file="${WATERMAPS_GITEA_CONFIG_TOKEN_FILE:-$WM_DEPLOY_DIR/.gitea-actions-config-token}" +registry_token_file="${WATERMAPS_GITEA_REGISTRY_TOKEN_FILE:-$WM_DEPLOY_DIR/.gitea-registry-publish-token}" +gitea_url="${WATERMAPS_GITEA_URL:-https://gitea.incoso.eu}" +repository="${WATERMAPS_GITEA_REPOSITORY:-kevin_janssen/watermaps}" + +jq_command="$(command -v jq 2>/dev/null || true)" +if [[ -z "$jq_command" && -x "$WM_ROOT_DIR/.tools/bin/jq" ]]; then + jq_command="$WM_ROOT_DIR/.tools/bin/jq" +fi +[[ -n "$jq_command" ]] || { + printf 'jq fehlt; bitte jq installieren oder unter .tools/bin/jq bereitstellen.\n' >&2 + exit 1 +} + +cleanup_secrets() { + unset config_token registry_token token_payload +} +trap cleanup_secrets EXIT + +read_secret_file() { + local label="$1" + local path="$2" + local mode + local value + + [[ -f "$path" && ! -L "$path" ]] || { + printf '%s fehlt oder ist keine reguläre Datei: %s\n' "$label" "$path" >&2 + return 1 + } + mode="$(stat --format=%a -- "$path")" + [[ "$mode" == "600" ]] || { + printf '%s muss Dateimodus 600 haben (chmod 600 %s; aktuell: %s).\n' \ + "$label" "$path" "$mode" >&2 + return 1 + } + + value="$(<"$path")" + [[ -n "$value" && "$value" != *[[:space:]]* ]] || { + printf '%s ist leer oder enthält Leerzeichen beziehungsweise Zeilenumbrüche.\n' \ + "$label" >&2 + return 1 + } + printf '%s' "$value" +} + +config_token="$( + read_secret_file \ + "Lokaler Gitea-Konfigurations-PAT" \ + "$config_token_file" +)" +registry_token="$( + read_secret_file \ + "Gitea-Registry-Publish-PAT" \ + "$registry_token_file" +)" +[[ ! "$config_token_file" -ef "$registry_token_file" ]] || { + printf 'Konfigurations- und Registry-PAT müssen in zwei getrennten Dateien liegen.\n' >&2 + exit 1 +} +[[ "$config_token" != "$registry_token" ]] || { + printf 'Konfigurations- und Registry-PAT müssen zwei unterschiedliche Tokens sein.\n' >&2 + exit 1 +} +[[ "$repository" =~ ^([A-Za-z0-9_.-]+)/([A-Za-z0-9_.-]+)$ ]] || { + printf 'Ungültiges Gitea-Repository: %s\n' "$repository" >&2 + exit 1 +} +registry_username="${BASH_REMATCH[1]}" +[[ "$gitea_url" =~ ^https://[^/?#]+(:[0-9]+)?$ ]] || { + printf 'Die Gitea-URL muss eine HTTPS-Origin ohne Pfad sein: %s\n' "$gitea_url" >&2 + exit 1 +} + +api_request() { + local method="$1" + local endpoint="$2" + local payload="${3:-}" + local auth_header="Authorization: token $config_token" + local args=( + --proto '=https' + --tlsv1.2 + --fail + --silent + --show-error + --connect-timeout 10 + --max-time 30 + --request "$method" + --header @/dev/fd/3 + ) + if [[ -n "$payload" ]]; then + args+=( + --header "Content-Type: application/json" + --data-binary @/dev/fd/4 + ) + curl "${args[@]}" "$gitea_url/api/v1/$endpoint" \ + 3<<<"$auth_header" \ + 4<<<"$payload" + else + curl "${args[@]}" "$gitea_url/api/v1/$endpoint" \ + 3<<<"$auth_header" + fi +} + +username_payload="$( + printf '%s' "$registry_username" | + "$jq_command" --raw-input --slurp '{data: .}' +)" +token_payload="$( + printf '%s' "$registry_token" | + "$jq_command" --raw-input --slurp '{data: .}' +)" +api_request \ + PUT \ + "repos/$repository/actions/secrets/REGISTRY_USERNAME" \ + "$username_payload" \ + >/dev/null +api_request \ + PUT \ + "repos/$repository/actions/secrets/REGISTRY_TOKEN" \ + "$token_payload" \ + >/dev/null + +unset config_token registry_token token_payload +printf 'Gitea Actions Registry-Secrets wurden für %s aktualisiert.\n' "$repository" diff --git a/deploy/scripts/deploy.sh b/deploy/scripts/deploy.sh index bb89815..a75a304 100755 --- a/deploy/scripts/deploy.sh +++ b/deploy/scripts/deploy.sh @@ -44,10 +44,14 @@ install -d -m 0755 \ "$WATERMAPS_RUNTIME_DIR/locks" \ "$WATERMAPS_RUNTIME_DIR/deployments" -wm_acquire_route_lock || - wm_die "Deployment abgebrochen, weil gerade Routingdaten aktualisiert werden." +if ! wm_acquire_route_lock; then + wm_log "Deployment wird später erneut versucht, weil gerade Routingdaten aktualisiert werden." + exit 75 +fi -active_images_file="$WM_DEPLOY_DIR/.env.images" +active_images_file="${WATERMAPS_ACTIVE_IMAGES_FILE:-${WATERMAPS_IMAGES_ENV_FILE:-$WM_DEPLOY_DIR/.env.images}}" +[[ "$active_images_file" == /* ]] || + wm_die "WATERMAPS_ACTIVE_IMAGES_FILE muss ein absoluter Pfad sein." rollback_images_file="" resolved_images_file="" deployment_complete=false @@ -65,7 +69,9 @@ wm_finish_deployment() { trap - EXIT HUP INT TERM set +e - if [[ "$status" -ne 0 && "$deployment_complete" != "true" ]]; then + if [[ "$status" -ne 0 && + "$deployment_complete" != "true" && + "${WATERMAPS_DISABLE_INTERNAL_ROLLBACK:-false}" != "true" ]]; then if [[ -n "$rollback_images_file" && -f "$rollback_images_file" ]]; then wm_log "Deployment fehlgeschlagen; vorheriges Container-Release wird wiederhergestellt." wm_use_images_env "$rollback_images_file" @@ -122,6 +128,60 @@ wm_log "Persistentes PostGIS wird vor App und Datenimport gestartet." wm_compose up --detach --no-build postgres wm_wait_for_health postgres 240 +wm_log "Versionierte Datenbankmigrationen für Commit $revision werden angewendet." +wm_compose exec --no-TTY postgres \ + psql \ + --no-psqlrc \ + --username seacompass \ + --dbname seacompass \ + --set ON_ERROR_STOP=1 \ + --command ' + CREATE TABLE IF NOT EXISTS watermaps_schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT now() + ); + ' + +shopt -s nullglob +migration_files=("$WM_ROOT_DIR"/database/migrations/*.sql) +shopt -u nullglob +[[ "${#migration_files[@]}" -gt 0 ]] || + wm_die "Keine Datenbankmigrationen im Release gefunden." + +for migration_file in "${migration_files[@]}"; do + migration_name="$(basename "$migration_file")" + [[ "$migration_name" =~ ^[0-9]{4}_[a-z0-9_]+\.sql$ ]] || + wm_die "Ungültiger Migrationsdateiname: $migration_name" + migration_version="${migration_name%.sql}" + migration_applied="$( + wm_postgres_query " + SELECT EXISTS ( + SELECT 1 + FROM watermaps_schema_migrations + WHERE version = '$migration_version' + ); + " + )" + migration_applied="${migration_applied//[[:space:]]/}" + if [[ "$migration_applied" == "t" ]]; then + continue + fi + + wm_log "Wende Datenbankmigration $migration_version atomar an." + wm_compose exec --no-TTY postgres \ + psql \ + --no-psqlrc \ + --username seacompass \ + --dbname seacompass \ + --set ON_ERROR_STOP=1 \ + --single-transaction \ + --file "/docker-entrypoint-initdb.d/migrations/$migration_name" \ + --command " + INSERT INTO watermaps_schema_migrations (version) + VALUES ('$migration_version'); + " +done + data_update_required=false update_args=() if [[ "${WATERMAPS_REBUILD_ROUTE_DATA:-false}" == "true" ]]; then diff --git a/deploy/scripts/remote-common.sh b/deploy/scripts/remote-common.sh index 3ea7ab2..f15d620 100755 --- a/deploy/scripts/remote-common.sh +++ b/deploy/scripts/remote-common.sh @@ -11,25 +11,36 @@ wm_local_die() { exit 1 } +wm_local_iac_command() { + if command -v tofu >/dev/null 2>&1; then + command -v tofu + elif command -v terraform >/dev/null 2>&1; then + command -v terraform + else + return 1 + fi +} + wm_local_resolve_ssh() { local requested_server="${1:-}" local requested_identity="${2:-}" + local iac_command WM_SERVER_IPV4="${requested_server:-${WATERMAPS_SERVER_IPV4:-}}" if [[ -z "$WM_SERVER_IPV4" ]]; then - command -v tofu >/dev/null 2>&1 || - wm_local_die "OpenTofu fehlt und WATERMAPS_SERVER_IPV4 wurde nicht gesetzt." + iac_command="$(wm_local_iac_command)" || + wm_local_die "OpenTofu/Terraform fehlt und WATERMAPS_SERVER_IPV4 wurde nicht gesetzt." [[ -d "$WM_LOCAL_INFRA_DIR" ]] || wm_local_die "OpenTofu-Verzeichnis fehlt: $WM_LOCAL_INFRA_DIR" - WM_SERVER_IPV4="$(tofu -chdir="$WM_LOCAL_INFRA_DIR" output -raw server_ipv4)" + WM_SERVER_IPV4="$("$iac_command" -chdir="$WM_LOCAL_INFRA_DIR" output -raw server_ipv4)" fi [[ "$WM_SERVER_IPV4" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]] || wm_local_die "Ungültige Server-IPv4: $WM_SERVER_IPV4" WM_SSH_IDENTITY="${requested_identity:-${WATERMAPS_SSH_KEY:-}}" - if [[ -z "$WM_SSH_IDENTITY" ]] && command -v tofu >/dev/null 2>&1 && [[ -d "$WM_LOCAL_INFRA_DIR" ]]; then - candidate_identity="$(tofu -chdir="$WM_LOCAL_INFRA_DIR" output -raw ssh_private_key_path 2>/dev/null || true)" - if [[ -n "$candidate_identity" && -f "$candidate_identity" ]]; then + if [[ -z "$WM_SSH_IDENTITY" ]]; then + candidate_identity="$HOME/.ssh/watermaps_deploy_ed25519" + if [[ -f "$candidate_identity" ]]; then WM_SSH_IDENTITY="$candidate_identity" fi fi diff --git a/deploy/scripts/remote-go-live.sh b/deploy/scripts/remote-go-live.sh index 091df73..23dadd0 100755 --- a/deploy/scripts/remote-go-live.sh +++ b/deploy/scripts/remote-go-live.sh @@ -45,4 +45,4 @@ if [[ -n "$WM_REMOTE_SUDO" ]]; then fi ssh "${WM_SSH_OPTIONS[@]}" "$WM_SSH_TARGET" \ - "${remote_prefix}/opt/watermaps/deploy/scripts/go-live.sh '$WM_SERVER_IPV4'" + "${remote_prefix}WATERMAPS_ENV_FILE=/opt/watermaps/deploy/.env.production WATERMAPS_IMAGES_ENV_FILE=/opt/watermaps/deploy/.env.images /opt/watermaps/current/deploy/scripts/go-live.sh '$WM_SERVER_IPV4'" diff --git a/deploy/scripts/remote-rollback.sh b/deploy/scripts/remote-rollback.sh index c552cc2..c964aae 100755 --- a/deploy/scripts/remote-rollback.sh +++ b/deploy/scripts/remote-rollback.sh @@ -58,4 +58,4 @@ if [[ -n "$WM_REMOTE_SUDO" ]]; then fi ssh "${WM_SSH_OPTIONS[@]}" "$WM_SSH_TARGET" \ - "${remote_prefix}/opt/watermaps/deploy/scripts/rollback.sh '$revision'" + "${remote_prefix}WATERMAPS_ENV_FILE=/opt/watermaps/deploy/.env.production WATERMAPS_IMAGES_ENV_FILE=/opt/watermaps/deploy/.env.images WATERMAPS_ACTIVE_IMAGES_FILE=/opt/watermaps/deploy/.env.images /opt/watermaps/current/deploy/scripts/rollback.sh '$revision'" diff --git a/deploy/scripts/rollback.sh b/deploy/scripts/rollback.sh index 4902e45..bb6d001 100755 --- a/deploy/scripts/rollback.sh +++ b/deploy/scripts/rollback.sh @@ -25,4 +25,38 @@ fi [[ -f "$images_file" ]] || wm_die "Gespeichertes Release nicht gefunden: $images_file" -exec "$WM_DEPLOY_DIR/scripts/deploy.sh" --images-file "$images_file" +target_revision="$(wm_env_value "$images_file" WATERMAPS_DEPLOY_REVISION)" +release_directory="$WATERMAPS_DATA_DIR/releases/$target_revision" +[[ -d "$release_directory" && + -x "$release_directory/deploy/scripts/deploy.sh" ]] || + wm_die "Unveränderliches Release-Bundle fehlt: $release_directory" + +install_root="${WATERMAPS_INSTALL_DIR:-/opt/watermaps}" +wm_require_safe_absolute_dir "$install_root" +active_images_file="${WATERMAPS_ACTIVE_IMAGES_FILE:-$install_root/deploy/.env.images}" + +WATERMAPS_ENV_FILE="$WM_ENV_FILE" \ +WATERMAPS_ACTIVE_IMAGES_FILE="$active_images_file" \ + "$release_directory/deploy/scripts/deploy.sh" \ + --images-file "$images_file" + +temporary_link="$install_root/.current.rollback.$$.new" +trap 'rm -f -- "$temporary_link"' EXIT HUP INT TERM +ln --symbolic "$release_directory" "$temporary_link" +mv --no-target-directory "$temporary_link" "$install_root/current" +temporary_link="" + +for unit in \ + watermaps-auto-deploy.service \ + watermaps-auto-deploy.timer \ + watermaps-route-update.service \ + watermaps-route-update.timer \ + watermaps-certbot-renew.service \ + watermaps-certbot-renew.timer; do + install -m 0644 \ + "$release_directory/deploy/systemd/$unit" \ + "/etc/systemd/system/$unit" +done +systemctl daemon-reload + +wm_log "Rollback auf Commit $target_revision ist geprüft und atomar aktiviert." diff --git a/deploy/scripts/tests/auto-deploy.test.sh b/deploy/scripts/tests/auto-deploy.test.sh new file mode 100755 index 0000000..45daa98 --- /dev/null +++ b/deploy/scripts/tests/auto-deploy.test.sh @@ -0,0 +1,605 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +AUTO_DEPLOY="$ROOT_DIR/deploy/scripts/auto-deploy.sh" +TEST_ROOT="$(mktemp -d)" +trap 'rm -rf "$TEST_ROOT"' EXIT + +REVISION="1111111111111111111111111111111111111111" +PREVIOUS_REVISION="2222222222222222222222222222222222222222" +APP_DIGEST="$(printf 'a%.0s' {1..64})" +ROUTE_DIGEST="$(printf 'b%.0s' {1..64})" +PREVIOUS_APP_DIGEST="$(printf 'c%.0s' {1..64})" +PREVIOUS_ROUTE_DIGEST="$(printf 'd%.0s' {1..64})" +APP_IMAGE="registry.example/team/watermaps@sha256:$APP_DIGEST" +ROUTE_IMAGE="registry.example/team/watermaps-route-data@sha256:$ROUTE_DIGEST" +PREVIOUS_APP_IMAGE="registry.example/team/watermaps@sha256:$PREVIOUS_APP_DIGEST" +PREVIOUS_ROUTE_IMAGE="registry.example/team/watermaps-route-data@sha256:$PREVIOUS_ROUTE_DIGEST" +RELEASE_POINTER="registry.example/team/watermaps-release:main" +RELEASE_IMAGE="registry.example/team/watermaps-release:$REVISION" +FAKE_BIN="$TEST_ROOT/bin" +mkdir -p "$FAKE_BIN" + +fail() { + printf 'Fehler: %s\n' "$*" >&2 + exit 1 +} + +assert_contains() { + local expected="$1" + local actual="$2" + grep -Fq -- "$expected" <<<"$actual" || + fail "Erwarteter Text fehlt: $expected" +} + +assert_file_contains() { + local expected="$1" + local file="$2" + grep -Fq -- "$expected" "$file" || + fail "$file enthält nicht: $expected" +} + +assert_file_not_contains() { + local unexpected="$1" + local file="$2" + if grep -Fq -- "$unexpected" "$file"; then + fail "$file enthält unerwartet: $unexpected" + fi +} + +assert_no_temporary_artifacts() { + local artifacts=() + + shopt -s nullglob + artifacts+=( + "$CASE_DATA/releases"/.candidate-* + "$CASE_RUNTIME"/.auto-deploy-images.* + "$CASE_INSTALL"/.current.*.new + ) + shopt -u nullglob + + if (( ${#artifacts[@]} > 0 )); then + printf 'Temporäre Artefakte wurden nicht aufgeräumt:\n' >&2 + printf ' %s\n' "${artifacts[@]}" >&2 + exit 1 + fi +} + +write_images_file() { + local destination="$1" + local revision="$2" + local app_image="$3" + local route_image="$4" + + install -d -m 0755 "$(dirname "$destination")" + { + printf 'WATERMAPS_DEPLOY_REVISION=%s\n' "$revision" + printf 'WATERMAPS_APP_IMAGE=%s\n' "$app_image" + printf 'WATERMAPS_ROUTE_DATA_IMAGE=%s\n' "$route_image" + } >"$destination" + chmod 0600 "$destination" +} + +cat >"$FAKE_BIN/fake-populate-release" <<'SH' +#!/usr/bin/env bash + +set -Eeuo pipefail + +destination="$1" +revision="$2" + +mkdir -p \ + "$destination/deploy/nginx" \ + "$destination/deploy/scripts" \ + "$destination/deploy/systemd" \ + "$destination/database/migrations" + +for file in \ + deploy/compose.production.yml \ + deploy/nginx/bootstrap.conf \ + deploy/nginx/https.conf.template \ + deploy/scripts/common.sh \ + deploy/scripts/auto-deploy.sh \ + deploy/scripts/auto-deploy-entrypoint.sh \ + deploy/scripts/bootstrap-server.sh \ + deploy/scripts/renew-certificate.sh \ + deploy/scripts/rollback.sh \ + deploy/scripts/update-route-data.sh \ + deploy/systemd/watermaps-auto-deploy.service \ + deploy/systemd/watermaps-auto-deploy.timer \ + deploy/systemd/watermaps-certbot-renew.service \ + deploy/systemd/watermaps-certbot-renew.timer \ + deploy/systemd/watermaps-route-update.service \ + deploy/systemd/watermaps-route-update.timer \ + database/schema.sql \ + database/migrations/0001_initial.sql; do + printf 'Testinhalt für %s\n' "$file" >"$destination/$file" +done + +cat >"$destination/deploy/scripts/deploy.sh" <<'DEPLOY' +#!/usr/bin/env bash + +set -Eeuo pipefail + +[[ "${1:-}" == "--images-file" && -n "${2:-}" ]] || + exit 97 +images_file="$2" +revision="$( + awk -F= '$1 == "WATERMAPS_DEPLOY_REVISION" { print substr($0, length($1) + 2) }' \ + "$images_file" +)" +app_image="$( + awk -F= '$1 == "WATERMAPS_APP_IMAGE" { print substr($0, length($1) + 2) }' \ + "$images_file" +)" +route_image="$( + awk -F= '$1 == "WATERMAPS_ROUTE_DATA_IMAGE" { print substr($0, length($1) + 2) }' \ + "$images_file" +)" + +printf '%s|%s|%s|%s|%s|%s\n' \ + "$0" \ + "$revision" \ + "${WATERMAPS_DISABLE_INTERNAL_ROLLBACK:-}" \ + "$images_file" \ + "$app_image" \ + "$route_image" \ + >>"$FAKE_DEPLOY_LOG" + +if [[ "$revision" == "$FAKE_REVISION" ]]; then + [[ "$app_image" == "$FAKE_APP_IMAGE" ]] || exit 98 + [[ "$route_image" == "$FAKE_ROUTE_IMAGE" ]] || exit 99 + if [[ "${FAKE_DEPLOY_STATUS:-0}" -ne 0 ]]; then + exit "$FAKE_DEPLOY_STATUS" + fi +fi + +cp "$images_file" "$WATERMAPS_ACTIVE_IMAGES_FILE" +DEPLOY +chmod 0755 "$destination/deploy/scripts/"*.sh + +{ + printf 'format_version=1\n' + printf 'revision=%s\n' "$revision" + printf 'source=https://registry.example/team/watermaps\n' + printf 'app_image=%s\n' "$FAKE_APP_IMAGE" + printf 'route_data_image=%s\n' "$FAKE_ROUTE_IMAGE" +} >"$destination/release.env" + +( + cd "$destination" + { + sha256sum release.env + find deploy database -type f -print | + LC_ALL=C sort | + xargs sha256sum + } >SHA256SUMS +) + +if [[ "${FAKE_CORRUPT_BUNDLE:-0}" == "1" ]]; then + printf 'nachträgliche Manipulation\n' >>"$destination/database/schema.sql" +fi +SH + +cat >"$FAKE_BIN/id" <<'SH' +#!/usr/bin/env bash +if [[ "${1:-}" == "-u" ]]; then + printf '0\n' +else + exec /usr/bin/id "$@" +fi +SH + +cat >"$FAKE_BIN/mountpoint" <<'SH' +#!/usr/bin/env bash +exit 0 +SH + +cat >"$FAKE_BIN/curl" <<'SH' +#!/usr/bin/env bash +printf 'curl|%s\n' "$*" >>"$FAKE_CURL_LOG" +printf '{"commit":{"id":"%s"}}\n' "$FAKE_MAIN_REVISION" +SH + +cat >"$FAKE_BIN/jq" <<'SH' +#!/usr/bin/env bash +set -Eeuo pipefail +input="$(tr -d '\r\n')" +revision="${input#*\"id\":\"}" +revision="${revision%%\"*}" +[[ "$revision" =~ ^[0-9a-f]{40}$ ]] || exit 1 +printf '%s\n' "$revision" +SH + +cat >"$FAKE_BIN/docker" <<'SH' +#!/usr/bin/env bash + +set -Eeuo pipefail + +printf 'docker|%s\n' "$*" >>"$FAKE_DOCKER_LOG" +command_name="${1:-}" +shift || true + +case "$command_name" in + pull) + [[ -n "${1:-}" ]] + ;; + image) + [[ "${1:-}" == "inspect" ]] || exit 91 + shift + reference="${!#}" + if [[ "$*" == *'org.opencontainers.image.revision'* ]]; then + if [[ "$reference" == "$FAKE_RELEASE_POINTER" ]]; then + printf '%s\n' "$FAKE_POINTER_REVISION" + else + printf '%s\n' "$FAKE_REVISION" + fi + elif [[ "$*" == *'{{.Id}}'* ]]; then + if [[ "$reference" == "$FAKE_RELEASE_POINTER" ]]; then + printf '%s\n' "$FAKE_POINTER_ID" + else + printf '%s\n' "$FAKE_IMMUTABLE_ID" + fi + elif [[ "$*" == *'.RepoDigests'* ]]; then + [[ "$reference" == *@sha256:* ]] || exit 92 + printf '%s\n' "$reference" + else + exit 93 + fi + ;; + create) + printf 'fake-release-container\n' + ;; + cp) + [[ "${1:-}" == "fake-release-container:/release/." ]] || exit 94 + destination="${2%/}" + fake-populate-release "$destination" "$FAKE_REVISION" + ;; + container) + [[ "${1:-}" == "rm" ]] || exit 95 + ;; + *) + printf 'Unerwarteter Docker-Aufruf: %s %s\n' "$command_name" "$*" >&2 + exit 96 + ;; +esac +SH + +cat >"$FAKE_BIN/install" <<'SH' +#!/usr/bin/env bash + +set -Eeuo pipefail + +if [[ "${1:-}" == "-d" ]]; then + exec /usr/bin/install "$@" +fi + +destination="${!#}" +if [[ "$destination" == "/usr/local/sbin/watermaps-auto-deploy" ]]; then + source_file="${@: -2:1}" + [[ -x "$source_file" ]] + printf '%s\n' "watermaps-auto-deploy-entrypoint" >>"$FAKE_INSTALL_LOG" + exit 0 +fi +if [[ "$destination" == "/etc/systemd/system/" ]]; then + source_file="${@: -2:1}" + [[ -f "$source_file" ]] + printf '%s\n' "$(basename "$source_file")" >>"$FAKE_INSTALL_LOG" + exit 0 +fi + +exec /usr/bin/install "$@" +SH + +cat >"$FAKE_BIN/systemctl" <<'SH' +#!/usr/bin/env bash +printf '%s\n' "$*" >>"$FAKE_SYSTEMCTL_LOG" +SH + +chmod 0755 "$FAKE_BIN/"* +export PATH="$FAKE_BIN:/usr/bin:/bin" + +new_case() { + local name="$1" + + CASE_DIR="$TEST_ROOT/$name" + CASE_DATA="$CASE_DIR/data" + CASE_RUNTIME="$CASE_DIR/runtime" + CASE_INSTALL="$CASE_DIR/install" + CASE_ENV="$CASE_DIR/production.env" + mkdir -p \ + "$CASE_DATA" \ + "$CASE_RUNTIME" \ + "$CASE_INSTALL/deploy" + + FAKE_DOCKER_LOG="$CASE_DIR/docker.log" + FAKE_DEPLOY_LOG="$CASE_DIR/deploy.log" + FAKE_SYSTEMCTL_LOG="$CASE_DIR/systemctl.log" + FAKE_INSTALL_LOG="$CASE_DIR/install.log" + FAKE_CURL_LOG="$CASE_DIR/curl.log" + : >"$FAKE_DOCKER_LOG" + : >"$FAKE_DEPLOY_LOG" + : >"$FAKE_SYSTEMCTL_LOG" + : >"$FAKE_INSTALL_LOG" + : >"$FAKE_CURL_LOG" + + cat >"$CASE_ENV" <&1)" + RUN_STATUS=$? + set -e +} + +assert_systemd_installed() { + local expected_units=( + watermaps-auto-deploy.service + watermaps-auto-deploy.timer + watermaps-route-update.service + watermaps-route-update.timer + watermaps-certbot-renew.service + watermaps-certbot-renew.timer + ) + local unit + + [[ "$(wc -l <"$FAKE_INSTALL_LOG")" -eq "$((${#expected_units[@]} + 1))" ]] || + fail "Es wurden nicht exakt sechs Systemd-Units und der stabile Einstiegspunkt installiert." + grep -Fxq "watermaps-auto-deploy-entrypoint" "$FAKE_INSTALL_LOG" || + fail "Stabiler Auto-Deploy-Einstiegspunkt wurde nicht installiert." + for unit in "${expected_units[@]}"; do + grep -Fxq "$unit" "$FAKE_INSTALL_LOG" || + fail "Systemd-Unit wurde nicht installiert: $unit" + done + grep -Fxq 'daemon-reload' "$FAKE_SYSTEMCTL_LOG" || + fail "systemctl daemon-reload fehlt." + assert_file_contains \ + 'enable --now watermaps-auto-deploy.timer watermaps-route-update.timer watermaps-certbot-renew.timer' \ + "$FAKE_SYSTEMCTL_LOG" +} + +test_already_active_repairs_current_link() { + new_case already-active + mkdir -p "$CASE_DATA/releases" + populate_release "$REVISION" + write_images_file \ + "$CASE_INSTALL/deploy/.env.images" \ + "$REVISION" \ + "$APP_IMAGE" \ + "$ROUTE_IMAGE" + mkdir -p "$CASE_DIR/wrong-release" + ln -s "$CASE_DIR/wrong-release" "$CASE_INSTALL/current" + + run_auto_deploy + + [[ "$RUN_STATUS" -eq 0 ]] || fail "Bereits aktives Release schlug fehl: $RUN_OUTPUT" + assert_contains "Der atomare Current-Link für Commit $REVISION wurde repariert." "$RUN_OUTPUT" + assert_contains "Commit $REVISION ist bereits aktiv." "$RUN_OUTPUT" + [[ "$(readlink --canonicalize "$CASE_INSTALL/current")" == \ + "$(readlink --canonicalize "$CASE_DATA/releases/$REVISION")" ]] || + fail "Current-Link wurde für das aktive Release nicht repariert." + [[ ! -s "$FAKE_DEPLOY_LOG" ]] || + fail "Ein bereits aktives Release darf deploy.sh nicht erneut ausführen." + assert_file_not_contains 'docker|create ' "$FAKE_DOCKER_LOG" + assert_file_not_contains 'docker|cp ' "$FAKE_DOCKER_LOG" + assert_systemd_installed + assert_no_temporary_artifacts +} + +test_successful_release_uses_manifest_digests() { + new_case successful-release + + run_auto_deploy + + [[ "$RUN_STATUS" -eq 0 ]] || fail "Neues Release schlug fehl: $RUN_OUTPUT" + assert_contains "Automatisches Deployment von Commit $REVISION abgeschlossen." "$RUN_OUTPUT" + [[ -f "$CASE_DATA/releases/$REVISION/SHA256SUMS" ]] || + fail "Validiertes Release-Bundle wurde nicht atomar installiert." + ( + cd "$CASE_DATA/releases/$REVISION" + sha256sum --check --strict SHA256SUMS >/dev/null + ) || fail "Installiertes Release-Bundle besteht seine SHA256-Prüfung nicht." + [[ "$(readlink --canonicalize "$CASE_INSTALL/current")" == \ + "$(readlink --canonicalize "$CASE_DATA/releases/$REVISION")" ]] || + fail "Current-Link zeigt nicht auf das neue Release." + + [[ "$(wc -l <"$FAKE_DEPLOY_LOG")" -eq 1 ]] || + fail "Das neue Release muss deploy.sh exakt einmal ausführen." + IFS='|' read -r invoked_script deployed_revision rollback_disabled \ + images_file deployed_app deployed_route <"$FAKE_DEPLOY_LOG" + [[ "$invoked_script" == \ + "$CASE_DATA/releases/$REVISION/deploy/scripts/deploy.sh" ]] || + fail "deploy.sh wurde nicht aus dem unveränderlichen Release-Bundle ausgeführt." + [[ "$deployed_revision" == "$REVISION" ]] || + fail "deploy.sh erhielt die falsche Revision." + [[ "$rollback_disabled" == "true" ]] || + fail "Der interne Rollback muss beim orchestrierten Deployment deaktiviert sein." + [[ "$images_file" == "$CASE_RUNTIME"/.auto-deploy-images.* ]] || + fail "deploy.sh erhielt keine temporäre, validierte Image-Datei." + [[ "$deployed_app" == "$APP_IMAGE" && "$deployed_route" == "$ROUTE_IMAGE" ]] || + fail "deploy.sh erhielt nicht die im Release-Manifest gebundenen Digests." + [[ ! -e "$images_file" ]] || + fail "Temporäre Image-Datei blieb nach dem Deployment bestehen." + + assert_file_contains "docker|pull $APP_IMAGE" "$FAKE_DOCKER_LOG" + assert_file_contains "docker|pull $ROUTE_IMAGE" "$FAKE_DOCKER_LOG" + assert_file_not_contains "docker|pull registry.example/team/watermaps:$REVISION" \ + "$FAKE_DOCKER_LOG" + assert_file_contains "WATERMAPS_DEPLOY_REVISION=$REVISION" \ + "$CASE_INSTALL/deploy/.env.images" + assert_file_contains "WATERMAPS_APP_IMAGE=$APP_IMAGE" \ + "$CASE_INSTALL/deploy/.env.images" + assert_file_contains "WATERMAPS_ROUTE_DATA_IMAGE=$ROUTE_IMAGE" \ + "$CASE_INSTALL/deploy/.env.images" + assert_systemd_installed + assert_no_temporary_artifacts +} + +test_pointer_main_mismatch_does_not_deploy() { + new_case pointer-main-mismatch + export FAKE_MAIN_REVISION="$PREVIOUS_REVISION" + + run_auto_deploy + + [[ "$RUN_STATUS" -eq 0 ]] || + fail "Pointer/main-Abweichung soll sauber warten: $RUN_OUTPUT" + assert_contains \ + "Release $REVISION ist nicht der aktuelle main-Commit $PREVIOUS_REVISION" \ + "$RUN_OUTPUT" + [[ ! -e "$CASE_DATA/releases/$REVISION" ]] || + fail "Bei Pointer/main-Abweichung darf kein Bundle extrahiert werden." + [[ ! -L "$CASE_INSTALL/current" ]] || + fail "Bei Pointer/main-Abweichung darf Current nicht verändert werden." + [[ ! -s "$FAKE_DEPLOY_LOG" ]] || + fail "Bei Pointer/main-Abweichung darf deploy.sh nicht laufen." + [[ ! -s "$FAKE_SYSTEMCTL_LOG" && ! -s "$FAKE_INSTALL_LOG" ]] || + fail "Bei Pointer/main-Abweichung darf Systemd nicht verändert werden." + assert_file_contains "docker|pull $RELEASE_POINTER" "$FAKE_DOCKER_LOG" + assert_file_not_contains "docker|pull $RELEASE_IMAGE" "$FAKE_DOCKER_LOG" + assert_file_not_contains 'docker|create ' "$FAKE_DOCKER_LOG" + assert_no_temporary_artifacts +} + +test_failed_release_restores_previous_immutable_release() { + new_case failed-release + mkdir -p "$CASE_DATA/releases" "$CASE_RUNTIME/deployments" + populate_release "$PREVIOUS_REVISION" + write_images_file \ + "$CASE_INSTALL/deploy/.env.images" \ + "$PREVIOUS_REVISION" \ + "$PREVIOUS_APP_IMAGE" \ + "$PREVIOUS_ROUTE_IMAGE" + cp \ + "$CASE_INSTALL/deploy/.env.images" \ + "$CASE_RUNTIME/deployments/$PREVIOUS_REVISION.env" + ln -s "$CASE_DATA/releases/$PREVIOUS_REVISION" "$CASE_INSTALL/current" + export FAKE_DEPLOY_STATUS=42 + + run_auto_deploy + + [[ "$RUN_STATUS" -eq 42 ]] || + fail "Fehlgeschlagenes Release muss seinen Status weitergeben: $RUN_OUTPUT" + assert_contains \ + "Das vorherige Release $PREVIOUS_REVISION wird aus seinem unveränderlichen Bundle wiederhergestellt." \ + "$RUN_OUTPUT" + assert_contains \ + "Vorheriges Release $PREVIOUS_REVISION wurde erneut geprüft und ist aktiv." \ + "$RUN_OUTPUT" + [[ "$(wc -l <"$FAKE_DEPLOY_LOG")" -eq 2 ]] || + fail "Nach einem Fehler müssen Kandidat und vorheriges Release genau einmal laufen." + + candidate_log="$(sed -n '1p' "$FAKE_DEPLOY_LOG")" + previous_log="$(sed -n '2p' "$FAKE_DEPLOY_LOG")" + assert_contains \ + "$CASE_DATA/releases/$REVISION/deploy/scripts/deploy.sh|$REVISION|true|" \ + "$candidate_log" + assert_contains "|$APP_IMAGE|$ROUTE_IMAGE" "$candidate_log" + assert_contains \ + "$CASE_DATA/releases/$PREVIOUS_REVISION/deploy/scripts/deploy.sh|$PREVIOUS_REVISION|true|$CASE_RUNTIME/deployments/$PREVIOUS_REVISION.env|" \ + "$previous_log" + assert_contains "|$PREVIOUS_APP_IMAGE|$PREVIOUS_ROUTE_IMAGE" "$previous_log" + + [[ "$(readlink --canonicalize "$CASE_INSTALL/current")" == \ + "$(readlink --canonicalize "$CASE_DATA/releases/$PREVIOUS_REVISION")" ]] || + fail "Current-Link wurde nach dem Fehler nicht auf das vorherige Release zurückgesetzt." + assert_file_contains "WATERMAPS_DEPLOY_REVISION=$PREVIOUS_REVISION" \ + "$CASE_INSTALL/deploy/.env.images" + [[ ! -s "$FAKE_SYSTEMCTL_LOG" && ! -s "$FAKE_INSTALL_LOG" ]] || + fail "Ein fehlgeschlagener Kandidat darf keine neuen Systemd-Units aktivieren." + assert_no_temporary_artifacts +} + +test_invalid_bundle_is_rejected_and_cleaned() { + new_case corrupt-release + export FAKE_CORRUPT_BUNDLE=1 + + run_auto_deploy + + [[ "$RUN_STATUS" -ne 0 ]] || + fail "Ein manipuliertes Release-Bundle wurde akzeptiert." + assert_contains "Release-Bundle hat die Integritätsprüfung nicht bestanden." "$RUN_OUTPUT" + [[ ! -e "$CASE_DATA/releases/$REVISION" ]] || + fail "Manipuliertes Bundle wurde in den Release-Store verschoben." + [[ ! -s "$FAKE_DEPLOY_LOG" ]] || + fail "Manipuliertes Bundle darf deploy.sh nicht ausführen." + [[ ! -s "$FAKE_SYSTEMCTL_LOG" && ! -s "$FAKE_INSTALL_LOG" ]] || + fail "Manipuliertes Bundle darf Systemd nicht verändern." + assert_file_contains 'docker|container rm fake-release-container' "$FAKE_DOCKER_LOG" + assert_no_temporary_artifacts +} + +test_stable_entrypoint_bootstraps_and_then_uses_current() { + local entrypoint_root="$TEST_ROOT/stable-entrypoint" + local output + + mkdir -p "$entrypoint_root/deploy/scripts" + cat >"$entrypoint_root/deploy/scripts/auto-deploy.sh" <<'SH' +#!/usr/bin/env bash +printf 'bootstrap\n' +SH + chmod 0755 "$entrypoint_root/deploy/scripts/auto-deploy.sh" + + output="$( + WATERMAPS_INSTALL_DIR="$entrypoint_root" \ + "$ROOT_DIR/deploy/scripts/auto-deploy-entrypoint.sh" + )" + [[ "$output" == "bootstrap" ]] || + fail "Stabiler Einstiegspunkt verwendet vor dem ersten Release nicht das Bootstrap-Skript." + + mkdir -p "$entrypoint_root/current/deploy/scripts" + cat >"$entrypoint_root/current/deploy/scripts/auto-deploy.sh" <<'SH' +#!/usr/bin/env bash +printf 'current\n' +SH + chmod 0755 "$entrypoint_root/current/deploy/scripts/auto-deploy.sh" + + output="$( + WATERMAPS_INSTALL_DIR="$entrypoint_root" \ + "$ROOT_DIR/deploy/scripts/auto-deploy-entrypoint.sh" + )" + [[ "$output" == "current" ]] || + fail "Stabiler Einstiegspunkt wechselt nach der Aktivierung nicht auf Current." +} + +test_stable_entrypoint_bootstraps_and_then_uses_current +test_already_active_repairs_current_link +test_successful_release_uses_manifest_digests +test_pointer_main_mismatch_does_not_deploy +test_failed_release_restores_previous_immutable_release +test_invalid_bundle_is_rejected_and_cleaned + +printf 'Auto-Deploy: Manifest-Digests, atomare Aktivierung und Rollback: OK\n' diff --git a/deploy/scripts/upload-and-deploy.sh b/deploy/scripts/upload-and-deploy.sh index 3b41715..eb5e9be 100755 --- a/deploy/scripts/upload-and-deploy.sh +++ b/deploy/scripts/upload-and-deploy.sh @@ -99,17 +99,6 @@ if [[ -n "$registry_username" || -n "$registry_token" ]]; then wm_local_die "WATERMAPS_REGISTRY_USERNAME enthält ungültige Zeichen." fi -images_env="$(mktemp)" -trap 'rm -f "$images_env"' EXIT -chmod 0600 "$images_env" -{ - printf 'WATERMAPS_DEPLOY_REVISION=%s\n' "$revision" - printf 'WATERMAPS_APP_IMAGE=%s/%s/watermaps:%s\n' \ - "$registry" "$registry_owner" "$revision" - printf 'WATERMAPS_ROUTE_DATA_IMAGE=%s/%s/watermaps-route-data:%s\n' \ - "$registry" "$registry_owner" "$revision" -} >"$images_env" - wm_local_resolve_ssh "$server_ipv4" "$identity_file" wm_local_wait_for_ssh @@ -124,9 +113,9 @@ printf '[watermaps] Warte auf Cloud-init und das persistente Hetzner-Volume.\n' ssh "${WM_SSH_OPTIONS[@]}" "$WM_SSH_TARGET" \ "cloud-init status --wait && ${remote_prefix}systemctl start watermaps-volume-setup.service && mountpoint --quiet /srv/watermaps-data" -printf '[watermaps] Übertrage Deployment-Dateien und Datenbankschema nach %s:/opt/watermaps\n' "$WM_SSH_TARGET" +printf '[watermaps] Übertrage Bootstrap-Dateien nach %s:/opt/watermaps\n' "$WM_SSH_TARGET" ssh "${WM_SSH_OPTIONS[@]}" "$WM_SSH_TARGET" \ - "${remote_prefix}install -d -m 0755 /opt/watermaps /opt/watermaps/deploy /opt/watermaps/database" + "${remote_prefix}install -d -m 0755 /opt/watermaps /opt/watermaps/deploy" rsync \ --archive \ @@ -136,18 +125,12 @@ rsync \ --rsync-path="$rsync_path" \ --exclude='.env.production' \ --exclude='.env.images*' \ + --exclude='.gitea-token' \ + --exclude='.gitea-*-token' \ -e "ssh ${WM_SSH_OPTIONS[*]@Q}" \ "$WM_DEPLOY_DIR/" \ "$WM_SSH_TARGET:/opt/watermaps/deploy/" -rsync \ - --archive \ - --chmod=F644 \ - --rsync-path="$rsync_path" \ - -e "ssh ${WM_SSH_OPTIONS[*]@Q}" \ - "$WM_LOCAL_ROOT_DIR/database/schema.sql" \ - "$WM_SSH_TARGET:/opt/watermaps/database/schema.sql" - rsync \ --archive \ --chmod=F600 \ @@ -156,14 +139,6 @@ rsync \ "$local_env" \ "$WM_SSH_TARGET:/opt/watermaps/deploy/.env.production" -rsync \ - --archive \ - --chmod=F600 \ - --rsync-path="$rsync_path" \ - -e "ssh ${WM_SSH_OPTIONS[*]@Q}" \ - "$images_env" \ - "$WM_SSH_TARGET:/opt/watermaps/deploy/.env.images.candidate" - if [[ -n "$registry_token" ]]; then printf '[watermaps] Aktualisiere den privaten Registry-Login auf dem Server.\n' printf '%s\n' "$registry_token" | @@ -173,14 +148,14 @@ fi remote_command="${remote_prefix}chmod +x /opt/watermaps/deploy/scripts/*.sh" remote_command+=" && ${remote_prefix}/opt/watermaps/deploy/scripts/bootstrap-server.sh" -remote_command+=" && ${remote_prefix}/opt/watermaps/deploy/scripts/deploy.sh --images-file /opt/watermaps/deploy/.env.images.candidate" +remote_command+=" && ${remote_prefix}WATERMAPS_ENV_FILE=/opt/watermaps/deploy/.env.production /opt/watermaps/deploy/scripts/auto-deploy.sh" +remote_command+=" && ${remote_prefix}test -L /opt/watermaps/current" +remote_command+=" && ${remote_prefix}grep -Fxq 'WATERMAPS_DEPLOY_REVISION=$revision' /opt/watermaps/deploy/.env.images" ssh "${WM_SSH_OPTIONS[@]}" "$WM_SSH_TARGET" "$remote_command" -ssh "${WM_SSH_OPTIONS[@]}" "$WM_SSH_TARGET" \ - "${remote_prefix}rm -f /opt/watermaps/deploy/.env.images.candidate" if [[ "$run_go_live" == "true" ]]; then ssh "${WM_SSH_OPTIONS[@]}" "$WM_SSH_TARGET" \ - "${remote_prefix}/opt/watermaps/deploy/scripts/go-live.sh '$WM_SERVER_IPV4'" + "${remote_prefix}WATERMAPS_ENV_FILE=/opt/watermaps/deploy/.env.production /opt/watermaps/current/deploy/scripts/go-live.sh '$WM_SERVER_IPV4'" else printf '[watermaps] Commit %s wurde aus der Registry deployt. Nach dem DNS-Eintrag:\n' "$revision" printf ' ./deploy/scripts/remote-go-live.sh --host %s\n' "$WM_SERVER_IPV4" diff --git a/deploy/systemd/watermaps-auto-deploy.service b/deploy/systemd/watermaps-auto-deploy.service new file mode 100644 index 0000000..9841b76 --- /dev/null +++ b/deploy/systemd/watermaps-auto-deploy.service @@ -0,0 +1,18 @@ +[Unit] +Description=Neues Watermaps-Release aus der Gitea Registry deployen +Wants=network-online.target +After=network-online.target docker.service watermaps-volume-setup.service +Requires=docker.service +RequiresMountsFor=/srv/watermaps-data + +[Service] +Type=oneshot +WorkingDirectory=/opt/watermaps +Environment=WATERMAPS_ENV_FILE=/opt/watermaps/deploy/.env.production +Environment=WATERMAPS_IMAGES_ENV_FILE=/opt/watermaps/deploy/.env.images +ExecStart=/usr/local/sbin/watermaps-auto-deploy +TimeoutStartSec=12h +UMask=0027 +Nice=5 +IOSchedulingClass=best-effort +IOSchedulingPriority=5 diff --git a/deploy/systemd/watermaps-auto-deploy.timer b/deploy/systemd/watermaps-auto-deploy.timer new file mode 100644 index 0000000..b9c735e --- /dev/null +++ b/deploy/systemd/watermaps-auto-deploy.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Gitea regelmäßig auf ein neues Watermaps-Release prüfen + +[Timer] +OnBootSec=3m +OnUnitInactiveSec=1m +RandomizedDelaySec=10s +Persistent=true +Unit=watermaps-auto-deploy.service + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/watermaps-certbot-renew.service b/deploy/systemd/watermaps-certbot-renew.service index da5a2a0..b01f5db 100644 --- a/deploy/systemd/watermaps-certbot-renew.service +++ b/deploy/systemd/watermaps-certbot-renew.service @@ -7,5 +7,7 @@ RequiresMountsFor=/srv/watermaps-data [Service] Type=oneshot -WorkingDirectory=/opt/watermaps -ExecStart=/opt/watermaps/deploy/scripts/renew-certificate.sh +WorkingDirectory=/opt/watermaps/current +Environment=WATERMAPS_ENV_FILE=/opt/watermaps/deploy/.env.production +Environment=WATERMAPS_IMAGES_ENV_FILE=/opt/watermaps/deploy/.env.images +ExecStart=/opt/watermaps/current/deploy/scripts/renew-certificate.sh diff --git a/deploy/systemd/watermaps-route-update.service b/deploy/systemd/watermaps-route-update.service index ce8f172..9c35051 100644 --- a/deploy/systemd/watermaps-route-update.service +++ b/deploy/systemd/watermaps-route-update.service @@ -7,8 +7,10 @@ RequiresMountsFor=/srv/watermaps-data [Service] Type=oneshot -WorkingDirectory=/opt/watermaps -ExecStart=/opt/watermaps/deploy/scripts/update-route-data.sh +WorkingDirectory=/opt/watermaps/current +Environment=WATERMAPS_ENV_FILE=/opt/watermaps/deploy/.env.production +Environment=WATERMAPS_IMAGES_ENV_FILE=/opt/watermaps/deploy/.env.images +ExecStart=/opt/watermaps/current/deploy/scripts/update-route-data.sh TimeoutStartSec=0 Nice=10 IOSchedulingClass=best-effort diff --git a/docker-compose.yml b/docker-compose.yml index 64384ff..25c91bc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,6 +52,7 @@ services: volumes: - postgres-data:/var/lib/postgresql/data - ./database/schema.sql:/docker-entrypoint-initdb.d/01-schema.sql:ro + - ./database/migrations:/docker-entrypoint-initdb.d/migrations:ro healthcheck: test: ["CMD-SHELL", "pg_isready -U seacompass -d seacompass"] interval: 10s diff --git a/infra/opentofu/README.md b/infra/opentofu/README.md index c913584..ea18169 100644 --- a/infra/opentofu/README.md +++ b/infra/opentofu/README.md @@ -33,9 +33,12 @@ chmod 600 terraform.tfvars ``` Auch `terraform.tfstate` bleibt lokal und wird nicht nach Git oder auf den -Anwendungsserver übertragen. Nach dem ersten Apply sollte die State-Datei -verschlüsselt gesichert werden, weil OpenTofu sie für spätere Änderungen an -denselben Ressourcen benötigt. +Anwendungsserver übertragen. Nach jedem Import oder Apply muss die State-Datei +mit Dateimodus `0600` verschlüsselt gesichert werden, weil OpenTofu sie für +spätere Änderungen an denselben Ressourcen benötigt. Ist ein älterer State +verloren, werden vorhandene Ressourcen vor jedem Apply anhand ihrer +Hetzner-IDs importiert; ein paralleles Neuerzeugen gleichnamiger Ressourcen ist +nicht zulässig. ## Infrastruktur erzeugen @@ -44,10 +47,15 @@ cd infra/opentofu tofu init tofu fmt -check tofu validate +umask 077 tofu plan -out=watermaps.tfplan tofu apply watermaps.tfplan ``` +Gespeicherte Plan-Dateien enthalten trotz als sensibel markierter Variablen +unter Umständen den Hetzner-Token. Sie müssen deshalb wie `terraform.tfvars` +und der State immer Modus `0600` haben und dürfen nicht weitergegeben werden. + Die feste IPv4-Adresse und den erforderlichen manuellen DNS-Eintrag zeigt OpenTofu anschließend an: @@ -79,3 +87,25 @@ Der Status des ersten Starts lässt sich so prüfen: ssh deploy@"$(tofu output -raw server_ipv4)" \ "cloud-init status --wait && systemctl status watermaps-volume-setup --no-pager" ``` + +Cloud-init bindet ausschließlich das von OpenTofu erzeugte Volume über den +stabilen Hetzner-Gerätepfad +`/dev/disk/by-id/scsi-0HC_Volume_` ein. Es wird nicht heuristisch +das erste Blockgerät mit dem Modellnamen `Volume` ausgewählt. Nach dem Mount +wird zusätzlich die UUID des tatsächlich eingehängten Geräts geprüft. Ein +Volume ohne lesbares Dateisystem wird standardmäßig nicht formatiert; nur bei +einem garantiert neuen, leeren Volume darf +`initialize_empty_routing_volume = true` für die einmalige Initialisierung +gesetzt werden. Bei Importen und Serverwechseln bleibt der Wert `false`. + +Der Löschschutz ist getrennt konfiguriert: + +- `enable_server_protection` für den austauschbaren Host, +- `enable_primary_ip_protection` für die feste öffentliche Adresse, +- `enable_volume_protection` für Routing-, Zertifikats- und PostGIS-Daten. + +Bei einem kontrollierten Servertausch bleiben IP- und Volume-Schutz immer +aktiv. Ein Plan, der `hcloud_primary_ip.main` oder +`hcloud_volume.routing_data` ersetzen beziehungsweise löschen will, darf nicht +angewendet werden. Zusätzlich verhindert OpenTofu für beide Ressourcen mit +`prevent_destroy` eine versehentliche Löschung durch den IaC-Plan. diff --git a/infra/opentofu/cloud-init.yaml.tftpl b/infra/opentofu/cloud-init.yaml.tftpl index b4d2813..913da07 100644 --- a/infra/opentofu/cloud-init.yaml.tftpl +++ b/infra/opentofu/cloud-init.yaml.tftpl @@ -50,36 +50,45 @@ write_files: set -euo pipefail mount_path="/srv/watermaps-data" + device="/dev/disk/by-id/scsi-0HC_Volume_${routing_volume_id}" + allow_initial_format="${initialize_empty_routing_volume}" install -d -m 0755 "$mount_path" - if ! mountpoint -q "$mount_path"; then - device="" - for attempt in $(seq 1 120); do - device="$(lsblk -dnpo NAME,MODEL | awk '$2 == "Volume" { print $1; exit }')" - if [ -n "$device" ] && [ -b "$device" ]; then - break - fi - device="" - sleep 5 - done + for attempt in $(seq 1 120); do + if [ -b "$device" ]; then + break + fi + sleep 5 + done - if [ -z "$device" ]; then - echo "Kein angehängtes Hetzner Cloud Volume gefunden." >&2 + if [ ! -b "$device" ]; then + echo "Hetzner Cloud Volume ${routing_volume_id} wurde nicht unter $device gefunden." >&2 + exit 1 + fi + + filesystem="$(blkid -o value -s TYPE "$device" || true)" + if [ -z "$filesystem" ]; then + signatures="$(wipefs --no-act --output TYPE --noheadings "$device" | tr -d '[:space:]')" + if [ -n "$signatures" ]; then + echo "Volume enthält unbekannte Signaturen und wird nicht verändert." >&2 exit 1 fi - - filesystem="$(blkid -o value -s TYPE "$device" || true)" - if [ -z "$filesystem" ]; then + if [ "$allow_initial_format" = "true" ]; then mkfs.ext4 -F "$device" filesystem="ext4" - fi - - if [ "$filesystem" != "ext4" ]; then - echo "Unerwartetes Dateisystem auf $device: $filesystem" >&2 + else + echo "Volume hat kein lesbares Dateisystem; automatische Formatierung ist deaktiviert." >&2 exit 1 fi + fi - uuid="$(blkid -o value -s UUID "$device")" + if [ "$filesystem" != "ext4" ]; then + echo "Unerwartetes Dateisystem auf $device: $filesystem" >&2 + exit 1 + fi + + uuid="$(blkid -o value -s UUID "$device")" + if ! mountpoint -q "$mount_path"; then if ! grep -q "^UUID=$uuid " /etc/fstab; then printf '%s\n' \ "UUID=$uuid $mount_path ext4 defaults,nofail,x-systemd.device-timeout=30 0 2" \ @@ -89,6 +98,13 @@ write_files: mount "$mount_path" fi + mounted_source="$(findmnt --noheadings --output SOURCE --target "$mount_path")" + mounted_uuid="$(blkid -o value -s UUID "$mounted_source" || true)" + if [ "$mounted_uuid" != "$uuid" ]; then + echo "Unter $mount_path ist nicht das erwartete Volume ${routing_volume_id} eingehängt." >&2 + exit 1 + fi + install -d -m 0755 -o ${deploy_user} -g ${deploy_user} \ "$mount_path/geofabrik" \ "$mount_path/local" \ diff --git a/infra/opentofu/main.tf b/infra/opentofu/main.tf index 227b0c5..6f23558 100644 --- a/infra/opentofu/main.tf +++ b/infra/opentofu/main.tf @@ -16,8 +16,12 @@ resource "hcloud_primary_ip" "main" { location = var.location type = "ipv4" auto_delete = false - delete_protection = var.enable_resource_protection + delete_protection = var.enable_primary_ip_protection labels = local.common_labels + + lifecycle { + prevent_destroy = true + } } resource "hcloud_firewall" "main" { @@ -75,8 +79,10 @@ resource "hcloud_server" "main" { firewall_ids = [hcloud_firewall.main.id] user_data = templatefile("${path.module}/cloud-init.yaml.tftpl", { - deploy_user = var.deploy_user - ssh_public_key = jsonencode(trimspace(var.ssh_public_key)) + deploy_user = var.deploy_user + ssh_public_key = jsonencode(trimspace(var.ssh_public_key)) + routing_volume_id = hcloud_volume.routing_data.id + initialize_empty_routing_volume = var.initialize_empty_routing_volume }) public_net { @@ -85,8 +91,8 @@ resource "hcloud_server" "main" { ipv6_enabled = false } - delete_protection = var.enable_resource_protection - rebuild_protection = var.enable_resource_protection + delete_protection = var.enable_server_protection + rebuild_protection = var.enable_server_protection shutdown_before_deletion = true labels = local.common_labels @@ -96,9 +102,12 @@ resource "hcloud_volume" "routing_data" { name = "${var.server_name}-routing-data" location = var.location size = var.routing_volume_size_gb - format = "ext4" - delete_protection = var.enable_resource_protection + delete_protection = var.enable_volume_protection labels = local.common_labels + + lifecycle { + prevent_destroy = true + } } resource "hcloud_volume_attachment" "routing_data" { diff --git a/infra/opentofu/terraform.tfvars.example b/infra/opentofu/terraform.tfvars.example index e10b359..b9924d7 100644 --- a/infra/opentofu/terraform.tfvars.example +++ b/infra/opentofu/terraform.tfvars.example @@ -19,5 +19,12 @@ server_type = "cx33" deploy_user = "deploy" routing_volume_size_gb = 40 -# Schützt Ressourcen vor versehentlicher Löschung über Console/API. -enable_resource_protection = true +# Nur für ein neu angelegtes, garantiert leeres Volume einmalig auf true +# setzen. Bei Import/Migration eines bestehenden Volumes immer false lassen. +initialize_empty_routing_volume = false + +# Der Server-Schutz kann für einen kontrollierten Hosttausch separat gelöst +# werden, ohne jemals IP- oder Volume-Schutz zu deaktivieren. +enable_server_protection = true +enable_primary_ip_protection = true +enable_volume_protection = true diff --git a/infra/opentofu/variables.tf b/infra/opentofu/variables.tf index 64c4772..8a3b93e 100644 --- a/infra/opentofu/variables.tf +++ b/infra/opentofu/variables.tf @@ -115,8 +115,26 @@ variable "routing_volume_size_gb" { } } -variable "enable_resource_protection" { - description = "Aktiviert Hetzner-Löschschutz für Server, Primary IP und Volume." +variable "initialize_empty_routing_volume" { + description = "Erlaubt genau bei einem nachweislich leeren Volume die initiale ext4-Formatierung. Für importierte oder bestehende Volumes muss dies false bleiben." + type = bool + default = false +} + +variable "enable_server_protection" { + description = "Aktiviert Hetzner-Lösch- und Rebuild-Schutz für den austauschbaren Server." + type = bool + default = true +} + +variable "enable_primary_ip_protection" { + description = "Aktiviert Hetzner-Löschschutz für die persistente öffentliche IPv4." + type = bool + default = true +} + +variable "enable_volume_protection" { + description = "Aktiviert Hetzner-Löschschutz für das persistente Datenvolume." type = bool default = true } diff --git a/package-lock.json b/package-lock.json index c905494..ad1e950 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3841,15 +3841,15 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.7", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz", - "integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==", + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.8.tgz", + "integrity": "sha512-JZyDyq3D4AUifKTPOB7DELf6XsB3WdPuNxCtob1vFXPsSXhdAiHBWJ/tJ8HAc9aH84BK+5JFZLNkJKx3G9kzQg==", "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" }, "engines": { - "node": "18 || 20 || >=22" + "node": "20 || >=22" } }, "node_modules/browserslist": { @@ -4904,9 +4904,9 @@ "license": "MIT" }, "node_modules/filelist/node_modules/brace-expansion": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.2.tgz", - "integrity": "sha512-w5JZcKgdhDOgOwm8H+KgbosopHMuGcl6qbulwjtz3SM7I7P3yW1eAjzMPLrIE+NQ9vjgANKHWeMHnrT0OXW1oA==", + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.3.tgz", + "integrity": "sha512-DRdx5neNsG/QXbniLFWi2YmC/68oeOOmKz6zOjVk6ZS1ZLXgLIKqVEc6hWsmkjBbgii0SwaBTcJ5XKj5gzY/4A==", "dev": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index d146381..93d7fca 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "setup:local-routing": "./scripts/setup-local-routing.sh", "sync:euris-locks": "node scripts/sync-euris-locks.mjs", "test": "npm run build --workspace @watermaps/shared && npm run test --workspace @watermaps/shared && npm run test --workspace @watermaps/api && npm run test --workspace @watermaps/web && npm run test:local-routing && npm run test:deployment", - "test:deployment": "bash deploy/scripts/tests/image-references.test.sh && bash deploy/scripts/tests/route-data-helpers.test.sh && bash deploy/scripts/tests/update-route-data.test.sh", + "test:deployment": "bash deploy/scripts/tests/image-references.test.sh && bash deploy/scripts/tests/auto-deploy.test.sh && bash deploy/scripts/tests/route-data-helpers.test.sh && bash deploy/scripts/tests/update-route-data.test.sh", "test:e2e": "npm run test:e2e --workspace @watermaps/web", "test:local-routing": "PYTHONPATH=.tools/python python3 -m unittest discover -s scripts/tests -p 'test_*.py'", "typecheck": "npm run build --workspace @watermaps/shared && npm run typecheck --workspace @watermaps/shared && npm run typecheck --workspace @watermaps/api && npm run typecheck --workspace @watermaps/web"