From 12eee8d2113f32c4f945ebc61b11a2009d91820e Mon Sep 17 00:00:00 2001 From: BuTzZ Date: Fri, 24 Jul 2026 23:10:17 +0200 Subject: [PATCH] feat: add Docker/OpenTofu deployment and DE/NL routing --- .dockerignore | 5 + .env.example | 2 + .gitignore | 7 + Dockerfile | 45 ++ README.md | 125 +++++- apps/api/package.json | 3 +- apps/api/src/app.ts | 15 +- apps/api/src/env.ts | 7 + apps/api/src/server.ts | 24 ++ apps/api/src/services/fairways.ts | 132 ++++++ apps/api/src/services/features.ts | 14 +- apps/api/tests/api.test.ts | 218 ++++++++++ apps/api/tests/fairways.test.ts | 85 ++++ apps/api/tests/features.test.ts | 19 + deploy/.env.production.example | 31 ++ deploy/.gitignore | 2 + deploy/README.md | 107 +++++ deploy/compose.production.yml | 100 +++++ deploy/nginx/bootstrap.conf | 15 + deploy/nginx/https.conf.template | 67 +++ deploy/route-data.Dockerfile | 13 + deploy/scripts/bootstrap-server.sh | 111 +++++ deploy/scripts/common.sh | 363 ++++++++++++++++ deploy/scripts/deploy.sh | 58 +++ deploy/scripts/go-live.sh | 115 +++++ deploy/scripts/prepare-route-data.sh | 38 ++ deploy/scripts/remote-common.sh | 77 ++++ deploy/scripts/remote-go-live.sh | 48 +++ deploy/scripts/renew-certificate.sh | 33 ++ .../scripts/tests/route-data-helpers.test.sh | 74 ++++ .../scripts/tests/update-route-data.test.sh | 153 +++++++ deploy/scripts/update-route-data.sh | 207 +++++++++ deploy/scripts/upload-and-deploy.sh | 131 ++++++ .../systemd/watermaps-certbot-renew.service | 11 + deploy/systemd/watermaps-certbot-renew.timer | 11 + deploy/systemd/watermaps-route-update.service | 14 + deploy/systemd/watermaps-route-update.timer | 11 + docker-compose.yml | 43 +- infra/opentofu/.gitignore | 18 + infra/opentofu/.terraform.lock.hcl | 35 ++ infra/opentofu/README.md | 73 ++++ infra/opentofu/cloud-init.yaml.tftpl | 138 ++++++ infra/opentofu/main.tf | 108 +++++ infra/opentofu/outputs.tf | 33 ++ infra/opentofu/provider.tf | 3 + infra/opentofu/terraform.tfvars.example | 23 + infra/opentofu/variables.tf | 122 ++++++ infra/opentofu/versions.tf | 10 + package-lock.json | 243 ++++++++--- package.json | 10 +- packages/shared/src/fairway-routing.ts | 268 ++++++++---- packages/shared/src/inland-seed.ts | 55 +++ packages/shared/src/route.ts | 8 +- packages/shared/tests/route.test.ts | 120 ++++++ scripts/build-local-fairways.py | 392 ++++++++++++++++++ scripts/download-geofabrik.sh | 112 ++++- scripts/setup-local-germany.sh | 6 + scripts/setup-local-routing.sh | 95 +++++ scripts/tests/test_build_local_fairways.py | 194 +++++++++ 59 files changed, 4452 insertions(+), 148 deletions(-) create mode 100644 Dockerfile create mode 100644 deploy/.env.production.example create mode 100644 deploy/.gitignore create mode 100644 deploy/README.md create mode 100644 deploy/compose.production.yml create mode 100644 deploy/nginx/bootstrap.conf create mode 100644 deploy/nginx/https.conf.template create mode 100644 deploy/route-data.Dockerfile create mode 100755 deploy/scripts/bootstrap-server.sh create mode 100755 deploy/scripts/common.sh create mode 100755 deploy/scripts/deploy.sh create mode 100755 deploy/scripts/go-live.sh create mode 100755 deploy/scripts/prepare-route-data.sh create mode 100755 deploy/scripts/remote-common.sh create mode 100755 deploy/scripts/remote-go-live.sh create mode 100755 deploy/scripts/renew-certificate.sh create mode 100755 deploy/scripts/tests/route-data-helpers.test.sh create mode 100755 deploy/scripts/tests/update-route-data.test.sh create mode 100755 deploy/scripts/update-route-data.sh create mode 100755 deploy/scripts/upload-and-deploy.sh create mode 100644 deploy/systemd/watermaps-certbot-renew.service create mode 100644 deploy/systemd/watermaps-certbot-renew.timer create mode 100644 deploy/systemd/watermaps-route-update.service create mode 100644 deploy/systemd/watermaps-route-update.timer create mode 100644 infra/opentofu/.gitignore create mode 100644 infra/opentofu/.terraform.lock.hcl create mode 100644 infra/opentofu/README.md create mode 100644 infra/opentofu/cloud-init.yaml.tftpl create mode 100644 infra/opentofu/main.tf create mode 100644 infra/opentofu/outputs.tf create mode 100644 infra/opentofu/provider.tf create mode 100644 infra/opentofu/terraform.tfvars.example create mode 100644 infra/opentofu/variables.tf create mode 100644 infra/opentofu/versions.tf create mode 100755 scripts/build-local-fairways.py create mode 100755 scripts/setup-local-germany.sh create mode 100755 scripts/setup-local-routing.sh create mode 100644 scripts/tests/test_build_local_fairways.py diff --git a/.dockerignore b/.dockerignore index c773b80..837019c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,8 +5,13 @@ dist coverage playwright-report test-results +__pycache__ +*.py[cod] data/geofabrik/*.osm.pbf data/geofabrik/*.osm.pbf.md5 +data/geofabrik/*.part +data/geofabrik/*.expected-md5 +data/local/*.json .env .env.* .DS_Store diff --git a/.env.example b/.env.example index b6c7e64..cd069e3 100644 --- a/.env.example +++ b/.env.example @@ -3,6 +3,8 @@ HOST=0.0.0.0 DATABASE_URL=postgres://seacompass:seacompass@localhost:55432/seacompass REDIS_URL=redis://localhost:6379 WATERMAPS_DEMO_DATA=true +WATERMAPS_LOCAL_FAIRWAYS_PATH=data/local/germany-netherlands-fairways.json +WATERMAPS_LIVE_FAIRWAYS=false # Serverseitiger EuRIS-Schleusenabgleich (`npm run sync:euris-locks`) EURIS_COUNTRIES=DE diff --git a/.gitignore b/.gitignore index 0cccdc5..3f944e6 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,11 @@ dist/ coverage/ playwright-report/ test-results/ +.playwright-cli/ +output/playwright/ *.tsbuildinfo +__pycache__/ +*.py[cod] .env .env.* !.env.example @@ -12,4 +16,7 @@ test-results/ .tools/ data/geofabrik/*.osm.pbf data/geofabrik/*.osm.pbf.md5 +data/geofabrik/*.part +data/geofabrik/*.expected-md5 +data/local/*.json /gitlogin diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..36efd24 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,45 @@ +FROM node:22-bookworm-slim AS build + +WORKDIR /app + +COPY package.json package-lock.json tsconfig.base.json ./ +COPY apps/api/package.json apps/api/package.json +COPY apps/web/package.json apps/web/package.json +COPY packages/shared/package.json packages/shared/package.json + +RUN npm ci + +COPY apps ./apps +COPY packages ./packages + +RUN npm run build \ + && npm prune --omit=dev + +FROM node:22-bookworm-slim AS runtime + +ENV NODE_ENV=production \ + HOST=0.0.0.0 \ + PORT=5174 \ + WATERMAPS_WEB_DIST_PATH=/app/apps/web/dist \ + WATERMAPS_LOCAL_FAIRWAYS_PATH=/data/germany-netherlands-fairways.json \ + WATERMAPS_LIVE_FAIRWAYS=false \ + WATERMAPS_DEMO_DATA=false + +WORKDIR /app + +COPY --from=build /app/package.json /app/package-lock.json ./ +COPY --from=build /app/node_modules ./node_modules +COPY --from=build /app/apps/api/package.json ./apps/api/package.json +COPY --from=build /app/apps/api/dist ./apps/api/dist +COPY --from=build /app/apps/web/dist ./apps/web/dist +COPY --from=build /app/packages/shared/package.json ./packages/shared/package.json +COPY --from=build /app/packages/shared/dist ./packages/shared/dist + +EXPOSE 5174 + +HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ + CMD ["node", "-e", "fetch('http://127.0.0.1:5174/health').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] + +USER node + +CMD ["node", "apps/api/dist/server.js"] diff --git a/README.md b/README.md index 7160ea4..3fa0fe4 100644 --- a/README.md +++ b/README.md @@ -22,18 +22,37 @@ iPhone-taugliche Browser-PWA für Bootsfahrer: Karte, GPS, Kompass, Wetter/Welle ## Start -Dieses Projekt benötigt Node `>=20.19`. In dieser Arbeitskopie liegt eine lokale Node-Version unter `.tools/`; nutze sie so: +Der reguläre lokale Betrieb läuft als einzelner Produktionscontainer. Der +gemeinsame Fahrwasserindex für Deutschland und die Niederlande muss einmalig +vorhanden sein: + +```bash +npm run setup:local-routing +npm run docker:up +``` + +Web und API: `http://localhost:5173`
+Healthcheck: `http://localhost:5173/health` + +Die sichtbaren Kartenkacheln kommen weiterhin von den konfigurierten externen +Kartendiensten. Lokal gespeichert werden nur die OSM-Rohdaten und der daraus +erzeugte Routingindex. + +Stoppen: + +```bash +npm run docker:down +``` + +Für die Entwicklung ohne Container benötigt das Projekt Node `>=20.19`: ```bash -export PATH="$PWD/.tools/bin:$PATH" npm install -npm run build -npm run test npm run dev ``` -Web: `http://localhost:5173`
-API: `http://localhost:5174` +Dev-Web: `http://localhost:5173`
+Dev-API: `http://localhost:5174` Für GPS-Tests auf dem iPhone muss die App über HTTPS laufen. Starte dafür: @@ -43,13 +62,75 @@ npm run dev:https Web HTTPS: `https://localhost:5173` bzw. die von Vite ausgegebene `https://192.168...:5173`-Adresse im gleichen WLAN. Beim lokalen Dev-Zertifikat muss Safari die Zertifikatswarnung einmal akzeptieren; falls iOS Geolocation danach weiterhin blockiert, nutze ein vertrauenswürdiges lokales Zertifikat oder einen HTTPS-Tunnel. +## Hetzner-Deployment mit OpenTofu + +Die produktive Infrastruktur besteht aus einem Hetzner-Server, einer festen +IPv4, Firewall und einem persistenten Volume. Der Docker-Stack enthält nur die +App, Nginx/Certbot und den Wartungscontainer für Deutschland- und +Niederlande-Routendaten; Kartenkacheln werden nicht selbst gehostet. + +Die beiden lokalen, von Git ignorierten Konfigurationsdateien sind bereits +angelegt: + +- `infra/opentofu/terraform.tfvars`: hier den Hetzner-Cloud-Read/Write-Token + eintragen und `admin_cidrs` bei Bedarf auf die aktuelle öffentliche IP + aktualisieren. +- `deploy/.env.production`: hier eine echte E-Mail-Adresse für Let's Encrypt + als `WATERMAPS_ACME_EMAIL` eintragen. + +Danach wird die Infrastruktur erzeugt: + +```bash +cd infra/opentofu +tofu init +tofu plan -out=watermaps.tfplan +tofu apply watermaps.tfplan +tofu output server_ipv4 +cd ../.. +``` + +Sobald `server_ipv4` ausgegeben wurde, kann der manuelle DNS-A-Record +`watermaps.incoso.eu` auf diese IPv4 gesetzt werden. Das erste Deployment darf +bereits vor der DNS-Propagation laufen: + +```bash +./deploy/scripts/upload-and-deploy.sh \ + --identity ~/.ssh/watermaps_hetzner_ed25519 +``` + +Dabei werden die vollständigen Geofabrik-Extrakte für Deutschland und die +Niederlande auf dem persistenten Server-Volume geladen und der lokale +Routingindex erstellt. Vor dem SSL-Livegang liefert Port 80 außer +ACME-Challenges nur 404. + +Erst wenn der DNS-A-Record propagiert ist, wird HTTPS mit dem finalen manuellen +Befehl aktiviert: + +```bash +./deploy/scripts/remote-go-live.sh \ + --identity ~/.ssh/watermaps_hetzner_ed25519 +``` + +Das Skript prüft DNS, beide Länder im Routingindex sowie Testrouten, fordert +das Zertifikat an und schaltet anschließend dauerhaft auf HTTPS um. +Ausführliche Hinweise stehen in +[`infra/opentofu/README.md`](infra/opentofu/README.md) und +[`deploy/README.md`](deploy/README.md). + ## Datenstatus Die freie Datenstrategie ist bewusst als Fahr- und Planungshilfe umgesetzt. Die App zeigt Attribution und den Status `Nicht amtlich`, weil freie Karten-/Modelldaten keine amtlich zugelassene Seekarte ersetzen. ## Routingstatus -`POST /api/routes` nutzt zuerst lokale PostGIS-Fahrwasserdaten aus `marine_fairway_edges`. Wenn in der Datenbank kein passender Graph liegt, versucht die API live extrahierte OSM/OpenSeaMap-Fahrwasserdaten für die Bounding Box zwischen Start, Wegpunkten und Ziel. Lokale und Live-Fragmente werden topologisch zusammengeführt. Berücksichtigt werden unter anderem `navigation_line`, `recommended_track`, `fairway`, navigierbare Kanäle und explizit für Boote oder Schiffe freigegebene Flüsse. Gesperrte, private, stillgelegte oder im Bau befindliche Wege werden ausgeschlossen. +`POST /api/routes` kann Fahrwasserdaten aus PostGIS, dem lokalen +Geofabrik-Dateiindex und – wenn explizit aktiviert – live aus Overpass +zusammenführen. Die Produktionskonfiguration verwendet ausschließlich den +lokalen Deutschland-/Niederlande-Index; sie benötigt weder PostGIS noch eine +laufende Overpass-Verbindung. Berücksichtigt werden unter anderem +`navigation_line`, `recommended_track`, `fairway`, navigierbare Kanäle und +explizit für Boote oder Schiffe freigegebene Flüsse. Gesperrte, private, +stillgelegte oder im Bau befindliche Wege werden ausgeschlossen. Wenn freie Laufzeitdaten fehlen, stehen zwei klar als nicht amtlich markierte Fallback-Korridore bereit: @@ -112,7 +193,7 @@ Die Küsten-PBFs lassen sich reproduzierbar von Geofabrik laden und ohne lokal i ```bash ./scripts/download-geofabrik.sh -docker compose up -d postgres redis martin +docker compose --profile postgis up -d postgres redis martin DATABASE_URL=postgres://seacompass:seacompass@localhost:55432/seacompass \ ./scripts/import-geofabrik-docker.sh \ data/geofabrik/germany-latest.osm.pbf \ @@ -206,11 +287,37 @@ Der lokale PostGIS-Container nutzt standardmäßig Host-Port `55432`, damit er n ## Lokale Infrastruktur ```bash -docker compose up -d postgres redis martin +docker compose --profile postgis up -d postgres redis martin ``` Danach `.env` aus `.env.example` ableiten und für echte PostGIS-Features `WATERMAPS_DEMO_DATA=false` setzen. Die frühere Variable `SEA_COMPASS_DEMO_DATA` wird übergangsweise weiterhin akzeptiert. +### Dateibasiertes Deutschland-/Niederlande-Routing + +Wenn Docker/PostGIS nicht verfügbar ist, kann die API den vollständigen +Geofabrik-Extrakt beider Länder als kompakte lokale Fahrwasserdatei verwenden: + +```bash +npm run setup:local-routing +``` + +Der Aufbau liest beide PBFs in Streaming-Durchläufen, führt überlappende +OSM-Wege zusammen und erzeugt +`data/local/germany-netherlands-fairways.json`. Für einen ausschließlich +lokalen Betrieb: + +```dotenv +DATABASE_URL= +REDIS_URL= +WATERMAPS_LOCAL_FAIRWAYS_PATH=data/local/germany-netherlands-fairways.json +WATERMAPS_LIVE_FAIRWAYS=false +``` + +Damit benötigen Routen innerhalb des heruntergeladenen Datenstands weder +PostGIS noch eine laufende Overpass-Verbindung. Nach einem neuen +Geofabrik-Snapshot prüft `npm run setup:local-routing` die Prüfsummen und baut +den Index bei Bedarf atomar neu. + ## APIs - `GET /api/config` diff --git a/apps/api/package.json b/apps/api/package.json index a67503d..9a545ab 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -13,8 +13,9 @@ }, "dependencies": { "@fastify/cors": "^11.0.1", + "@fastify/static": "^10.1.2", "@watermaps/shared": "0.1.0", - "fastify": "^5.4.0", + "fastify": "^5.10.0", "ioredis": "^5.6.1", "pg": "^8.16.3", "zod": "^3.25.76" diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index 03fb6da..4600158 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -3,6 +3,7 @@ import Fastify, { type FastifyInstance } from "fastify"; import { z } from "zod"; import { buildFairwayRoutes, + EMDEN_EAST_EMS_GRAPH, EMDEN_HAMM_GRAPH, type FairwayGraph, type RouteOption, @@ -110,7 +111,8 @@ export async function buildServer(deps: AppDeps = {}): Promise cache, fetcher, liveEnabled: env.liveOsmFairways, - databaseUrl: env.databaseUrl + databaseUrl: env.databaseUrl, + localDataPath: env.localFairwaysPath }); const app = Fastify({ logger: { @@ -183,12 +185,21 @@ export async function buildServer(deps: AppDeps = {}): Promise return reply.code(400).send({ error: "invalid_route", details: parsed.error.flatten() }); } + let sourceError: unknown; const dynamicGraphs = await fairwayService.getGraphsForRoute(parsed.data).catch((error) => { + sourceError = error; app.log.warn({ error }, "fairway extraction failed"); return []; }); const route = buildRouteFromGraphs(parsed.data, dynamicGraphs); if (!route) { + if (sourceError) { + return reply.code(503).send({ + error: "fairway_sources_unavailable", + message: + "Fahrwasserdaten sind momentan nicht verfügbar. Prüfe den lokalen Deutschland-/Niederlande-Index oder versuche es später erneut." + }); + } return reply.code(422).send({ error: "no_fairway_route", message: @@ -224,7 +235,7 @@ function buildRouteFromGraphs(request: RouteRequest, graphs: FairwayGraph[]) { } } - for (const graph of [undefined, EMDEN_HAMM_GRAPH] as const) { + for (const graph of [undefined, EMDEN_EAST_EMS_GRAPH, EMDEN_HAMM_GRAPH] as const) { const routes = graph ? buildFairwayRoutes(request, graph) : buildFairwayRoutes(request); if (routes.length > 0) { return routeResultWithAlternatives(routes); diff --git a/apps/api/src/env.ts b/apps/api/src/env.ts index f6e5b6f..2b8a771 100644 --- a/apps/api/src/env.ts +++ b/apps/api/src/env.ts @@ -3,6 +3,7 @@ export type ApiEnv = { host: string; databaseUrl?: string; redisUrl?: string; + localFairwaysPath?: string; demoData: boolean; liveOsmFairways: boolean; }; @@ -13,6 +14,12 @@ export function loadEnv(env: NodeJS.ProcessEnv = process.env): ApiEnv { host: env.HOST ?? "0.0.0.0", databaseUrl: env.DATABASE_URL, redisUrl: env.REDIS_URL, + localFairwaysPath: + env.WATERMAPS_LOCAL_FAIRWAYS_PATH ?? + env.SEA_COMPASS_LOCAL_FAIRWAYS_PATH ?? + (env.NODE_ENV === "test" + ? undefined + : "data/local/germany-netherlands-fairways.json"), demoData: (env.WATERMAPS_DEMO_DATA ?? env.SEA_COMPASS_DEMO_DATA) !== "false", liveOsmFairways: (env.WATERMAPS_LIVE_FAIRWAYS ?? env.SEA_COMPASS_LIVE_FAIRWAYS) !== "false" && diff --git a/apps/api/src/server.ts b/apps/api/src/server.ts index 1ab2ffa..22fabf8 100644 --- a/apps/api/src/server.ts +++ b/apps/api/src/server.ts @@ -1,3 +1,4 @@ +import fastifyStatic from "@fastify/static"; import { existsSync } from "node:fs"; import { resolve } from "node:path"; import { buildServer } from "./app.js"; @@ -12,6 +13,29 @@ for (const envFile of [resolve(process.cwd(), ".env"), resolve(process.cwd(), ". const env = loadEnv(); const app = await buildServer({ env }); +const configuredWebDist = process.env.WATERMAPS_WEB_DIST_PATH; +const webDistCandidates = configuredWebDist + ? [resolve(configuredWebDist)] + : [ + resolve(process.cwd(), "apps/web/dist"), + resolve(process.cwd(), "../web/dist") + ]; +const webDistPath = webDistCandidates.find((candidate) => + existsSync(resolve(candidate, "index.html")) +); + +if (webDistPath) { + await app.register(fastifyStatic, { + root: webDistPath, + prefix: "/" + }); + app.setNotFoundHandler((request, reply) => { + if (request.method === "GET" && !request.url.startsWith("/api/")) { + return reply.sendFile("index.html"); + } + return reply.code(404).send({ error: "not_found" }); + }); +} try { await app.listen({ port: env.port, host: env.host }); diff --git a/apps/api/src/services/fairways.ts b/apps/api/src/services/fairways.ts index e7aee27..10af572 100644 --- a/apps/api/src/services/fairways.ts +++ b/apps/api/src/services/fairways.ts @@ -1,4 +1,6 @@ import pg from "pg"; +import { readFile } from "node:fs/promises"; +import { isAbsolute, resolve } from "node:path"; import type { Coordinate, FairwayEdge, FairwayGraph, FairwayNode, RouteRequest } from "@watermaps/shared"; import type { Cache } from "./cache.js"; import type { FetchLike } from "./http.js"; @@ -14,11 +16,25 @@ type OverpassResponse = { elements?: OverpassElement[]; }; +type LocalFairwayWay = { + id: string; + bbox: [number, number, number, number]; + tags?: Record; + coordinates: [number, number][]; +}; + +type LocalFairwayDocument = { + version: number; + source: string; + ways: LocalFairwayWay[]; +}; + type FairwayDeps = { cache: Cache; fetcher: FetchLike; liveEnabled: boolean; databaseUrl?: string; + localDataPath?: string; }; export type FairwayRow = { @@ -38,6 +54,7 @@ const OVERPASS_URL = "https://overpass-api.de/api/interpreter"; const CACHE_TTL_MS = 1000 * 60 * 60 * 12; const MAX_BBOX_SPAN_DEG = 3; const MAX_POSTGIS_BBOX_SPAN_DEG = 6; +const MAX_LOCAL_BBOX_SPAN_DEG = 15; const BBOX_MARGIN_DEG = 0.15; const ENDPOINT_SNAP_DEG = 0.0001; const CONNECTOR_DISTANCE_NM = 0.08; @@ -48,17 +65,21 @@ export class FairwayService { private readonly fetcher: FetchLike; private readonly liveEnabled: boolean; private readonly pool: pg.Pool | null; + private readonly localDataPath?: string; + private localDocument: Promise | null = null; constructor(deps: FairwayDeps) { this.cache = deps.cache; this.fetcher = deps.fetcher; this.liveEnabled = deps.liveEnabled; + this.localDataPath = deps.localDataPath; this.pool = deps.databaseUrl ? new pg.Pool({ connectionString: deps.databaseUrl }) : null; } async getGraphsForRoute(request: RouteRequest): Promise { const results = await Promise.allSettled([ this.getPostgisGraphForRoute(request), + this.getLocalGraphForRoute(request), this.getLiveGraphForRoute(request) ]); const graphs = results.flatMap((result) => @@ -118,6 +139,51 @@ export class FairwayService { }); } + private async getLocalGraphForRoute(request: RouteRequest): Promise { + const bbox = routeBbox( + [request.start, ...(request.waypoints ?? []), request.destination], + MAX_LOCAL_BBOX_SPAN_DEG + ); + if (!bbox || !this.localDataPath) { + return null; + } + + const document = await this.loadLocalDocument(); + if (!document) { + return null; + } + + return localFairwaysToGraph( + document.ways.filter((way) => bboxesIntersect(way.bbox, bbox)), + bbox, + document.source + ); + } + + private async loadLocalDocument(): Promise { + if (!this.localDocument) { + this.localDocument = readFirstExistingFile(localDataPathCandidates(this.localDataPath!)) + .then((raw) => JSON.parse(raw) as LocalFairwayDocument) + .then((document) => { + if (document.version !== 1 || !Array.isArray(document.ways)) { + throw new Error(`Unsupported local fairway data format: ${this.localDataPath}`); + } + return document; + }) + .catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") { + return null; + } + throw error; + }); + } + const document = await this.localDocument; + if (!document) { + this.localDocument = null; + } + return document; + } + private async getLiveGraphForRoute(request: RouteRequest): Promise { if (!this.liveEnabled) { return null; @@ -176,6 +242,29 @@ export class FairwayService { } } +async function readFirstExistingFile(paths: string[]): Promise { + let lastError: NodeJS.ErrnoException | undefined; + for (const path of paths) { + try { + return await readFile(path, "utf8"); + } catch (error) { + const fileError = error as NodeJS.ErrnoException; + if (fileError.code !== "ENOENT") { + throw error; + } + lastError = fileError; + } + } + throw lastError ?? new Error("No local fairway data path configured"); +} + +function localDataPathCandidates(path: string): string[] { + if (isAbsolute(path)) { + return [path]; + } + return [...new Set([resolve(process.cwd(), path), resolve(process.cwd(), "../..", path)])]; +} + function routeBbox(points: Coordinate[], maxSpanDeg = MAX_BBOX_SPAN_DEG): [number, number, number, number] | null { const lons = points.map((point) => point.lon); const lats = points.map((point) => point.lat); @@ -238,6 +327,37 @@ export function overpassToGraph(response: OverpassResponse, bbox: [number, numbe ); } +export function localFairwaysToGraph( + ways: LocalFairwayWay[], + bbox: [number, number, number, number], + source: string +): FairwayGraph | null { + return waysToGraph( + ways + .map((way) => { + const tags = way.tags ?? {}; + const coordinates = way.coordinates + .map(([lat, lon]) => ({ lat, lon })) + .filter(isValidCoordinate); + if (coordinates.length < 2 || !isRoutableWay(tags, coordinates)) { + return null; + } + + return { + id: `local-osm-way-${way.id}`, + name: tags.name ?? tags.ref ?? `OSM ${way.id}`, + coordinates, + minDepthM: parseDepth(tags), + source: `local-geofabrik-${source}`, + ...edgeRestrictions(tags) + }; + }) + .filter((way): way is NonNullable => way !== null), + `local-geofabrik-${bbox.map((value) => value.toFixed(3)).join("-")}`, + `Lokale Geofabrik-Fahrwasser (${source})` + ); +} + export function mergeConnectedFairwayGraphs(graphs: FairwayGraph[]): FairwayGraph | null { const nodes = new Map(); const edges = new Map(); @@ -351,6 +471,18 @@ function isValidCoordinate(coordinate: Coordinate) { ); } +function bboxesIntersect( + first: [number, number, number, number], + second: [number, number, number, number] +) { + return !( + first[2] < second[0] || + first[0] > second[2] || + first[3] < second[1] || + first[1] > second[3] + ); +} + function isRoutableWay(tags: Record, coordinates: Coordinate[]) { if ( ["no", "private"].includes(tags.access ?? "") || diff --git a/apps/api/src/services/features.ts b/apps/api/src/services/features.ts index e7d6550..fad5377 100644 --- a/apps/api/src/services/features.ts +++ b/apps/api/src/services/features.ts @@ -20,7 +20,7 @@ type FeatureCollection = { properties: Record; }>; metadata: { - source: "postgis" | "demo"; + source: "postgis" | "demo" | "unavailable"; warning?: string; deduplication?: { inputPoiCount: number; @@ -78,6 +78,18 @@ export class FeatureService { return this.getPostgisFeatures(query); } + if (!this.demoData) { + return { + type: "FeatureCollection", + features: [], + metadata: { + source: "unavailable", + warning: + "Keine lokale Feature-Datenbank konfiguriert. Kartenkacheln und Live-Dienste bleiben davon unberührt." + } + }; + } + const [minLon, minLat, maxLon, maxLat] = query.bbox; return { type: "FeatureCollection", diff --git a/apps/api/tests/api.test.ts b/apps/api/tests/api.test.ts index e68963d..3251196 100644 --- a/apps/api/tests/api.test.ts +++ b/apps/api/tests/api.test.ts @@ -184,6 +184,31 @@ describe("Watermaps API", () => { await app.close(); }); + it("reports unavailable fairway sources instead of claiming that no route exists", async () => { + const app = await buildServer({ + cache: createCache(), + fairwayService: { + async getGraphsForRoute() { + throw new AggregateError([new Error("local data missing"), new Error("Overpass timeout")]); + }, + async close() {} + } + }); + const response = await app.inject({ + method: "POST", + url: "/api/routes", + payload: { + start: { lat: 54.1749, lon: 12.0731 }, + destination: { lat: 54.1833, lon: 12.0928 }, + vesselProfile: { draughtM: 1.4, safetyReserveM: 0.5, cruiseSpeedKn: 6 } + } + }); + + expect(response.statusCode).toBe(503); + expect(response.json().error).toBe("fairway_sources_unavailable"); + await app.close(); + }); + it("returns a fairway route from Emden Außenhafen to Borkum Reede", async () => { const app = await buildServer({ cache: createCache() }); const response = await app.inject({ @@ -205,6 +230,103 @@ describe("Watermaps API", () => { await app.close(); }); + it("plans the Norddeich–Norderney route for the reported coordinates", async () => { + const app = await buildServer({ + cache: createCache(), + fairwayService: { + async getGraphsForRoute() { + return [ + { + id: "norddeich-norderney-test", + name: "Norddeich–Norderney", + maxSnapDistanceNm: 0.5, + nodes: [ + { id: "norddeich", coordinate: { lat: 53.6234, lon: 7.1559 } }, + { id: "fairway", coordinate: { lat: 53.66, lon: 7.16 } }, + { id: "norderney", coordinate: { lat: 53.7023, lon: 7.1658 } } + ], + edges: [ + { + id: "norddeich-approach", + name: "Norddeich Fahrwasser", + from: "norddeich", + to: "fairway", + minDepthM: null, + source: "local-geofabrik-test", + coordinates: [ + { lat: 53.6234, lon: 7.1559 }, + { lat: 53.66, lon: 7.16 } + ] + }, + { + id: "norderney-approach", + name: "Norderney Fahrwasser", + from: "fairway", + to: "norderney", + minDepthM: null, + source: "local-geofabrik-test", + coordinates: [ + { lat: 53.66, lon: 7.16 }, + { lat: 53.7023, lon: 7.1658 } + ] + } + ] + } + ]; + }, + async close() {} + } + }); + const response = await app.inject({ + method: "POST", + url: "/api/routes", + payload: { + start: { lat: 53.6234, lon: 7.1559 }, + destination: { lat: 53.7023, lon: 7.1658 }, + vesselProfile: { draughtM: 1.4, safetyReserveM: 0.5, cruiseSpeedKn: 6 } + } + }); + const body = response.json(); + + expect(response.statusCode).toBe(200); + expect(body.routingMode).toBe("fairway"); + expect(body.geometry.coordinates[0]).toEqual([7.1559, 53.6234]); + expect(body.geometry.coordinates.at(-1)).toEqual([7.1658, 53.7023]); + expect(body.distanceNm).toBeGreaterThan(4); + expect(body.distanceNm).toBeLessThan(6); + await app.close(); + }); + + it("routes from Emden into the eastern lower Ems when all dynamic sources fail", async () => { + const app = await buildServer({ + cache: createCache(), + fairwayService: { + async getGraphsForRoute() { + throw new AggregateError([new Error("PostGIS unavailable"), new Error("Overpass timeout")]); + }, + async close() {} + } + }); + const response = await app.inject({ + method: "POST", + url: "/api/routes", + payload: { + start: { lat: 53.3422, lon: 7.1871 }, + destination: { lat: 53.465, lon: 7.4734 }, + vesselProfile: { draughtM: 1.4, safetyReserveM: 0.5, cruiseSpeedKn: 6 } + } + }); + const body = response.json(); + + expect(response.statusCode).toBe(200); + expect(body.routingMode).toBe("fairway"); + expect(body.dataSources).toContain("fairway-graph:emden-east-ems-seed"); + expect(body.geometry.coordinates[0]).toEqual([7.1871, 53.3422]); + expect(body.geometry.coordinates.at(-1)?.[0]).toBeCloseTo(7.4734, 3); + expect(body.geometry.coordinates.at(-1)?.[1]).toBeCloseTo(53.465, 3); + await app.close(); + }); + it("returns the inland fallback route from Emden to Hamm", async () => { const app = await buildServer({ cache: createCache() }); const response = await app.inject({ @@ -290,6 +412,102 @@ describe("Watermaps API", () => { await app.close(); }); + it("uses a shared local component for the reported Emden-Delfzijl coordinates", async () => { + const coordinate = (lat: number, lon: number) => ({ lat, lon }); + const app = await buildServer({ + cache: createCache(), + fairwayService: { + async getGraphsForRoute() { + return [ + { + id: "local-geofabrik-component-snap", + name: "Lokaler Geofabrik-Komponententest", + maxSnapDistanceNm: 0.3, + nodes: [ + { id: "start-decoy-a", coordinate: coordinate(53.3416, 7.186) }, + { id: "start-decoy-b", coordinate: coordinate(53.342, 7.187) }, + { id: "destination-decoy-a", coordinate: coordinate(53.3282, 6.9304) }, + { id: "destination-decoy-b", coordinate: coordinate(53.3286, 6.9294) }, + { id: "shared-start", coordinate: coordinate(53.3395697, 7.1848883) }, + { id: "shared-east", coordinate: coordinate(53.3321722, 7.1329034) }, + { id: "shared-south", coordinate: coordinate(53.313849, 7.0011017) }, + { id: "shared-destination", coordinate: coordinate(53.3303531, 6.9334715) } + ], + edges: [ + { + id: "start-decoy", + name: "Nähere getrennte Startkante", + from: "start-decoy-a", + to: "start-decoy-b", + coordinates: [coordinate(53.3416, 7.186), coordinate(53.342, 7.187)], + minDepthM: null, + source: "closer-but-disconnected-start" + }, + { + id: "destination-decoy", + name: "Nähere getrennte Zielkante", + from: "destination-decoy-a", + to: "destination-decoy-b", + coordinates: [coordinate(53.3282, 6.9304), coordinate(53.3286, 6.9294)], + minDepthM: null, + source: "closer-but-disconnected-destination" + }, + { + id: "shared-east", + name: "Gemeinsamer lokaler Korridor Ost", + from: "shared-start", + to: "shared-east", + coordinates: [coordinate(53.3395697, 7.1848883), coordinate(53.3321722, 7.1329034)], + minDepthM: null, + source: "local-geofabrik-germany+netherlands" + }, + { + id: "shared-south", + name: "Gemeinsamer lokaler Korridor Süd", + from: "shared-east", + to: "shared-south", + coordinates: [coordinate(53.3321722, 7.1329034), coordinate(53.313849, 7.0011017)], + minDepthM: null, + source: "local-geofabrik-germany+netherlands" + }, + { + id: "shared-west", + name: "Gemeinsamer lokaler Korridor West", + from: "shared-south", + to: "shared-destination", + coordinates: [coordinate(53.313849, 7.0011017), coordinate(53.3303531, 6.9334715)], + minDepthM: null, + source: "local-geofabrik-germany+netherlands" + } + ] + } + ]; + }, + async close() {} + } + }); + const response = await app.inject({ + method: "POST", + url: "/api/routes", + payload: { + start: { lat: 53.3416, lon: 7.186 }, + destination: { lat: 53.3282, lon: 6.9304 }, + vesselProfile: { draughtM: 1, safetyReserveM: 0.3 } + } + }); + const body = response.json(); + + expect(response.statusCode).toBe(200); + expect(body.routingMode).toBe("fairway"); + expect(body.geometry.coordinates[0]).toEqual([7.186, 53.3416]); + expect(body.geometry.coordinates.at(-1)).toEqual([6.9304, 53.3282]); + expect(body.dataSources).toContain("local-geofabrik-germany+netherlands"); + expect(body.dataSources).not.toContain("fairway-graph:ems-borkum-seed"); + expect(body.dataSources).not.toContain("closer-but-disconnected-start"); + expect(body.dataSources).not.toContain("closer-but-disconnected-destination"); + await app.close(); + }); + it("returns distinct route alternatives when the waterway graph contains them", async () => { const coordinate = (lat: number, lon: number) => ({ lat, lon }); const edge = (id: string, from: string, to: string, coordinates: Array<{ lat: number; lon: number }>) => ({ diff --git a/apps/api/tests/fairways.test.ts b/apps/api/tests/fairways.test.ts index f4dcfdb..48c39a4 100644 --- a/apps/api/tests/fairways.test.ts +++ b/apps/api/tests/fairways.test.ts @@ -1,12 +1,97 @@ import { describe, expect, it } from "vitest"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { buildRoute } from "@watermaps/shared"; +import { createCache } from "../src/services/cache.js"; import { + FairwayService, fairwayRowsToGraph, + localFairwaysToGraph, mergeConnectedFairwayGraphs, overpassToGraph } from "../src/services/fairways.js"; describe("fairway graph extraction", () => { + it("builds a routable graph from the persistent local Geofabrik format", () => { + const graph = localFairwaysToGraph( + [ + { + id: "local-1", + bbox: [7.15, 53.62, 7.17, 53.71], + tags: { route: "ferry", name: "Norddeich–Norderney" }, + coordinates: [ + [53.6234, 7.1559], + [53.66, 7.16], + [53.7023, 7.1658] + ] + } + ], + [7.0, 53.47, 7.32, 53.85], + "germany-test.osm.pbf" + ); + const route = buildRoute( + { + start: { lat: 53.6234, lon: 7.1559 }, + destination: { lat: 53.7023, lon: 7.1658 }, + vesselProfile: { draughtM: 1.4, safetyReserveM: 0.5, cruiseSpeedKn: 6 } + }, + graph ?? undefined + ); + + expect(route?.routingMode).toBe("fairway"); + expect(route?.dataSources).toContain("local-geofabrik-germany-test.osm.pbf"); + }); + + it("allows country-wide requests against the persistent local index", async () => { + const temporaryDirectory = await mkdtemp(join(tmpdir(), "watermaps-local-span-")); + const localDataPath = join(temporaryDirectory, "germany-netherlands-fairways.json"); + const request = { + start: { lat: 52, lon: 4 }, + destination: { lat: 52, lon: 12 }, + vesselProfile: { draughtM: 1, safetyReserveM: 0.3 } + }; + + try { + await writeFile( + localDataPath, + JSON.stringify({ + version: 1, + source: "germany+netherlands", + ways: [ + { + id: "country-wide-test", + bbox: [4, 52, 12, 52], + tags: { waterway: "canal", name: "Country-wide test fairway" }, + coordinates: [ + [52, 4], + [52, 12] + ] + } + ] + }) + ); + + const service = new FairwayService({ + cache: createCache(), + fetcher: fetch, + liveEnabled: false, + localDataPath + }); + const graphs = await service.getGraphsForRoute(request); + const route = buildRoute(request, graphs[0]); + + expect(graphs).toHaveLength(1); + expect(route?.routingMode).toBe("fairway"); + expect(route?.dataSources).toContain( + "local-geofabrik-germany+netherlands" + ); + await service.close(); + } finally { + await rm(temporaryDirectory, { recursive: true, force: true }); + } + }); + it("builds a routable graph from PostGIS fairway rows", () => { const graph = fairwayRowsToGraph( [ diff --git a/apps/api/tests/features.test.ts b/apps/api/tests/features.test.ts index a9fba5a..d66c571 100644 --- a/apps/api/tests/features.test.ts +++ b/apps/api/tests/features.test.ts @@ -1,10 +1,29 @@ import { describe, expect, it } from "vitest"; import { deduplicateMarineContactFeatures, + FeatureService, normalizeDepthFeatureProperties, normalizeMarineFeatureProperties } from "../src/services/features.js"; +describe("marine feature data modes", () => { + it("does not expose demo markers when production disables demo data without PostGIS", async () => { + const service = new FeatureService({ + databaseUrl: undefined, + demoData: false + }); + + const result = await service.getFeatures({ + bbox: [5, 50, 15, 56], + layers: ["seamarks", "locks", "harbours"] + }); + + expect(result.features).toEqual([]); + expect(result.metadata.source).toBe("unavailable"); + await service.close(); + }); +}); + describe("marine feature normalization", () => { it("formats bridge clearance labels from known OSM height tags", () => { const properties = normalizeMarineFeatureProperties({ diff --git a/deploy/.env.production.example b/deploy/.env.production.example new file mode 100644 index 0000000..5eab590 --- /dev/null +++ b/deploy/.env.production.example @@ -0,0 +1,31 @@ +# Diese Datei nach deploy/.env.production kopieren. +# Sie ist zugleich eine POSIX-Shell- und Docker-Compose-kompatible Env-Datei. + +WATERMAPS_DOMAIN=watermaps.incoso.eu +WATERMAPS_ACME_EMAIL=REPLACE_WITH_REAL_EMAIL + +# Wird von go-live.sh geprüft. upload-and-deploy.sh übergibt die von OpenTofu +# ausgegebene IPv4 automatisch; bei direkter Serverausführung hier eintragen. +WATERMAPS_EXPECTED_IPV4= + +# Dauerhafte Routing-Rohdaten und der erzeugte Routingindex. +WATERMAPS_DATA_DIR=/srv/watermaps-data + +# Generierte Nginx-Konfiguration und lokale Sperrdateien. +WATERMAPS_RUNTIME_DIR=/srv/watermaps-runtime + +# Verhindert Speicherabbrüche beim vollständigen DE/NL-Indexaufbau auf 4-GB-Servern. +WATERMAPS_SWAP_SIZE_GB=4 + +WATERMAPS_APP_IMAGE=watermaps:production +WATERMAPS_NGINX_IMAGE=nginx:1.30.4-alpine +WATERMAPS_CERTBOT_IMAGE=certbot/certbot:v5.7.0 +WATERMAPS_ROUTE_DATA_IMAGE=watermaps-route-data:production + +# Nur auf true setzen, wenn bei jedem Deployment Deutschland und die +# Niederlande erneut geprüft und der Routingindex neu gebaut werden sollen. +WATERMAPS_REBUILD_ROUTE_DATA=false + +# Für einen Test gegen Let's Encrypt Staging auf true setzen. +# Das damit ausgestellte Zertifikat ist im Browser nicht vertrauenswürdig. +WATERMAPS_CERTBOT_STAGING=false diff --git a/deploy/.gitignore b/deploy/.gitignore new file mode 100644 index 0000000..13e0a5d --- /dev/null +++ b/deploy/.gitignore @@ -0,0 +1,2 @@ +.env.production +!.env.production.example diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..1ad0e3f --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,107 @@ +# Watermaps-Produktion + +Dieser Stack hostet die Watermaps-App und die lokalen Fahrrouten für +Deutschland und die Niederlande. Die sichtbaren Kartenkacheln bleiben externe +Dienste. Öffentlich gebunden werden ausschließlich TCP 80 und 443; die +Watermaps-App ist nur im internen Docker-Netz erreichbar. + +## Konfiguration + +```bash +cp deploy/.env.production.example deploy/.env.production +editor deploy/.env.production +``` + +Mindestens `WATERMAPS_ACME_EMAIL` muss angepasst werden. Der Hetzner-API-Token +gehört **nicht** in diese Datei. Er bleibt lokal in der ignorierten Datei +`infra/opentofu/terraform.tfvars` (alternativ kann der Provider +`TF_VAR_hcloud_token` lesen). Der Upload schließt Terraform-Variablen, Pläne, +State, lokale Env-Dateien und `.terraform` in jeder Verzeichnistiefe aus. + +Die private SSH-Keydatei wird ebenfalls nicht gespeichert. Sie kann beim +Deployment mit `--identity` oder über `WATERMAPS_SSH_KEY` angegeben werden. +Falls OpenTofu keinen Output `ssh_private_key_path` bereitstellt und der Key +nicht bereits über den SSH-Agenten verfügbar ist, ist eine dieser beiden +Angaben erforderlich. Der vorbereitete SSH-Benutzer heißt standardmäßig +`deploy`; die privilegierten Installationsschritte laufen über dessen +passwortloses `sudo`. + +## Deployment + +Nach `tofu apply` liest dieses Skript standardmäßig den Output `server_ipv4`, +überträgt das Projekt ohne State, lokale Daten oder Node-Module und startet den +Serverbootstrap sowie den Docker-Stack: + +```bash +./deploy/scripts/upload-and-deploy.sh --identity ~/.ssh/watermaps_hetzner_ed25519 +``` + +Alternativ: + +```bash +WATERMAPS_SERVER_IPV4=203.0.113.10 \ +WATERMAPS_SSH_KEY=~/.ssh/watermaps_hetzner_ed25519 \ +./deploy/scripts/upload-and-deploy.sh +``` + +Der erste Datenaufbau lädt die Geofabrik-Extrakte für Deutschland und die +Niederlande herunter und kann entsprechend der Serverleistung längere Zeit +dauern. Der produktive Index liegt auf dem Server unter: + +```text +/srv/watermaps-data/local/germany-netherlands-fairways.json +``` + +Der Upload wartet zuerst auf SSH und den Abschluss von Cloud-init. Anschließend +startet und prüft er `watermaps-volume-setup.service`. Ohne tatsächlich unter +`/srv/watermaps-data` eingehängtes Volume wird kein Download gestartet, damit +die großen PBF-Dateien nicht versehentlich auf dem Root-Dateisystem landen. +Für den kurzzeitigen Speicherpeak beim kombinierten Indexaufbau richtet der +Bootstrap zusätzlich die über `WATERMAPS_SWAP_SIZE_GB` konfigurierte, +persistente Swap-Reserve ein. + +Vor dem Livegang antwortet Nginx nur für ACME-Challenges. Alle anderen +HTTP-Anfragen erhalten 404. + +## DNS und SSL-Livegang + +Sobald der OpenTofu-Output bekannt ist, kann folgender DNS-Eintrag gesetzt +werden: + +```text +A watermaps.incoso.eu +``` + +Nach der DNS-Propagation wird der Livegang lokal ausgelöst: + +```bash +./deploy/scripts/remote-go-live.sh --identity ~/.ssh/watermaps_hetzner_ed25519 +``` + +Das Serverskript prüft, dass sämtliche A-Records ausschließlich auf die +erwartete IPv4 zeigen, testet den Routingindex mit je einer Route in +Deutschland und den Niederlanden, prüft den ACME-Webroot, fordert das +Zertifikat an und aktiviert erst anschließend HTTPS. HTTP leitet danach auf +HTTPS um. + +## Automatik + +`bootstrap-server.sh` installiert zwei systemd-Timer: + +- `watermaps-route-update.timer`: täglich neue Deutschland- und + Niederlande-Daten; bei Build- oder Routentestfehler bleibt der vorherige + Index aktiv. +- `watermaps-certbot-renew.timer`: zweimal täglich Certbot-Prüfung mit + anschließendem Nginx-Reload. + +Status und Logs: + +```bash +systemctl list-timers 'watermaps-*' +journalctl -u watermaps-route-update.service +journalctl -u watermaps-certbot-renew.service +docker compose \ + --project-directory /opt/watermaps \ + --env-file /opt/watermaps/deploy/.env.production \ + -f /opt/watermaps/deploy/compose.production.yml ps +``` diff --git a/deploy/compose.production.yml b/deploy/compose.production.yml new file mode 100644 index 0000000..68f429c --- /dev/null +++ b/deploy/compose.production.yml @@ -0,0 +1,100 @@ +name: watermaps-production + +services: + watermaps: + image: ${WATERMAPS_APP_IMAGE:-watermaps:production} + build: + context: . + dockerfile: Dockerfile + environment: + NODE_ENV: production + HOST: 0.0.0.0 + PORT: 5174 + DATABASE_URL: "" + REDIS_URL: "" + WATERMAPS_WEB_DIST_PATH: /app/apps/web/dist + WATERMAPS_LOCAL_FAIRWAYS_PATH: /data/germany-netherlands-fairways.json + WATERMAPS_LIVE_FAIRWAYS: "false" + WATERMAPS_DEMO_DATA: "false" + expose: + - "5174" + volumes: + - type: bind + source: ${WATERMAPS_DATA_DIR:-/srv/watermaps-data}/local + target: /data + read_only: true + restart: unless-stopped + init: true + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + healthcheck: + test: + - CMD + - node + - -e + - >- + fetch('http://127.0.0.1:5174/health') + .then(response => { if (!response.ok) process.exit(1) }) + .catch(() => process.exit(1)) + interval: 15s + timeout: 5s + start_period: 15s + retries: 5 + + nginx: + image: ${WATERMAPS_NGINX_IMAGE:-nginx:1.30.4-alpine} + depends_on: + watermaps: + condition: service_healthy + ports: + - "80:80" + - "443:443" + volumes: + - type: bind + source: ${WATERMAPS_RUNTIME_DIR:-/srv/watermaps-runtime}/nginx/conf.d + target: /etc/nginx/conf.d + read_only: true + - type: bind + source: ${WATERMAPS_DATA_DIR:-/srv/watermaps-data}/certbot/www + target: /var/www/certbot + read_only: true + - type: bind + source: ${WATERMAPS_DATA_DIR:-/srv/watermaps-data}/certbot/letsencrypt + target: /etc/letsencrypt + read_only: true + restart: unless-stopped + security_opt: + - no-new-privileges:true + healthcheck: + test: ["CMD", "nginx", "-t"] + interval: 30s + timeout: 5s + retries: 3 + + certbot: + image: ${WATERMAPS_CERTBOT_IMAGE:-certbot/certbot:v5.7.0} + profiles: ["maintenance"] + volumes: + - type: bind + source: ${WATERMAPS_DATA_DIR:-/srv/watermaps-data}/certbot/www + target: /var/www/certbot + - type: bind + source: ${WATERMAPS_DATA_DIR:-/srv/watermaps-data}/certbot/letsencrypt + target: /etc/letsencrypt + + route-data: + image: ${WATERMAPS_ROUTE_DATA_IMAGE:-watermaps-route-data:production} + build: + context: . + dockerfile: deploy/route-data.Dockerfile + profiles: ["maintenance"] + environment: + WATERMAPS_GEOFABRIK_DIR: /workspace/data/geofabrik + WATERMAPS_LOCAL_FAIRWAYS_PATH: /workspace/data/local/germany-netherlands-fairways.json + volumes: + - type: bind + source: ${WATERMAPS_DATA_DIR:-/srv/watermaps-data} + target: /workspace/data + restart: "no" diff --git a/deploy/nginx/bootstrap.conf b/deploy/nginx/bootstrap.conf new file mode 100644 index 0000000..b9fd00c --- /dev/null +++ b/deploy/nginx/bootstrap.conf @@ -0,0 +1,15 @@ +server { + listen 80 default_server; + listen [::]:80 default_server; + server_name _; + + location ^~ /.well-known/acme-challenge/ { + root /var/www/certbot; + default_type text/plain; + try_files $uri =404; + } + + location / { + return 404; + } +} diff --git a/deploy/nginx/https.conf.template b/deploy/nginx/https.conf.template new file mode 100644 index 0000000..3769327 --- /dev/null +++ b/deploy/nginx/https.conf.template @@ -0,0 +1,67 @@ +server { + listen 80 default_server; + listen [::]:80 default_server; + server_name _; + + location ^~ /.well-known/acme-challenge/ { + root /var/www/certbot; + default_type text/plain; + try_files $uri =404; + } + + location / { + return 404; + } +} + +server { + listen 80; + listen [::]:80; + server_name __WATERMAPS_DOMAIN__; + + location ^~ /.well-known/acme-challenge/ { + root /var/www/certbot; + default_type text/plain; + try_files $uri =404; + } + + location / { + return 308 https://__WATERMAPS_DOMAIN__$request_uri; + } +} + +server { + listen 443 ssl; + listen [::]:443 ssl; + http2 on; + server_name __WATERMAPS_DOMAIN__; + + ssl_certificate /etc/letsencrypt/live/__WATERMAPS_CERT_NAME__/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/__WATERMAPS_CERT_NAME__/privkey.pem; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 1d; + ssl_session_tickets off; + + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Content-Type-Options nosniff always; + add_header Referrer-Policy strict-origin-when-cross-origin always; + + client_max_body_size 2m; + + location / { + if ($host != "__WATERMAPS_DOMAIN__") { + return 404; + } + + proxy_pass http://watermaps:5174; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Proto https; + proxy_read_timeout 60s; + proxy_send_timeout 60s; + } +} diff --git a/deploy/route-data.Dockerfile b/deploy/route-data.Dockerfile new file mode 100644 index 0000000..441b3e3 --- /dev/null +++ b/deploy/route-data.Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.12-slim + +RUN apt-get update \ + && apt-get install --yes --no-install-recommends ca-certificates curl coreutils libexpat1 \ + && rm -rf /var/lib/apt/lists/* \ + && python3 -m pip install --no-cache-dir "osmium==4.3.1" + +WORKDIR /workspace + +COPY scripts ./scripts +COPY deploy/scripts/prepare-route-data.sh ./deploy/scripts/prepare-route-data.sh + +ENTRYPOINT ["/workspace/deploy/scripts/prepare-route-data.sh"] diff --git a/deploy/scripts/bootstrap-server.sh b/deploy/scripts/bootstrap-server.sh new file mode 100755 index 0000000..02f3af2 --- /dev/null +++ b/deploy/scripts/bootstrap-server.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail +export DEBIAN_FRONTEND=noninteractive + +if [[ "$(id -u)" -ne 0 ]]; then + printf 'Dieses Skript muss als root ausgeführt werden.\n' >&2 + exit 1 +fi + +WM_DEPLOY_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +WM_ROOT_DIR="$(cd "$WM_DEPLOY_DIR/.." && pwd)" +WM_ENV_FILE="${WATERMAPS_ENV_FILE:-$WM_DEPLOY_DIR/.env.production}" + +install_docker() { + if command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then + return + fi + + . /etc/os-release + case "${ID:-}" in + ubuntu|debian) ;; + *) + printf 'Nicht unterstützte Distribution für die automatische Docker-Installation: %s\n' "${ID:-unbekannt}" >&2 + exit 1 + ;; + esac + + apt-get update + apt-get install --yes ca-certificates curl gnupg + install -m 0755 -d /etc/apt/keyrings + curl --fail --silent --show-error --location \ + "https://download.docker.com/linux/$ID/gpg" \ + --output /etc/apt/keyrings/docker.asc + chmod a+r /etc/apt/keyrings/docker.asc + + architecture="$(dpkg --print-architecture)" + codename="${VERSION_CODENAME:?VERSION_CODENAME fehlt in /etc/os-release}" + printf 'deb [arch=%s signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/%s %s stable\n' \ + "$architecture" "$ID" "$codename" \ + >/etc/apt/sources.list.d/docker.list + + apt-get update + apt-get install --yes docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +} + +install_docker +apt-get update +apt-get install --yes bind9-dnsutils curl jq rsync +systemctl enable --now docker + +if [[ ! -f "$WM_ENV_FILE" ]]; then + install -m 0600 "$WM_DEPLOY_DIR/.env.production.example" "$WM_ENV_FILE" + printf 'Konfiguration aus Vorlage angelegt: %s\n' "$WM_ENV_FILE" +fi + +# shellcheck source=common.sh +source "$WM_DEPLOY_DIR/scripts/common.sh" +wm_load_env +wm_assert_data_mount + +ensure_swap_reserve() { + local swap_file="/swapfile" + local swap_size_gb="${WATERMAPS_SWAP_SIZE_GB:-4}" + + [[ "$swap_size_gb" =~ ^[1-9][0-9]*$ ]] && + ((swap_size_gb <= 16)) || + wm_die "WATERMAPS_SWAP_SIZE_GB muss eine ganze Zahl zwischen 1 und 16 sein." + + if swapon --show=NAME --noheadings | awk '{$1=$1; print}' | grep -Fxq "$swap_file"; then + return + fi + + wm_log "Persistente ${swap_size_gb}-GB-Swap-Reserve wird vorbereitet." + fallocate --length "${swap_size_gb}G" "$swap_file" + chmod 0600 "$swap_file" + mkswap --force "$swap_file" >/dev/null + if ! grep -Fq "$swap_file none swap sw 0 0" /etc/fstab; then + printf '%s\n' "$swap_file none swap sw 0 0" >>/etc/fstab + fi + swapon "$swap_file" +} + +ensure_swap_reserve + +install -d -m 0755 \ + "$WATERMAPS_DATA_DIR" \ + "$WATERMAPS_DATA_DIR/geofabrik" \ + "$WATERMAPS_DATA_DIR/local" \ + "$WATERMAPS_DATA_DIR/certbot" \ + "$WATERMAPS_DATA_DIR/certbot/www" \ + "$WATERMAPS_DATA_DIR/certbot/letsencrypt" \ + "$WATERMAPS_RUNTIME_DIR" \ + "$WATERMAPS_RUNTIME_DIR/nginx" \ + "$WATERMAPS_RUNTIME_DIR/nginx/conf.d" \ + "$WATERMAPS_RUNTIME_DIR/locks" + +if [[ ! -f "$WATERMAPS_RUNTIME_DIR/nginx/conf.d/default.conf" ]]; then + install -m 0644 \ + "$WM_DEPLOY_DIR/nginx/bootstrap.conf" \ + "$WATERMAPS_RUNTIME_DIR/nginx/conf.d/default.conf" +fi + +install -m 0644 "$WM_DEPLOY_DIR/systemd/watermaps-route-update.service" /etc/systemd/system/ +install -m 0644 "$WM_DEPLOY_DIR/systemd/watermaps-route-update.timer" /etc/systemd/system/ +install -m 0644 "$WM_DEPLOY_DIR/systemd/watermaps-certbot-renew.service" /etc/systemd/system/ +install -m 0644 "$WM_DEPLOY_DIR/systemd/watermaps-certbot-renew.timer" /etc/systemd/system/ +systemctl daemon-reload +systemctl enable --now watermaps-route-update.timer watermaps-certbot-renew.timer + +wm_log "Server-Bootstrap abgeschlossen." diff --git a/deploy/scripts/common.sh b/deploy/scripts/common.sh new file mode 100755 index 0000000..62c96ab --- /dev/null +++ b/deploy/scripts/common.sh @@ -0,0 +1,363 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +WM_DEPLOY_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +WM_ROOT_DIR="$(cd "$WM_DEPLOY_DIR/.." && pwd)" +WM_ENV_FILE="${WATERMAPS_ENV_FILE:-$WM_DEPLOY_DIR/.env.production}" +WM_COMPOSE_FILE="$WM_DEPLOY_DIR/compose.production.yml" + +wm_die() { + printf 'Fehler: %s\n' "$*" >&2 + exit 1 +} + +wm_log() { + printf '[watermaps] %s\n' "$*" +} + +wm_load_env() { + if [[ ! -f "$WM_ENV_FILE" ]]; then + wm_die "Konfiguration fehlt: $WM_ENV_FILE (Vorlage: deploy/.env.production.example)" + fi + + set -a + # shellcheck disable=SC1090 + source "$WM_ENV_FILE" + set +a + + WATERMAPS_DATA_DIR="${WATERMAPS_DATA_DIR:-/srv/watermaps-data}" + WATERMAPS_RUNTIME_DIR="${WATERMAPS_RUNTIME_DIR:-/srv/watermaps-runtime}" + export WATERMAPS_DATA_DIR WATERMAPS_RUNTIME_DIR + + wm_require_safe_absolute_dir "$WATERMAPS_DATA_DIR" + wm_require_safe_absolute_dir "$WATERMAPS_RUNTIME_DIR" +} + +wm_require_safe_absolute_dir() { + local directory="$1" + [[ "$directory" == /* ]] || wm_die "Pfad muss absolut sein: $directory" + [[ "$directory" != "/" ]] || wm_die "Das Wurzelverzeichnis darf nicht als Datenpfad verwendet werden." + [[ "$directory" != "/srv" ]] || wm_die "Bitte ein Unterverzeichnis von /srv verwenden." +} + +wm_assert_data_mount() { + mountpoint --quiet "$WATERMAPS_DATA_DIR" || + wm_die "Persistentes Datenvolume ist nicht unter $WATERMAPS_DATA_DIR eingehängt." +} + +wm_compose() { + docker compose \ + --project-directory "$WM_ROOT_DIR" \ + --env-file "$WM_ENV_FILE" \ + --file "$WM_COMPOSE_FILE" \ + "$@" +} + +wm_route_file() { + printf '%s/local/germany-netherlands-fairways.json\n' "$WATERMAPS_DATA_DIR" +} + +wm_route_marker() { + printf '%s/local/.germany-netherlands-fairways.ready\n' "$WATERMAPS_DATA_DIR" +} + +wm_acquire_route_lock() { + install -d -m 0755 "$WATERMAPS_RUNTIME_DIR/locks" + exec 9>"$WATERMAPS_RUNTIME_DIR/locks/route-update.lock" + if ! flock --nonblock 9; then + wm_log "Ein Routingdaten-Update oder Deployment läuft bereits." + return 1 + fi +} + +wm_assert_route_data() { + wm_route_data_ready || wm_die "Routingdaten sind nicht vollständig bereit." +} + +wm_route_data_ready() { + wm_route_data_files_ready "$(wm_route_file)" "$(wm_route_marker)" +} + +wm_marker_value() { + local marker="$1" + local requested_key="$2" + awk -F= -v requested_key="$requested_key" ' + $1 == requested_key { + value = substr($0, length($1) + 2) + matches += 1 + } + END { + if (matches != 1) { + exit 1 + } + print value + } + ' "$marker" +} + +wm_write_route_marker() { + local route_file="$1" + local marker="$2" + local temporary_marker + + [[ -s "$route_file" ]] || return 1 + chmod 0644 "$route_file" + temporary_marker="$(mktemp "$(dirname "$marker")/.routing-ready.XXXXXX")" + { + printf 'format_version=1\n' + printf 'generated_at=%s\n' "$(date --utc +%Y-%m-%dT%H:%M:%SZ)" + printf 'route_file_name=%s\n' "$(basename "$route_file")" + printf 'size_bytes=%s\n' "$(stat --format=%s "$route_file")" + printf 'sha256=%s\n' "$(sha256sum "$route_file" | awk '{ print $1 }')" + printf 'generator_version=2\n' + printf 'source=germany+netherlands\n' + } >"$temporary_marker" + chmod 0644 "$temporary_marker" + mv -f "$temporary_marker" "$marker" +} + +wm_route_data_files_ready() { + local route_file="$1" + local marker="$2" + local file_mode marker_format marker_file_name marker_size marker_checksum + local marker_generator marker_source actual_size actual_checksum + + if [[ ! -s "$route_file" ]]; then + wm_log "Routingindex fehlt oder ist leer: $route_file" + return 1 + fi + if [[ ! -s "$marker" ]]; then + wm_log "Bereitschaftsmarker fehlt: $marker" + return 1 + fi + if [[ ! "$marker" -nt "$route_file" ]]; then + wm_log "Bereitschaftsmarker ist älter als der Routingindex." + return 1 + fi + + if [[ ! -r "$route_file" ]]; then + wm_log "Routingindex ist für den prüfenden Benutzer nicht lesbar: $route_file" + return 1 + fi + file_mode="$(stat --format=%a "$route_file")" + if (( (8#$file_mode & 4) == 0 )); then + wm_log "Routingindex ist für den unprivilegierten App-Container nicht lesbar (Modus $file_mode)." + return 1 + fi + + if ! jq --exit-status ' + (.version == 1) + and (.generatorVersion == 2) + and (.source == "germany+netherlands") + and (.sources | type == "array" and length == 2) + and (([.sources[].region] | sort) == ["germany", "netherlands"]) + and (([.sources[].file] | sort) == [ + "germany-latest.osm.pbf", + "netherlands-latest.osm.pbf" + ]) + and (all(.sources[]; + (.checksumMd5 | type == "string" and test("^[0-9a-f]{32}$")) + and (.sizeBytes | type == "number" and . > 0) + )) + and (.ways | type == "array" and length > 0) + ' "$route_file" >/dev/null; then + wm_log "Routingindex enthält nicht exakt die erwarteten Deutschland-/Niederlande-Quellen." + return 1 + fi + + marker_format="$(wm_marker_value "$marker" format_version 2>/dev/null || true)" + marker_file_name="$(wm_marker_value "$marker" route_file_name 2>/dev/null || true)" + marker_size="$(wm_marker_value "$marker" size_bytes 2>/dev/null || true)" + marker_checksum="$(wm_marker_value "$marker" sha256 2>/dev/null || true)" + marker_generator="$(wm_marker_value "$marker" generator_version 2>/dev/null || true)" + marker_source="$(wm_marker_value "$marker" source 2>/dev/null || true)" + actual_size="$(stat --format=%s "$route_file")" + actual_checksum="$(sha256sum "$route_file" | awk '{ print $1 }')" + + if [[ "$marker_format" != "1" || + "$marker_file_name" != "$(basename "$route_file")" || + "$marker_size" != "$actual_size" || + ! "$marker_checksum" =~ ^[0-9a-f]{64}$ || + "$marker_checksum" != "$actual_checksum" || + "$marker_generator" != "2" || + "$marker_source" != "germany+netherlands" ]]; then + wm_log "Bereitschaftsmarker stimmt nicht mit dem Routingindex überein: $marker" + return 1 + fi + return 0 +} + +wm_wait_for_health() { + local service="$1" + local timeout_seconds="${2:-180}" + local container_id status + local deadline=$((SECONDS + timeout_seconds)) + + container_id="$(wm_compose ps --quiet "$service")" + [[ -n "$container_id" ]] || wm_die "Container für $service läuft nicht." + + while ((SECONDS < deadline)); do + status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container_id")" + case "$status" in + healthy|running) + wm_log "$service ist bereit." + return 0 + ;; + unhealthy|exited|dead) + docker inspect --format '{{json .State}}' "$container_id" >&2 || true + return 1 + ;; + esac + sleep 3 + done + + wm_log "Timeout beim Warten auf $service." + return 1 +} + +wm_smoke_test_route() { + wm_compose exec --no-TTY watermaps node --input-type=module --eval ' + const expectedSource = "local-geofabrik-germany+netherlands"; + const routeChecks = [ + { + name: "Emden–Ditzum", + start: { lat: 53.3422, lon: 7.1871 }, + destination: { lat: 53.465, lon: 7.4734 }, + minimumAlternatives: 2 + }, + { + name: "Norddeich–Norderney", + start: { lat: 53.6234, lon: 7.1559 }, + destination: { lat: 53.7023, lon: 7.1658 }, + minimumAlternatives: 2 + }, + { + name: "Emden–Delfzijl", + start: { lat: 53.3416, lon: 7.186 }, + destination: { lat: 53.3282, lon: 6.9304 }, + minimumAlternatives: 0, + minimumCoordinates: 30, + minimumDistanceNm: 9.5, + maximumDistanceNm: 10.5, + maximumSegmentNm: 2, + maximumLongitude: 7.19, + corridorCoordinates: [ + [7.1848883, 53.3395697], + [7.0011017, 53.313849], + [6.9427276, 53.3256427] + ] + }, + { + name: "Weesp–Utrecht", + start: { lat: 52.309, lon: 5.0423 }, + destination: { lat: 52.105, lon: 5.085 }, + minimumAlternatives: 2 + }, + { + name: "Lemmer–Sneek", + start: { lat: 52.844, lon: 5.71 }, + destination: { lat: 53.033, lon: 5.66 }, + minimumAlternatives: 2 + }, + { + name: "Smal Weesp (Niederlande)", + start: { lat: 52.3043984, lon: 5.0210794 }, + destination: { lat: 52.307897, lon: 5.0330976 }, + minimumAlternatives: 0 + } + ]; + + for (const routeCheck of routeChecks) { + const response = await fetch("http://127.0.0.1:5174/api/routes", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + start: routeCheck.start, + destination: routeCheck.destination, + vesselProfile: { draughtM: 1, safetyReserveM: 0.3 } + }) + }); + if (!response.ok) { + console.error(routeCheck.name, response.status, await response.text()); + process.exit(1); + } + + const route = await response.json(); + const alternatives = Array.isArray(route.alternatives) ? route.alternatives : []; + const routeCoordinates = route.geometry?.coordinates; + const distanceNm = (first, second) => { + const toRadians = (value) => value * Math.PI / 180; + const deltaLat = toRadians(second[1] - first[1]); + const deltaLon = toRadians(second[0] - first[0]); + const firstLat = toRadians(first[1]); + const secondLat = toRadians(second[1]); + const haversine = + Math.sin(deltaLat / 2) ** 2 + + Math.cos(firstLat) * Math.cos(secondLat) * Math.sin(deltaLon / 2) ** 2; + return 3440.065 * 2 * Math.atan2(Math.sqrt(haversine), Math.sqrt(1 - haversine)); + }; + const largestSegmentNm = Array.isArray(routeCoordinates) + ? routeCoordinates.slice(1).reduce( + (largest, coordinate, index) => + Math.max(largest, distanceNm(routeCoordinates[index], coordinate)), + 0 + ) + : Number.POSITIVE_INFINITY; + const expectedCorridorCoordinates = routeCheck.corridorCoordinates ?? []; + const corridorCoordinatesAreValid = + expectedCorridorCoordinates.length === 0 || + ( + Array.isArray(routeCoordinates) && + expectedCorridorCoordinates.every((expectedCoordinate) => + routeCoordinates.some( + (coordinate) => distanceNm(coordinate, expectedCoordinate) <= 0.15 + ) + ) + ); + const alternativeRoutesAreValid = alternatives.every((alternative) => + alternative.routingMode === "fairway" + && Array.isArray(alternative.geometry?.coordinates) + && alternative.geometry.coordinates.length >= 2 + && Array.isArray(alternative.dataSources) + && alternative.dataSources.includes(expectedSource) + ); + const routeSignatures = [ + route.geometry?.coordinates, + ...alternatives.map((alternative) => alternative.geometry?.coordinates) + ].map((coordinates) => JSON.stringify(coordinates)); + if ( + route.routingMode !== "fairway" || + !Array.isArray(routeCoordinates) || + routeCoordinates.length < (routeCheck.minimumCoordinates ?? 2) || + !Array.isArray(route.dataSources) || + !route.dataSources.includes(expectedSource) || + alternatives.length < routeCheck.minimumAlternatives || + route.distanceNm < (routeCheck.minimumDistanceNm ?? 0) || + route.distanceNm > (routeCheck.maximumDistanceNm ?? Number.POSITIVE_INFINITY) || + largestSegmentNm > (routeCheck.maximumSegmentNm ?? Number.POSITIVE_INFINITY) || + routeCoordinates.some( + ([lon]) => lon > (routeCheck.maximumLongitude ?? Number.POSITIVE_INFINITY) + ) || + !corridorCoordinatesAreValid || + !alternativeRoutesAreValid || + new Set(routeSignatures).size !== routeSignatures.length + ) { + console.error( + `${routeCheck.name}: Routen- oder Alternativenprüfung des lokalen Deutschland-/Niederlande-Index fehlgeschlagen.`, + JSON.stringify({ + routingMode: route.routingMode, + dataSources: route.dataSources, + distanceNm: route.distanceNm, + coordinateCount: routeCoordinates?.length, + largestSegmentNm, + alternativeCount: alternatives.length, + minimumAlternatives: routeCheck.minimumAlternatives, + corridorCoordinatesAreValid + }) + ); + process.exit(1); + } + } + ' +} diff --git a/deploy/scripts/deploy.sh b/deploy/scripts/deploy.sh new file mode 100755 index 0000000..e0197dd --- /dev/null +++ b/deploy/scripts/deploy.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +WM_DEPLOY_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=common.sh +source "$WM_DEPLOY_DIR/scripts/common.sh" +wm_load_env +wm_assert_data_mount + +if [[ "$(id -u)" -ne 0 ]]; then + wm_die "Dieses Skript muss als root ausgeführt werden." +fi + +install -d -m 0755 \ + "$WATERMAPS_DATA_DIR/geofabrik" \ + "$WATERMAPS_DATA_DIR/local" \ + "$WATERMAPS_DATA_DIR/certbot/www" \ + "$WATERMAPS_DATA_DIR/certbot/letsencrypt" \ + "$WATERMAPS_RUNTIME_DIR/nginx/conf.d" \ + "$WATERMAPS_RUNTIME_DIR/locks" + +wm_acquire_route_lock || + wm_die "Deployment abgebrochen, weil gerade Routingdaten aktualisiert werden." + +if [[ ! -f "$WATERMAPS_RUNTIME_DIR/nginx/conf.d/default.conf" ]]; then + install -m 0644 \ + "$WM_DEPLOY_DIR/nginx/bootstrap.conf" \ + "$WATERMAPS_RUNTIME_DIR/nginx/conf.d/default.conf" +fi + +wm_log "Anwendungsimage wird gebaut." +wm_compose build watermaps +wm_compose up --detach --remove-orphans watermaps nginx + +wm_wait_for_health watermaps 240 +wm_wait_for_health nginx 120 + +route_rebuild_required=false +update_args=() +if [[ "${WATERMAPS_REBUILD_ROUTE_DATA:-false}" == "true" ]]; then + route_rebuild_required=true + update_args+=(--force) +elif ! wm_route_data_ready; then + route_rebuild_required=true +fi + +if [[ "$route_rebuild_required" == "true" ]]; then + wm_log "Deutschland- und Niederlande-Routingdaten werden sicher vorbereitet." + WATERMAPS_ROUTE_LOCK_HELD=true \ + "$WM_DEPLOY_DIR/scripts/update-route-data.sh" "${update_args[@]}" +fi + +wm_assert_route_data +wm_wait_for_health watermaps 240 +wm_smoke_test_route + +wm_log "Deployment ist bereit. Vor dem Livegang liefert Port 80 nur ACME-Challenges und ansonsten HTTP 404." diff --git a/deploy/scripts/go-live.sh b/deploy/scripts/go-live.sh new file mode 100755 index 0000000..49eb472 --- /dev/null +++ b/deploy/scripts/go-live.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +WM_DEPLOY_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=common.sh +source "$WM_DEPLOY_DIR/scripts/common.sh" +wm_load_env +wm_assert_data_mount + +if [[ "$(id -u)" -ne 0 ]]; then + wm_die "Dieses Skript muss als root ausgeführt werden." +fi + +domain="${WATERMAPS_DOMAIN:-}" +email="${WATERMAPS_ACME_EMAIL:-}" +expected_ipv4="${1:-${WATERMAPS_EXPECTED_IPV4:-}}" + +[[ "$domain" =~ ^([A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)+[A-Za-z]{2,63}$ ]] || + wm_die "Ungültiger WATERMAPS_DOMAIN: $domain" +[[ "$email" == *@*.* && "$email" != "admin@example.com" ]] || + wm_die "Bitte WATERMAPS_ACME_EMAIL in deploy/.env.production konfigurieren." +[[ "$expected_ipv4" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]] || + wm_die "Erwartete Server-IPv4 fehlt. Als Argument übergeben oder WATERMAPS_EXPECTED_IPV4 setzen." + +wm_assert_route_data +wm_wait_for_health watermaps 120 +wm_wait_for_health nginx 60 +wm_smoke_test_route + +mapfile -t resolved_ipv4s < <(dig +short A "$domain" | awk '/^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/') +[[ "${#resolved_ipv4s[@]}" -gt 0 ]] || wm_die "Für $domain ist noch kein A-Record auflösbar." + +for resolved_ipv4 in "${resolved_ipv4s[@]}"; do + [[ "$resolved_ipv4" == "$expected_ipv4" ]] || + wm_die "$domain zeigt zusätzlich/abweichend auf $resolved_ipv4 statt ausschließlich auf $expected_ipv4." +done + +challenge_name="watermaps-preflight-$$" +challenge_dir="$WATERMAPS_DATA_DIR/certbot/www/.well-known/acme-challenge" +challenge_file="$challenge_dir/$challenge_name" +install -d -m 0755 "$challenge_dir" +printf 'watermaps-acme-preflight\n' >"$challenge_file" +trap 'rm -f "$challenge_file"' EXIT + +curl --fail --silent --show-error \ + --header "Host: $domain" \ + "http://127.0.0.1/.well-known/acme-challenge/$challenge_name" \ + | grep -qx 'watermaps-acme-preflight' || + wm_die "Nginx stellt den ACME-Webroot nicht korrekt bereit." + +certbot_args=( + certonly + --webroot + --webroot-path /var/www/certbot + --domain "$domain" + --email "$email" + --agree-tos + --non-interactive + --keep-until-expiring +) +cert_name="$domain" +if [[ "${WATERMAPS_CERTBOT_STAGING:-false}" == "true" ]]; then + cert_name="${domain}-staging" + certbot_args+=(--staging) +fi +certbot_args+=(--cert-name "$cert_name") + +wm_log "Let's-Encrypt-Zertifikat wird angefordert." +wm_compose --profile maintenance run --rm certbot "${certbot_args[@]}" + +certificate_path="$WATERMAPS_DATA_DIR/certbot/letsencrypt/live/$cert_name/fullchain.pem" +private_key_path="$WATERMAPS_DATA_DIR/certbot/letsencrypt/live/$cert_name/privkey.pem" +[[ -s "$certificate_path" && -s "$private_key_path" ]] || + wm_die "Certbot war beendet, aber Zertifikat oder privater Schlüssel fehlen." + +candidate="$(mktemp "$WATERMAPS_RUNTIME_DIR/nginx/conf.d/.https.XXXXXX")" +trap 'rm -f "$challenge_file" "$candidate"' EXIT +sed \ + -e "s/__WATERMAPS_DOMAIN__/$domain/g" \ + -e "s/__WATERMAPS_CERT_NAME__/$cert_name/g" \ + "$WM_DEPLOY_DIR/nginx/https.conf.template" \ + >"$candidate" + +wm_compose run --rm --no-deps \ + --volume "$candidate:/etc/nginx/conf.d/default.conf:ro" \ + nginx nginx -t + +active_config="$WATERMAPS_RUNTIME_DIR/nginx/conf.d/default.conf" +previous_config="$WATERMAPS_RUNTIME_DIR/nginx/conf.d/.default.conf.previous" +cp --preserve=mode,timestamps "$active_config" "$previous_config" +install -m 0644 "$candidate" "$active_config" + +if ! wm_compose exec --no-TTY nginx nginx -t; then + install -m 0644 "$previous_config" "$active_config" + wm_compose exec --no-TTY nginx nginx -s reload || true + wm_die "HTTPS-Konfiguration war ungültig; Bootstrap-Konfiguration wurde wiederhergestellt." +fi +wm_compose exec --no-TTY nginx nginx -s reload + +https_curl_args=(--fail --silent --show-error) +if [[ "${WATERMAPS_CERTBOT_STAGING:-false}" == "true" ]]; then + https_curl_args+=(--insecure) +fi +if ! curl "${https_curl_args[@]}" \ + --resolve "$domain:443:$expected_ipv4" \ + "https://$domain/health" \ + >/dev/null; then + install -m 0644 "$previous_config" "$active_config" + wm_compose exec --no-TTY nginx nginx -t + wm_compose exec --no-TTY nginx nginx -s reload + wm_die "HTTPS-Smoke-Test fehlgeschlagen; vorherige Nginx-Konfiguration wurde wiederhergestellt." +fi + +wm_log "Livegang erfolgreich: https://$domain" diff --git a/deploy/scripts/prepare-route-data.sh b/deploy/scripts/prepare-route-data.sh new file mode 100755 index 0000000..3d8cb69 --- /dev/null +++ b/deploy/scripts/prepare-route-data.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +cd /workspace + +route_file="${WATERMAPS_LOCAL_FAIRWAYS_PATH:-/workspace/data/local/germany-netherlands-fairways.json}" +marker_file="${WATERMAPS_ROUTE_MARKER_PATH:-$(dirname "$route_file")/.germany-netherlands-fairways.ready}" + +mkdir -p "$(dirname "$route_file")" /workspace/data/geofabrik + +if [[ ! -x /workspace/scripts/setup-local-routing.sh ]]; then + printf 'Fehler: scripts/setup-local-routing.sh fehlt oder ist nicht ausführbar.\n' >&2 + exit 1 +fi + +/workspace/scripts/setup-local-routing.sh + +if [[ ! -s "$route_file" ]]; then + printf 'Fehler: Der Routingindex wurde nicht erzeugt: %s\n' "$route_file" >&2 + exit 1 +fi + +chmod 0644 "$route_file" +temporary_marker="$(mktemp "$(dirname "$marker_file")/.routing-ready.XXXXXX")" +{ + printf 'format_version=1\n' + printf 'generated_at=%s\n' "$(date --utc +%Y-%m-%dT%H:%M:%SZ)" + printf 'route_file_name=%s\n' "$(basename "$route_file")" + printf 'size_bytes=%s\n' "$(stat --format=%s "$route_file")" + printf 'sha256=%s\n' "$(sha256sum "$route_file" | awk '{ print $1 }')" + printf 'generator_version=2\n' + printf 'source=germany+netherlands\n' +} >"$temporary_marker" +chmod 0644 "$temporary_marker" +mv -f "$temporary_marker" "$marker_file" + +printf 'Routingdaten sind bereit: %s\n' "$route_file" diff --git a/deploy/scripts/remote-common.sh b/deploy/scripts/remote-common.sh new file mode 100755 index 0000000..3ea7ab2 --- /dev/null +++ b/deploy/scripts/remote-common.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +WM_LOCAL_DEPLOY_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +WM_LOCAL_ROOT_DIR="$(cd "$WM_LOCAL_DEPLOY_DIR/.." && pwd)" +WM_LOCAL_INFRA_DIR="${WATERMAPS_INFRA_DIR:-$WM_LOCAL_ROOT_DIR/infra/opentofu}" + +wm_local_die() { + printf 'Fehler: %s\n' "$*" >&2 + exit 1 +} + +wm_local_resolve_ssh() { + local requested_server="${1:-}" + local requested_identity="${2:-}" + + WM_SERVER_IPV4="${requested_server:-${WATERMAPS_SERVER_IPV4:-}}" + if [[ -z "$WM_SERVER_IPV4" ]]; then + command -v tofu >/dev/null 2>&1 || + wm_local_die "OpenTofu fehlt und WATERMAPS_SERVER_IPV4 wurde nicht gesetzt." + [[ -d "$WM_LOCAL_INFRA_DIR" ]] || + wm_local_die "OpenTofu-Verzeichnis fehlt: $WM_LOCAL_INFRA_DIR" + WM_SERVER_IPV4="$(tofu -chdir="$WM_LOCAL_INFRA_DIR" output -raw server_ipv4)" + fi + [[ "$WM_SERVER_IPV4" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]] || + wm_local_die "Ungültige Server-IPv4: $WM_SERVER_IPV4" + + WM_SSH_IDENTITY="${requested_identity:-${WATERMAPS_SSH_KEY:-}}" + if [[ -z "$WM_SSH_IDENTITY" ]] && command -v tofu >/dev/null 2>&1 && [[ -d "$WM_LOCAL_INFRA_DIR" ]]; then + candidate_identity="$(tofu -chdir="$WM_LOCAL_INFRA_DIR" output -raw ssh_private_key_path 2>/dev/null || true)" + if [[ -n "$candidate_identity" && -f "$candidate_identity" ]]; then + WM_SSH_IDENTITY="$candidate_identity" + fi + fi + if [[ -n "$WM_SSH_IDENTITY" && ! -f "$WM_SSH_IDENTITY" ]]; then + wm_local_die "SSH-Keydatei nicht gefunden: $WM_SSH_IDENTITY" + fi + + WM_SSH_USER="${WATERMAPS_SSH_USER:-deploy}" + [[ "$WM_SSH_USER" =~ ^[a-z_][a-z0-9_-]{0,30}$ ]] || + wm_local_die "Ungültiger SSH-Benutzer: $WM_SSH_USER" + if [[ -n "${WATERMAPS_REMOTE_SUDO+x}" ]]; then + WM_REMOTE_SUDO="$WATERMAPS_REMOTE_SUDO" + elif [[ "$WM_SSH_USER" == "root" ]]; then + WM_REMOTE_SUDO="" + else + WM_REMOTE_SUDO="sudo" + fi + case "$WM_REMOTE_SUDO" in + ""|sudo) ;; + *) wm_local_die "WATERMAPS_REMOTE_SUDO darf nur leer oder 'sudo' sein." ;; + esac + + WM_SSH_TARGET="$WM_SSH_USER@$WM_SERVER_IPV4" + WM_SSH_OPTIONS=(-o BatchMode=yes -o StrictHostKeyChecking=accept-new) + if [[ -n "$WM_SSH_IDENTITY" ]]; then + WM_SSH_OPTIONS+=(-i "$WM_SSH_IDENTITY" -o IdentitiesOnly=yes) + fi +} + +wm_local_wait_for_ssh() { + local timeout_seconds="${WATERMAPS_SSH_WAIT_SECONDS:-600}" + local deadline=$((SECONDS + timeout_seconds)) + + while ((SECONDS < deadline)); do + if ssh "${WM_SSH_OPTIONS[@]}" \ + -o ConnectTimeout=5 \ + "$WM_SSH_TARGET" true >/dev/null 2>&1; then + return 0 + fi + printf '[watermaps] Warte auf SSH und Cloud-init-Benutzer %s …\n' "$WM_SSH_TARGET" + sleep 5 + done + + wm_local_die "SSH war nach ${timeout_seconds}s nicht erreichbar: $WM_SSH_TARGET" +} diff --git a/deploy/scripts/remote-go-live.sh b/deploy/scripts/remote-go-live.sh new file mode 100755 index 0000000..091df73 --- /dev/null +++ b/deploy/scripts/remote-go-live.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +WM_DEPLOY_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=remote-common.sh +source "$WM_DEPLOY_DIR/scripts/remote-common.sh" + +server_ipv4="" +identity_file="" + +while [[ "$#" -gt 0 ]]; do + case "$1" in + --host) + [[ "$#" -ge 2 ]] || wm_local_die "Wert für --host fehlt." + server_ipv4="$2" + shift 2 + ;; + --identity) + [[ "$#" -ge 2 ]] || wm_local_die "Wert für --identity fehlt." + identity_file="$2" + shift 2 + ;; + --user) + [[ "$#" -ge 2 ]] || wm_local_die "Wert für --user fehlt." + WATERMAPS_SSH_USER="$2" + export WATERMAPS_SSH_USER + shift 2 + ;; + -h|--help) + printf 'Verwendung: %s [--host IPV4] [--identity DATEI] [--user BENUTZER]\n' "$0" + exit 0 + ;; + *) + wm_local_die "Unbekannte Option: $1" + ;; + esac +done + +wm_local_resolve_ssh "$server_ipv4" "$identity_file" +wm_local_wait_for_ssh +remote_prefix="" +if [[ -n "$WM_REMOTE_SUDO" ]]; then + remote_prefix="$WM_REMOTE_SUDO " +fi + +ssh "${WM_SSH_OPTIONS[@]}" "$WM_SSH_TARGET" \ + "${remote_prefix}/opt/watermaps/deploy/scripts/go-live.sh '$WM_SERVER_IPV4'" diff --git a/deploy/scripts/renew-certificate.sh b/deploy/scripts/renew-certificate.sh new file mode 100755 index 0000000..4affa0f --- /dev/null +++ b/deploy/scripts/renew-certificate.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +WM_DEPLOY_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=common.sh +source "$WM_DEPLOY_DIR/scripts/common.sh" +wm_load_env +wm_assert_data_mount + +if [[ "$(id -u)" -ne 0 ]]; then + wm_die "Dieses Skript muss als root ausgeführt werden." +fi + +domain="${WATERMAPS_DOMAIN:-}" +cert_name="$domain" +if [[ "${WATERMAPS_CERTBOT_STAGING:-false}" == "true" ]]; then + cert_name="${domain}-staging" +fi +[[ -s "$WATERMAPS_DATA_DIR/certbot/letsencrypt/live/$cert_name/fullchain.pem" ]] || { + wm_log "Noch kein Zertifikat vorhanden; Erneuerung wird übersprungen." + exit 0 +} + +wm_compose --profile maintenance run --rm certbot \ + renew \ + --webroot \ + --webroot-path /var/www/certbot \ + --quiet + +wm_compose exec --no-TTY nginx nginx -t +wm_compose exec --no-TTY nginx nginx -s reload +wm_log "Zertifikatserneuerung geprüft und Nginx neu geladen." diff --git a/deploy/scripts/tests/route-data-helpers.test.sh b/deploy/scripts/tests/route-data-helpers.test.sh new file mode 100755 index 0000000..e573d0e --- /dev/null +++ b/deploy/scripts/tests/route-data-helpers.test.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +TEST_DIR="$(mktemp -d)" +trap 'rm -rf "$TEST_DIR"' EXIT + +export WATERMAPS_DATA_DIR="$TEST_DIR/data" +export WATERMAPS_RUNTIME_DIR="$TEST_DIR/runtime" +mkdir -p "$WATERMAPS_DATA_DIR/local" "$WATERMAPS_RUNTIME_DIR" + +# shellcheck source=../common.sh +source "$ROOT_DIR/deploy/scripts/common.sh" + +route_file="$(wm_route_file)" +route_marker="$(wm_route_marker)" + +write_valid_route() { + cat >"$route_file" <<'JSON' +{ + "version": 1, + "generatorVersion": 2, + "source": "germany+netherlands", + "sources": [ + { + "region": "germany", + "file": "germany-latest.osm.pbf", + "checksumMd5": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "sizeBytes": 1 + }, + { + "region": "netherlands", + "file": "netherlands-latest.osm.pbf", + "checksumMd5": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "sizeBytes": 1 + } + ], + "ways": [{"id": "1"}] +} +JSON + chmod 0644 "$route_file" +} + +write_valid_route +wm_write_route_marker "$route_file" "$route_marker" +wm_route_data_ready + +sed -i 's/^size_bytes=.*/size_bytes=1/' "$route_marker" +if wm_route_data_ready; then + printf 'Ungültige Markergröße wurde akzeptiert.\n' >&2 + exit 1 +fi + +wm_write_route_marker "$route_file" "$route_marker" +jq '.sources = [.sources[0]] | .source = "germany"' \ + "$route_file" >"$route_file.tmp" +mv "$route_file.tmp" "$route_file" +chmod 0644 "$route_file" +wm_write_route_marker "$route_file" "$route_marker" +if wm_route_data_ready; then + printf 'Deutschland-only-Index wurde akzeptiert.\n' >&2 + exit 1 +fi + +write_valid_route +wm_write_route_marker "$route_file" "$route_marker" +chmod 0600 "$route_file" +if wm_route_data_ready; then + printf 'Für den App-Container unlesbarer Index wurde akzeptiert.\n' >&2 + exit 1 +fi + +printf 'Routingdaten- und Markerprüfungen: OK\n' diff --git a/deploy/scripts/tests/update-route-data.test.sh b/deploy/scripts/tests/update-route-data.test.sh new file mode 100755 index 0000000..50b5fed --- /dev/null +++ b/deploy/scripts/tests/update-route-data.test.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +TEST_DIR="$(mktemp -d)" +trap 'rm -rf "$TEST_DIR"' EXIT + +FAKE_BIN="$TEST_DIR/bin" +DATA_DIR="$TEST_DIR/data" +RUNTIME_DIR="$TEST_DIR/runtime" +ENV_FILE="$TEST_DIR/production.env" +mkdir -p "$FAKE_BIN" "$DATA_DIR/local" "$RUNTIME_DIR" + +cat >"$ENV_FILE" <"$FAKE_BIN/id" <<'SH' +#!/usr/bin/env bash +if [[ "${1:-}" == "-u" ]]; then + printf '0\n' +else + exec /usr/bin/id "$@" +fi +SH + +cat >"$FAKE_BIN/mountpoint" <<'SH' +#!/usr/bin/env bash +exit 0 +SH + +cat >"$FAKE_BIN/docker" <<'SH' +#!/usr/bin/env bash +set -Eeuo pipefail + +if [[ "${1:-}" == "inspect" ]]; then + printf 'healthy\n' + exit 0 +fi + +arguments=" $* " +if [[ "$arguments" == *" ps --quiet watermaps "* ]]; then + printf 'watermaps-test-container\n' + exit 0 +fi + +if [[ "$arguments" == *" run "*" route-data "* || + "$arguments" == *" run "*" route-data" ]]; then + route_file="$WATERMAPS_DATA_DIR/local/.germany-netherlands-fairways.candidate.json" + marker_file="$WATERMAPS_DATA_DIR/local/.germany-netherlands-fairways.candidate.ready" + cat >"$route_file" </dev/null || true)" + if [[ -n "${FAKE_FAIL_REVISION:-}" && "$active_revision" == "$FAKE_FAIL_REVISION" ]]; then + exit 1 + fi + exit 0 +fi + +exit 0 +SH + +chmod +x "$FAKE_BIN/id" "$FAKE_BIN/mountpoint" "$FAKE_BIN/docker" + +export PATH="$FAKE_BIN:$PATH" +export WATERMAPS_ENV_FILE="$ENV_FILE" +export WM_TEST_ROOT="$ROOT_DIR" + +# A first successful activation creates an active index and a validated marker. +FAKE_ROUTE_REVISION=1 "$ROOT_DIR/deploy/scripts/update-route-data.sh" +active_file="$DATA_DIR/local/germany-netherlands-fairways.json" +active_marker="$DATA_DIR/local/.germany-netherlands-fairways.ready" +[[ "$(jq --raw-output '.revision' "$active_file")" == "1" ]] + +export WATERMAPS_DATA_DIR="$DATA_DIR" +export WATERMAPS_RUNTIME_DIR="$RUNTIME_DIR" +# shellcheck source=../common.sh +source "$ROOT_DIR/deploy/scripts/common.sh" +wm_route_data_files_ready "$active_file" "$active_marker" + +# A candidate that fails its route smoke test must restore and retest revision 1. +set +e +FAKE_ROUTE_REVISION=2 FAKE_FAIL_REVISION=2 \ + "$ROOT_DIR/deploy/scripts/update-route-data.sh" +update_status=$? +set -e +[[ "$update_status" -ne 0 ]] +[[ "$(jq --raw-output '.revision' "$active_file")" == "1" ]] +wm_route_data_files_ready "$active_file" "$active_marker" +[[ ! -e "$RUNTIME_DIR/locks/route-update.transaction" ]] + +# A hard-crash marker restores the durable previous index before any new build. +backup_file="$DATA_DIR/local/.germany-netherlands-fairways.previous.json" +backup_marker="$DATA_DIR/local/.germany-netherlands-fairways.previous.ready" +cp "$active_file" "$backup_file" +wm_write_route_marker "$backup_file" "$backup_marker" +jq '.revision = 9' "$active_file" >"$active_file.tmp" +mv "$active_file.tmp" "$active_file" +chmod 0644 "$active_file" +wm_write_route_marker "$active_file" "$active_marker" +printf 'had_previous=true\n' >"$RUNTIME_DIR/locks/route-update.transaction" + +FAKE_ROUTE_REVISION=1 "$ROOT_DIR/deploy/scripts/update-route-data.sh" +[[ "$(jq --raw-output '.revision' "$active_file")" == "1" ]] +wm_route_data_files_ready "$active_file" "$active_marker" +[[ ! -e "$RUNTIME_DIR/locks/route-update.transaction" ]] + +# Without a previous valid index, a failed first candidate must not stay ready. +rm -f "$active_file" "$active_marker" \ + "$DATA_DIR/local/.germany-netherlands-fairways.previous.json" \ + "$DATA_DIR/local/.germany-netherlands-fairways.previous.ready" +set +e +FAKE_ROUTE_REVISION=3 FAKE_FAIL_REVISION=3 \ + "$ROOT_DIR/deploy/scripts/update-route-data.sh" +first_update_status=$? +set -e +[[ "$first_update_status" -ne 0 ]] +[[ ! -e "$active_file" ]] +[[ ! -e "$active_marker" ]] +[[ ! -e "$RUNTIME_DIR/locks/route-update.transaction" ]] + +printf 'Staging-, Rollback- und First-Update-Prüfungen: OK\n' diff --git a/deploy/scripts/update-route-data.sh b/deploy/scripts/update-route-data.sh new file mode 100755 index 0000000..cee6afd --- /dev/null +++ b/deploy/scripts/update-route-data.sh @@ -0,0 +1,207 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +WM_DEPLOY_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=common.sh +source "$WM_DEPLOY_DIR/scripts/common.sh" +wm_load_env +wm_assert_data_mount + +if [[ "$(id -u)" -ne 0 ]]; then + wm_die "Dieses Skript muss als root ausgeführt werden." +fi + +force_rebuild=false +if [[ "${1:-}" == "--force" ]]; then + force_rebuild=true + shift +fi +[[ "$#" -eq 0 ]] || wm_die "Unbekannte Argumente für update-route-data.sh." + +install -d -m 0755 "$WATERMAPS_RUNTIME_DIR/locks" "$WATERMAPS_DATA_DIR/local" +if [[ "${WATERMAPS_ROUTE_LOCK_HELD:-false}" != "true" ]]; then + if ! wm_acquire_route_lock; then + exit 0 + fi +fi + +route_file="$(wm_route_file)" +route_marker="$(wm_route_marker)" +route_dir="$(dirname "$route_file")" +candidate_file="$route_dir/.germany-netherlands-fairways.candidate.json" +candidate_marker="$route_dir/.germany-netherlands-fairways.candidate.ready" +backup_file="$route_dir/.germany-netherlands-fairways.previous.json" +backup_marker="$route_dir/.germany-netherlands-fairways.previous.ready" +transaction_file="$WATERMAPS_RUNTIME_DIR/locks/route-update.transaction" +container_candidate="/workspace/data/local/$(basename "$candidate_file")" +container_candidate_marker="/workspace/data/local/$(basename "$candidate_marker")" + +transaction_active=false +rollback_restored_previous=false + +wm_copy_route_atomically() { + local source_file="$1" + local destination_file="$2" + local temporary_file + + temporary_file="$(mktemp "$(dirname "$destination_file")/.route-copy.XXXXXX")" + if ! cp --reflink=auto "$source_file" "$temporary_file"; then + rm -f "$temporary_file" + return 1 + fi + chmod 0644 "$temporary_file" + mv -f "$temporary_file" "$destination_file" +} + +wm_watermaps_is_running() { + [[ -n "$(wm_compose ps --quiet watermaps)" ]] +} + +wm_restore_previous_route() { + rollback_restored_previous=false + if [[ -s "$backup_file" && -s "$backup_marker" ]] && + wm_route_data_files_ready "$backup_file" "$backup_marker"; then + if ! wm_copy_route_atomically "$backup_file" "$route_file" || + ! wm_write_route_marker "$route_file" "$route_marker" || + ! wm_route_data_ready; then + rm -f "$route_file" "$route_marker" + return 1 + fi + rollback_restored_previous=true + return 0 + fi + + rm -f "$route_file" "$route_marker" + return 1 +} + +wm_restart_after_restore() { + if ! wm_watermaps_is_running; then + wm_log "Watermaps läuft nicht; wiederhergestellte Routingdaten können noch nicht getestet werden." + return 1 + fi + + wm_compose restart watermaps + wm_wait_for_health watermaps 240 || return 1 + if [[ "$rollback_restored_previous" == "true" ]]; then + wm_smoke_test_route + fi +} + +wm_rollback_transaction() { + local restore_status=0 + + if ! wm_restore_previous_route; then + restore_status=1 + fi + rm -f "$transaction_file" + transaction_active=false + + if ! wm_restart_after_restore; then + return 1 + fi + return "$restore_status" +} + +wm_cleanup_on_exit() { + local status=$? + trap - EXIT HUP INT TERM + set +e + + if [[ "$transaction_active" == "true" ]]; then + wm_log "Unvollständige Routingaktivierung wird zurückgerollt." + wm_rollback_transaction + fi + rm -f "$candidate_file" "$candidate_marker" + exit "$status" +} + +trap wm_cleanup_on_exit EXIT +trap 'exit 130' HUP INT TERM + +# A hard interruption cannot run shell traps. The durable transaction marker +# makes the next invocation restore the last validated index before proceeding. +if [[ -e "$transaction_file" ]]; then + wm_log "Unterbrochene Routingaktualisierung erkannt; letzter gültiger Stand wird wiederhergestellt." + if wm_restore_previous_route; then + wm_restart_after_restore || + wm_die "Der wiederhergestellte Routingindex bestand den Routentest nicht." + else + wm_restart_after_restore || + wm_log "Es existierte noch kein vorheriger Routingindex; aktiver Marker wurde entfernt." + fi + rm -f "$transaction_file" +fi + +rm -f "$candidate_file" "$candidate_marker" +previous_ready=false +if wm_route_data_ready; then + previous_ready=true + if [[ "$force_rebuild" != "true" ]]; then + cp --reflink=auto "$route_file" "$candidate_file" + chmod 0644 "$candidate_file" + fi +fi + +wm_log "Deutschland-/Niederlande-Routingdaten werden in einer Stagingdatei gebaut." +if ! wm_compose --profile maintenance run --rm --build \ + --env "WATERMAPS_LOCAL_FAIRWAYS_PATH=$container_candidate" \ + --env "WATERMAPS_ROUTE_MARKER_PATH=$container_candidate_marker" \ + route-data; then + wm_die "Der Datenaufbau ist fehlgeschlagen; der aktive Index wurde nicht verändert." +fi + +if ! wm_route_data_files_ready "$candidate_file" "$candidate_marker"; then + wm_die "Der neu gebaute Routingindex oder sein Bereitschaftsmarker ist ungültig." +fi + +if [[ "$previous_ready" == "true" ]] && cmp --silent "$candidate_file" "$route_file"; then + wm_log "Geofabrik-Snapshots und Routingindex sind unverändert." + rm -f "$candidate_file" "$candidate_marker" + exit 0 +fi + +if [[ "$previous_ready" == "true" ]]; then + wm_copy_route_atomically "$route_file" "$backup_file" + wm_write_route_marker "$backup_file" "$backup_marker" + wm_route_data_files_ready "$backup_file" "$backup_marker" || + wm_die "Der bisherige Routingindex konnte nicht sicher gesichert werden." +else + rm -f "$backup_file" "$backup_marker" +fi + +temporary_transaction="$(mktemp "$WATERMAPS_RUNTIME_DIR/locks/.route-update-transaction.XXXXXX")" +{ + printf 'started_at=%s\n' "$(date --utc +%Y-%m-%dT%H:%M:%SZ)" + printf 'had_previous=%s\n' "$previous_ready" +} >"$temporary_transaction" +chmod 0644 "$temporary_transaction" +mv -f "$temporary_transaction" "$transaction_file" +transaction_active=true + +mv -f "$candidate_file" "$route_file" +rm -f "$candidate_marker" +wm_write_route_marker "$route_file" "$route_marker" +wm_route_data_ready || + wm_die "Der aktivierte Routingindex besitzt keinen gültigen Bereitschaftsmarker." + +wm_compose restart watermaps +if wm_wait_for_health watermaps 240 && wm_smoke_test_route; then + rm -f "$transaction_file" + transaction_active=false + rm -f "$backup_file" "$backup_marker" + wm_log "Routingdaten-Update wurde erfolgreich aktiviert." + exit 0 +fi + +wm_log "Der neue Routingindex bestand die Routentests nicht; Rollback wird ausgeführt." +if wm_rollback_transaction; then + rm -f "$backup_file" "$backup_marker" + wm_die "Neuer Routingindex verworfen; der vorherige, erneut getestete Index ist aktiv." +fi + +if [[ "$rollback_restored_previous" == "true" ]]; then + wm_die "Rollback-Dateien wurden wiederhergestellt, bestanden aber den Routentest nicht." +fi +wm_die "Neuer Routingindex verworfen; es existierte noch kein vorheriger gültiger Index." diff --git a/deploy/scripts/upload-and-deploy.sh b/deploy/scripts/upload-and-deploy.sh new file mode 100755 index 0000000..2daf29c --- /dev/null +++ b/deploy/scripts/upload-and-deploy.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +WM_DEPLOY_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +WM_ROOT_DIR="$(cd "$WM_DEPLOY_DIR/.." && pwd)" +# shellcheck source=remote-common.sh +source "$WM_DEPLOY_DIR/scripts/remote-common.sh" + +server_ipv4="" +identity_file="" +run_go_live=false + +usage() { + cat <<'USAGE' +Verwendung: + ./deploy/scripts/upload-and-deploy.sh [Optionen] + +Optionen: + --host IPV4 Server-IP statt `tofu output -raw server_ipv4` + --identity DATEI Privater SSH-Key (alternativ WATERMAPS_SSH_KEY) + --user BENUTZER SSH-Benutzer; Standard: deploy + --go-live Nach dem Deployment sofort den SSL-Livegang versuchen + -h, --help Hilfe anzeigen + +Die Datei deploy/.env.production muss lokal vorhanden sein. Sie wird separat +mit Dateimodus 0600 übertragen. Terraform-State, .terraform, data, +node_modules, lokale Env-Dateien und Git-Metadaten werden ausgeschlossen. +USAGE +} + +while [[ "$#" -gt 0 ]]; do + case "$1" in + --host) + [[ "$#" -ge 2 ]] || wm_local_die "Wert für --host fehlt." + server_ipv4="$2" + shift 2 + ;; + --identity) + [[ "$#" -ge 2 ]] || wm_local_die "Wert für --identity fehlt." + identity_file="$2" + shift 2 + ;; + --user) + [[ "$#" -ge 2 ]] || wm_local_die "Wert für --user fehlt." + WATERMAPS_SSH_USER="$2" + export WATERMAPS_SSH_USER + shift 2 + ;; + --go-live) + run_go_live=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + wm_local_die "Unbekannte Option: $1" + ;; + esac +done + +command -v rsync >/dev/null 2>&1 || wm_local_die "rsync ist lokal nicht installiert." +local_env="$WM_DEPLOY_DIR/.env.production" +[[ -f "$local_env" ]] || + wm_local_die "Bitte zuerst deploy/.env.production aus der Vorlage erstellen und konfigurieren." + +wm_local_resolve_ssh "$server_ipv4" "$identity_file" +wm_local_wait_for_ssh + +remote_prefix="" +rsync_path="rsync" +if [[ -n "$WM_REMOTE_SUDO" ]]; then + remote_prefix="$WM_REMOTE_SUDO " + rsync_path="$WM_REMOTE_SUDO rsync" +fi + +printf '[watermaps] Warte auf Cloud-init und das persistente Hetzner-Volume.\n' +ssh "${WM_SSH_OPTIONS[@]}" "$WM_SSH_TARGET" \ + "cloud-init status --wait && ${remote_prefix}systemctl start watermaps-volume-setup.service && mountpoint --quiet /srv/watermaps-data" + +printf '[watermaps] Übertrage Projekt nach %s:/opt/watermaps\n' "$WM_SSH_TARGET" +ssh "${WM_SSH_OPTIONS[@]}" "$WM_SSH_TARGET" \ + "${remote_prefix}install -d -m 0755 /opt/watermaps /opt/watermaps/deploy" + +rsync \ + --archive \ + --compress \ + --delete-delay \ + --human-readable \ + --rsync-path="$rsync_path" \ + --exclude='.git/' \ + --exclude='.terraform/' \ + --exclude='.terraform.tfstate.lock.info' \ + --exclude='*.tfvars' \ + --exclude='*.tfvars.json' \ + --exclude='*.tfplan' \ + --exclude='*.tfstate*' \ + --exclude='crash.log' \ + --exclude='crash.*.log' \ + --exclude='/data/' \ + --exclude='/.tools/' \ + --exclude='node_modules/' \ + --exclude='**/.env' \ + --exclude='**/.env.*' \ + --exclude='/deploy/.env.production' \ + -e "ssh ${WM_SSH_OPTIONS[*]@Q}" \ + "$WM_ROOT_DIR/" \ + "$WM_SSH_TARGET:/opt/watermaps/" + +rsync \ + --archive \ + --chmod=F600 \ + --rsync-path="$rsync_path" \ + -e "ssh ${WM_SSH_OPTIONS[*]@Q}" \ + "$local_env" \ + "$WM_SSH_TARGET:/opt/watermaps/deploy/.env.production" + +remote_command="${remote_prefix}chmod +x /opt/watermaps/deploy/scripts/*.sh" +remote_command+=" && ${remote_prefix}/opt/watermaps/deploy/scripts/bootstrap-server.sh" +remote_command+=" && ${remote_prefix}/opt/watermaps/deploy/scripts/deploy.sh" +ssh "${WM_SSH_OPTIONS[@]}" "$WM_SSH_TARGET" "$remote_command" + +if [[ "$run_go_live" == "true" ]]; then + ssh "${WM_SSH_OPTIONS[@]}" "$WM_SSH_TARGET" \ + "${remote_prefix}/opt/watermaps/deploy/scripts/go-live.sh '$WM_SERVER_IPV4'" +else + printf '[watermaps] Deployment abgeschlossen. Nach dem DNS-Eintrag:\n' + printf ' ./deploy/scripts/remote-go-live.sh --host %s\n' "$WM_SERVER_IPV4" +fi diff --git a/deploy/systemd/watermaps-certbot-renew.service b/deploy/systemd/watermaps-certbot-renew.service new file mode 100644 index 0000000..da5a2a0 --- /dev/null +++ b/deploy/systemd/watermaps-certbot-renew.service @@ -0,0 +1,11 @@ +[Unit] +Description=Watermaps Let's-Encrypt-Zertifikat erneuern +Wants=network-online.target +After=network-online.target docker.service watermaps-volume-setup.service +Requires=docker.service +RequiresMountsFor=/srv/watermaps-data + +[Service] +Type=oneshot +WorkingDirectory=/opt/watermaps +ExecStart=/opt/watermaps/deploy/scripts/renew-certificate.sh diff --git a/deploy/systemd/watermaps-certbot-renew.timer b/deploy/systemd/watermaps-certbot-renew.timer new file mode 100644 index 0000000..d182135 --- /dev/null +++ b/deploy/systemd/watermaps-certbot-renew.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Watermaps-Zertifikat zweimal täglich prüfen + +[Timer] +OnCalendar=*-*-* 03,15:17:00 +RandomizedDelaySec=45m +Persistent=true +Unit=watermaps-certbot-renew.service + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/watermaps-route-update.service b/deploy/systemd/watermaps-route-update.service new file mode 100644 index 0000000..2407b43 --- /dev/null +++ b/deploy/systemd/watermaps-route-update.service @@ -0,0 +1,14 @@ +[Unit] +Description=Watermaps Deutschland-/Niederlande-Routingdaten aktualisieren +Wants=network-online.target +After=network-online.target docker.service watermaps-volume-setup.service +Requires=docker.service +RequiresMountsFor=/srv/watermaps-data + +[Service] +Type=oneshot +WorkingDirectory=/opt/watermaps +ExecStart=/opt/watermaps/deploy/scripts/update-route-data.sh +Nice=10 +IOSchedulingClass=best-effort +IOSchedulingPriority=7 diff --git a/deploy/systemd/watermaps-route-update.timer b/deploy/systemd/watermaps-route-update.timer new file mode 100644 index 0000000..7580435 --- /dev/null +++ b/deploy/systemd/watermaps-route-update.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Täglich Watermaps-Routingdaten aktualisieren + +[Timer] +OnCalendar=*-*-* 06:15:00 +RandomizedDelaySec=30m +Persistent=true +Unit=watermaps-route-update.service + +[Install] +WantedBy=timers.target diff --git a/docker-compose.yml b/docker-compose.yml index ebaf83e..905446c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,44 @@ -version: "2.4" - services: + watermaps: + build: + context: . + dockerfile: Dockerfile + image: watermaps:local + ports: + - "${WATERMAPS_HTTP_PORT:-5173}:5174" + environment: + NODE_ENV: production + HOST: 0.0.0.0 + PORT: 5174 + DATABASE_URL: "" + REDIS_URL: "" + WATERMAPS_WEB_DIST_PATH: /app/apps/web/dist + WATERMAPS_LOCAL_FAIRWAYS_PATH: /data/germany-netherlands-fairways.json + WATERMAPS_LIVE_FAIRWAYS: "false" + WATERMAPS_DEMO_DATA: "true" + volumes: + - type: bind + source: ./data/local + target: /data + read_only: true + bind: + create_host_path: false + restart: unless-stopped + healthcheck: + test: + [ + "CMD", + "node", + "-e", + "fetch('http://127.0.0.1:5174/health').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))" + ] + interval: 30s + timeout: 5s + start_period: 10s + retries: 3 + postgres: + profiles: ["postgis"] image: postgis/postgis:16-3.4 environment: POSTGRES_DB: seacompass @@ -19,6 +56,7 @@ services: retries: 5 redis: + profiles: ["postgis"] image: redis:7-alpine ports: - "6379:6379" @@ -29,6 +67,7 @@ services: retries: 5 martin: + profiles: ["postgis"] image: ghcr.io/maplibre/martin:latest command: ["--config", "/config/martin.yaml"] ports: diff --git a/infra/opentofu/.gitignore b/infra/opentofu/.gitignore new file mode 100644 index 0000000..f6a92c1 --- /dev/null +++ b/infra/opentofu/.gitignore @@ -0,0 +1,18 @@ +.terraform/ +.terraform.tfstate.lock.info +*.tfstate +*.tfstate.* +*.tfplan +crash.log +crash.*.log + +# Lokale Konfiguration mit Zugangsdaten +terraform.tfvars +*.auto.tfvars +!terraform.tfvars.example + +# Lokale Overrides +override.tf +override.tf.json +*_override.tf +*_override.tf.json diff --git a/infra/opentofu/.terraform.lock.hcl b/infra/opentofu/.terraform.lock.hcl new file mode 100644 index 0000000..deb6a48 --- /dev/null +++ b/infra/opentofu/.terraform.lock.hcl @@ -0,0 +1,35 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hetznercloud/hcloud" { + version = "1.66.1" + constraints = "1.66.1" + hashes = [ + "h1:2xnL6r7iH/7sheeBlcSC8KpT9kUSYiboo0Ujc7B0RLw=", + "h1:4umCzzfMlXTswvnn46dX0tPNl267O72eIXRkO/+Nfng=", + "h1:7CSl5SdPjP2VK96O/4rz4zem2WSljipRr2r2DQWDbRw=", + "h1:JO2NGqSohoL+EfmcuUC6LdvoZh8Qd4fl0+Fqfyz+3Oc=", + "h1:QHgbvrPciaSf3B9q4tdfPq4yK5sJpA0vq7Z5Mjfj/sg=", + "h1:R+S77AiRWQQnVROR+HNqMXnhX8fDxZVBG1PwlpxGyss=", + "h1:WU+1RcBhDQNRY5CcurLApBjj8o2SOXKhJCjObIX4Ic0=", + "h1:WztH73MMX1CpUuglqMmnugXLS20nMHsntteg1hcM2xQ=", + "h1:Z/ipcn9gDF25Sjbp0v80Alt/6v4XwiBBmd9V6mYo0I0=", + "h1:bKv1qR7TB/x6a1u1OmacaGgFXmkAukokAPizicGvHAo=", + "h1:cc0E0i8geVFX5z8Kt5qrRew3eyB1OaWxTBzBbkoU7tA=", + "h1:lTWPM6oRkJ7988uDQ2bR8SUeezk7ux/rwg3WIuN2oRU=", + "h1:rxfnUOt0pPl2XXqIzMaEJimpWjBquLOCHv5Z/4/AD6g=", + "zh:113070176eb4fb26a3758b3d1031bb904e34a74f7c4f99f90df2301ca4468a51", + "zh:1bfc988bdcd7c9422e09c262c736ad265a205684f0402fa4a83e63c0b08e09ea", + "zh:37d92b4cf0f344295b0d780aabbc1408f02db31141cd4408276455a458071e54", + "zh:386bd1207b3ed284b513294ddad9b9f59047a693c3aa375605f858f9d9758b58", + "zh:43d26f0a4f5a64bf0ade1c5a278b40153d4ae8c77508933b9c8bb7f7860dae6d", + "zh:6a1fe681a1706be87f0d03b749f1dc69c838d663c6ab08a00ae8d87be2e4425e", + "zh:7032556ae2a74b2e1b7d68add9b96158e4f7202ebb8fbd75e0e8a1581719df7b", + "zh:893296927373b4afa7ec3639abb1ebab3c1b6cb9cfe427877cfa7bf69a33480d", + "zh:8d58b340e21428a59cc27dc4f5a7fcd2035ee1aa9372c3932f7962647532fe0d", + "zh:93509170347bf097ca38e288a6e921811bb769bb2dfac3c339eb2032ae8454c1", + "zh:9c00443cb5ae2401089a62e223d7d741eedb04f6a73c357b6a7443b6ae71b0be", + "zh:9c356be5ce8cb7b1d83d5cf2276823242f106dbd0c43fe992055f0fa11290f95", + "zh:f619116583c7e47751e0baefffca216c83f9f42edf99325121b85ee075db78cc", + ] +} diff --git a/infra/opentofu/README.md b/infra/opentofu/README.md new file mode 100644 index 0000000..af36b6c --- /dev/null +++ b/infra/opentofu/README.md @@ -0,0 +1,73 @@ +# Watermaps auf Hetzner Cloud + +Diese OpenTofu-Konfiguration erstellt einen einzelnen Ubuntu-24.04-Server mit +fester IPv4-Adresse, vorgeschalteter Hetzner-Firewall und einem persistenten +ext4-Volume für die Routingdaten von Deutschland und den Niederlanden. +Kartenkacheln, DNS-Einträge und Anwendungsdeployment sind bewusst nicht Teil +dieses Infrastrukturmoduls. + +## Konfiguration + +Die lokale Datei `terraform.tfvars` ist bereits angelegt und wird durch die +lokale `.gitignore` ausgeschlossen. Vor dem ersten Plan müssen dort mindestens +folgende Werte ersetzt werden: + +- `hcloud_token`: Read/Write-API-Token des richtigen Hetzner-Cloud-Projekts +- `ssh_public_key`: vollständiger Inhalt des öffentlichen SSH-Schlüssels +- `admin_cidrs`: öffentliche Administrator-IP mit `/32` beziehungsweise ein + bewusst gewähltes Netz + +`terraform.tfvars.example` bleibt als geheimnisfreie Vorlage versioniert. +Da `terraform.tfvars` den Token im Klartext enthält, sollte sie nur für den +lokalen Benutzer lesbar sein: + +```sh +chmod 600 terraform.tfvars +``` + +Auch `terraform.tfstate` bleibt lokal und wird nicht nach Git oder auf den +Anwendungsserver übertragen. Nach dem ersten Apply sollte die State-Datei +verschlüsselt gesichert werden, weil OpenTofu sie für spätere Änderungen an +denselben Ressourcen benötigt. + +## Infrastruktur erzeugen + +```sh +cd infra/opentofu +tofu init +tofu fmt -check +tofu validate +tofu plan -out=watermaps.tfplan +tofu apply watermaps.tfplan +``` + +Die feste IPv4-Adresse und den erforderlichen manuellen DNS-Eintrag zeigt +OpenTofu anschließend an: + +```sh +tofu output server_ipv4 +tofu output dns_a_record +tofu output ssh_command +``` + +Der A-Record `watermaps.incoso.eu` kann direkt nach dem Apply auf +`server_ipv4` gesetzt werden. Das eigentliche SSL-Livegehen erfolgt erst durch +das separate Deployment-Skript, nachdem DNS propagiert ist und die Routingdaten +bereitstehen. + +## Auf dem Server + +Cloud-init installiert Docker einschließlich Compose v2, `curl`, `jq` und +`rsync`. Der Benutzer `deploy` erhält Zugriff per SSH und auf Docker. Das +persistente Volume wird nach dem Anhängen unter `/srv/watermaps-data` gemountet +und enthält die vom Produktions-Stack verwendeten Verzeichnisse `geofabrik`, +`local` und `certbot`. Ein systemd-Drop-in lässt Docker bei jedem Start auf den +erfolgreichen Volume-Mount warten, bevor bestehende Container neu gestartet +werden. + +Der Status des ersten Starts lässt sich so prüfen: + +```sh +ssh deploy@"$(tofu output -raw server_ipv4)" \ + "cloud-init status --wait && systemctl status watermaps-volume-setup --no-pager" +``` diff --git a/infra/opentofu/cloud-init.yaml.tftpl b/infra/opentofu/cloud-init.yaml.tftpl new file mode 100644 index 0000000..8a02606 --- /dev/null +++ b/infra/opentofu/cloud-init.yaml.tftpl @@ -0,0 +1,138 @@ +#cloud-config + +package_update: true +package_upgrade: false + +groups: + - docker + +users: + - default + - name: ${deploy_user} + gecos: Watermaps deployment user + shell: /bin/bash + lock_passwd: true + sudo: "ALL=(ALL) NOPASSWD:ALL" + groups: + - docker + - sudo + ssh_authorized_keys: + - ${ssh_public_key} + +ssh_pwauth: false + +packages: + - ca-certificates + - curl + - docker.io + - docker-compose-v2 + - jq + - rsync + +write_files: + - path: /etc/docker/daemon.json + owner: root:root + permissions: "0644" + content: | + { + "log-driver": "json-file", + "log-opts": { + "max-size": "10m", + "max-file": "3" + } + } + + - path: /usr/local/sbin/watermaps-prepare-volume + owner: root:root + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + mount_path="/srv/watermaps-data" + install -d -m 0755 "$mount_path" + + if ! mountpoint -q "$mount_path"; then + device="" + for attempt in $(seq 1 120); do + device="$(lsblk -dnpo NAME,MODEL | awk '$2 == "Volume" { print $1; exit }')" + if [ -n "$device" ] && [ -b "$device" ]; then + break + fi + device="" + sleep 5 + done + + if [ -z "$device" ]; then + echo "Kein angehängtes Hetzner Cloud Volume gefunden." >&2 + exit 1 + fi + + filesystem="$(blkid -o value -s TYPE "$device" || true)" + if [ -z "$filesystem" ]; then + mkfs.ext4 -F "$device" + filesystem="ext4" + fi + + if [ "$filesystem" != "ext4" ]; then + echo "Unerwartetes Dateisystem auf $device: $filesystem" >&2 + exit 1 + fi + + uuid="$(blkid -o value -s UUID "$device")" + if ! grep -q "^UUID=$uuid " /etc/fstab; then + printf '%s\n' \ + "UUID=$uuid $mount_path ext4 defaults,nofail,x-systemd.device-timeout=30 0 2" \ + >> /etc/fstab + fi + + mount "$mount_path" + fi + + install -d -m 0755 -o ${deploy_user} -g ${deploy_user} \ + "$mount_path/geofabrik" \ + "$mount_path/local" \ + "$mount_path/certbot" \ + "$mount_path/certbot/www" \ + "$mount_path/certbot/letsencrypt" + chown ${deploy_user}:${deploy_user} "$mount_path" + + - path: /etc/systemd/system/watermaps-volume-setup.service + owner: root:root + permissions: "0644" + content: | + [Unit] + Description=Prepare persistent Watermaps routing data volume + Wants=network-online.target + After=network-online.target local-fs.target + + [Service] + Type=oneshot + ExecStart=/usr/local/sbin/watermaps-prepare-volume + RemainAfterExit=yes + Restart=on-failure + RestartSec=15 + TimeoutStartSec=0 + + [Install] + WantedBy=multi-user.target + + - path: /etc/systemd/system/docker.service.d/watermaps-volume.conf + owner: root:root + permissions: "0644" + content: | + [Unit] + Requires=watermaps-volume-setup.service + After=watermaps-volume-setup.service + +runcmd: + - [install, -d, -m, "0755", -o, ${deploy_user}, -g, ${deploy_user}, /opt/watermaps] + - [install, -d, -m, "0755", -o, ${deploy_user}, -g, ${deploy_user}, /opt/watermaps/releases] + - [install, -d, -m, "0755", -o, ${deploy_user}, -g, ${deploy_user}, /opt/watermaps/shared] + - [install, -d, -m, "0755", -o, ${deploy_user}, -g, ${deploy_user}, /var/www/certbot] + - [systemctl, daemon-reload] + - [systemctl, enable, --now, watermaps-volume-setup.service] + - [systemctl, enable, docker.service] + - [systemctl, restart, docker.service] + +final_message: "Watermaps host bootstrap completed after $UPTIME seconds." diff --git a/infra/opentofu/main.tf b/infra/opentofu/main.tf new file mode 100644 index 0000000..227b0c5 --- /dev/null +++ b/infra/opentofu/main.tf @@ -0,0 +1,108 @@ +locals { + common_labels = { + application = "watermaps" + managed_by = "opentofu" + } +} + +resource "hcloud_ssh_key" "deploy" { + name = "${var.server_name}-deploy" + public_key = trimspace(var.ssh_public_key) + labels = local.common_labels +} + +resource "hcloud_primary_ip" "main" { + name = "${var.server_name}-ipv4" + location = var.location + type = "ipv4" + auto_delete = false + delete_protection = var.enable_resource_protection + labels = local.common_labels +} + +resource "hcloud_firewall" "main" { + name = "${var.server_name}-firewall" + labels = local.common_labels + + rule { + description = "SSH from configured administrator networks" + direction = "in" + protocol = "tcp" + port = "22" + source_ips = var.admin_cidrs + } + + rule { + description = "HTTP for ACME challenge and HTTPS redirect" + direction = "in" + protocol = "tcp" + port = "80" + source_ips = [ + "0.0.0.0/0", + "::/0", + ] + } + + rule { + description = "HTTPS" + direction = "in" + protocol = "tcp" + port = "443" + source_ips = [ + "0.0.0.0/0", + "::/0", + ] + } + + rule { + description = "ICMP diagnostics" + direction = "in" + protocol = "icmp" + source_ips = [ + "0.0.0.0/0", + "::/0", + ] + } +} + +resource "hcloud_server" "main" { + name = var.server_name + image = "ubuntu-24.04" + server_type = var.server_type + location = var.location + + ssh_keys = [hcloud_ssh_key.deploy.id] + firewall_ids = [hcloud_firewall.main.id] + + user_data = templatefile("${path.module}/cloud-init.yaml.tftpl", { + deploy_user = var.deploy_user + ssh_public_key = jsonencode(trimspace(var.ssh_public_key)) + }) + + public_net { + ipv4_enabled = true + ipv4 = hcloud_primary_ip.main.id + ipv6_enabled = false + } + + delete_protection = var.enable_resource_protection + rebuild_protection = var.enable_resource_protection + shutdown_before_deletion = true + + labels = local.common_labels +} + +resource "hcloud_volume" "routing_data" { + name = "${var.server_name}-routing-data" + location = var.location + size = var.routing_volume_size_gb + format = "ext4" + delete_protection = var.enable_resource_protection + labels = local.common_labels +} + +resource "hcloud_volume_attachment" "routing_data" { + volume_id = hcloud_volume.routing_data.id + server_id = hcloud_server.main.id + automount = false +} diff --git a/infra/opentofu/outputs.tf b/infra/opentofu/outputs.tf new file mode 100644 index 0000000..966b2a9 --- /dev/null +++ b/infra/opentofu/outputs.tf @@ -0,0 +1,33 @@ +output "server_ipv4" { + description = "Persistente öffentliche IPv4-Adresse des Watermaps-Servers." + value = hcloud_primary_ip.main.ip_address +} + +output "server_id" { + description = "Hetzner-ID des Servers." + value = hcloud_server.main.id +} + +output "ssh_command" { + description = "SSH-Befehl für den vorbereiteten Deploy-Benutzer." + value = "ssh ${var.deploy_user}@${hcloud_primary_ip.main.ip_address}" +} + +output "dns_a_record" { + description = "A-Record, der nach dem Apply manuell beim DNS-Provider angelegt werden soll." + value = { + type = "A" + name = var.dns_name + value = hcloud_primary_ip.main.ip_address + } +} + +output "routing_data_mount_path" { + description = "Persistenter Pfad für heruntergeladene PBFs und erzeugte Fahrrouten." + value = "/srv/watermaps-data" +} + +output "routing_volume_id" { + description = "Hetzner-ID des persistenten Routingdaten-Volumes." + value = hcloud_volume.routing_data.id +} diff --git a/infra/opentofu/provider.tf b/infra/opentofu/provider.tf new file mode 100644 index 0000000..706ed68 --- /dev/null +++ b/infra/opentofu/provider.tf @@ -0,0 +1,3 @@ +provider "hcloud" { + token = var.hcloud_token +} diff --git a/infra/opentofu/terraform.tfvars.example b/infra/opentofu/terraform.tfvars.example new file mode 100644 index 0000000..e10b359 --- /dev/null +++ b/infra/opentofu/terraform.tfvars.example @@ -0,0 +1,23 @@ +# Diese Datei enthält ausschließlich Beispiele. Kopiere sie nach +# terraform.tfvars und ersetze alle REPLACE_*-Werte. + +hcloud_token = "REPLACE_WITH_HETZNER_CLOUD_READ_WRITE_TOKEN" + +# Inhalt einer öffentlichen .pub-Datei, niemals den privaten Schlüssel eintragen. +ssh_public_key = "ssh-ed25519 REPLACE_WITH_PUBLIC_KEY user@example" + +# Auf deine aktuelle öffentliche IP begrenzen. Mehrere Netze sind möglich. +admin_cidrs = [ + "203.0.113.10/32", +] + +server_name = "watermaps" +dns_name = "watermaps.incoso.eu" +location = "fsn1" +server_type = "cx33" + +deploy_user = "deploy" +routing_volume_size_gb = 40 + +# Schützt Ressourcen vor versehentlicher Löschung über Console/API. +enable_resource_protection = true diff --git a/infra/opentofu/variables.tf b/infra/opentofu/variables.tf new file mode 100644 index 0000000..4e101d8 --- /dev/null +++ b/infra/opentofu/variables.tf @@ -0,0 +1,122 @@ +variable "hcloud_token" { + description = "Read/Write API-Token des Hetzner-Cloud-Projekts." + type = string + sensitive = true + nullable = false + + validation { + condition = length(trimspace(var.hcloud_token)) == 64 && !startswith(var.hcloud_token, "REPLACE_") + error_message = "hcloud_token muss durch einen gültigen, 64 Zeichen langen Hetzner-Cloud-API-Token ersetzt werden." + } +} + +variable "server_name" { + description = "Name des Hetzner-Servers und Präfix der zugehörigen Ressourcen." + type = string + default = "watermaps" + + validation { + condition = can(regex("^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$", var.server_name)) + error_message = "server_name muss ein gültiger Hostname aus Kleinbuchstaben, Ziffern und Bindestrichen sein." + } +} + +variable "dns_name" { + description = "Öffentlicher DNS-Name. Der DNS-Eintrag selbst wird bewusst nicht von OpenTofu verwaltet." + type = string + default = "watermaps.incoso.eu" + + validation { + condition = can(regex("^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+$", var.dns_name)) + error_message = "dns_name muss ein gültiger, vollständig qualifizierter DNS-Name sein." + } +} + +variable "location" { + description = "Hetzner-Cloud-Location für Server, Primary IP und Volume." + type = string + default = "fsn1" + + validation { + condition = length(trimspace(var.location)) > 0 + error_message = "location darf nicht leer sein." + } +} + +variable "server_type" { + description = "Hetzner-Cloud-Servertyp. cx33 ist für den initialen Routingdaten-Import konservativ gewählt." + type = string + default = "cx33" + + validation { + condition = length(trimspace(var.server_type)) > 0 + error_message = "server_type darf nicht leer sein." + } +} + +variable "ssh_public_key" { + description = "Öffentlicher SSH-Schlüssel für root und den Deploy-Benutzer." + type = string + nullable = false + + validation { + condition = ( + !strcontains(var.ssh_public_key, "REPLACE_") && + can(regex( + "^(ssh-(ed25519|rsa)|ecdsa-sha2-nistp(256|384|521)|sk-ssh-ed25519@openssh\\.com|sk-ecdsa-sha2-nistp256@openssh\\.com) ", + trimspace(var.ssh_public_key) + )) + ) + error_message = "ssh_public_key muss ein gültiger öffentlicher OpenSSH-Schlüssel sein." + } +} + +variable "admin_cidrs" { + description = "IPv4- oder IPv6-CIDRs, aus denen SSH auf Port 22 erlaubt ist, zum Beispiel [\"203.0.113.10/32\"]." + type = list(string) + nullable = false + + validation { + condition = ( + length(var.admin_cidrs) > 0 && + alltrue([for cidr in var.admin_cidrs : can(cidrhost(cidr, 0))]) && + !contains(var.admin_cidrs, "203.0.113.10/32") + ) + error_message = "admin_cidrs muss mindestens ein gültiges, echtes IPv4- oder IPv6-CIDR enthalten; die Beispieladresse muss ersetzt werden." + } +} + +variable "deploy_user" { + description = "Unprivilegierter Benutzer für Upload und Betrieb der Anwendung." + type = string + default = "deploy" + + validation { + condition = ( + var.deploy_user != "root" && + can(regex("^[a-z_][a-z0-9_-]{0,30}$", var.deploy_user)) + ) + error_message = "deploy_user muss ein gültiger Linux-Benutzername sein und darf nicht root heißen." + } +} + +variable "routing_volume_size_gb" { + description = "Größe des persistenten ext4-Volumes für Deutschland-/Niederlande-PBFs und den Routingindex." + type = number + default = 40 + + validation { + condition = ( + var.routing_volume_size_gb >= 10 && + var.routing_volume_size_gb <= 10000 && + floor(var.routing_volume_size_gb) == var.routing_volume_size_gb + ) + error_message = "routing_volume_size_gb muss eine ganze Zahl zwischen 10 und 10000 sein." + } +} + +variable "enable_resource_protection" { + description = "Aktiviert Hetzner-Löschschutz für Server, Primary IP und Volume." + type = bool + default = true +} diff --git a/infra/opentofu/versions.tf b/infra/opentofu/versions.tf new file mode 100644 index 0000000..3cf963a --- /dev/null +++ b/infra/opentofu/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.8.0, < 2.0.0" + + required_providers { + hcloud = { + source = "hetznercloud/hcloud" + version = "= 1.66.1" + } + } +} diff --git a/package-lock.json b/package-lock.json index 4e3f8d1..c905494 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "packages/*" ], "devDependencies": { - "concurrently": "^9.1.2", + "concurrently": "^9.2.4", "typescript": "^5.8.3", "vitest": "^3.2.4" }, @@ -25,8 +25,9 @@ "version": "0.1.0", "dependencies": { "@fastify/cors": "^11.0.1", + "@fastify/static": "^10.1.2", "@watermaps/shared": "0.1.0", - "fastify": "^5.4.0", + "fastify": "^5.10.0", "ioredis": "^5.6.1", "pg": "^8.16.3", "zod": "^3.25.76" @@ -2246,6 +2247,22 @@ "node": ">=18" } }, + "node_modules/@fastify/accept-negotiator": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@fastify/accept-negotiator/-/accept-negotiator-2.0.1.tgz", + "integrity": "sha512-/c/TW2bO/v9JeEgoD/g1G5GxGeCF1Hafdf79WPmUlgYiBXummY0oX3VVq4yFkKKVBKDNlaDUYoab7g38RpPqCQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" + }, "node_modules/@fastify/ajv-compiler": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/@fastify/ajv-compiler/-/ajv-compiler-4.0.5.tgz", @@ -2377,6 +2394,71 @@ "ipaddr.js": "^2.1.0" } }, + "node_modules/@fastify/send": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@fastify/send/-/send-4.1.0.tgz", + "integrity": "sha512-TMYeQLCBSy2TOFmV95hQWkiTYgC/SEx7vMdV+wnZVX4tt8VBLKzmH8vV9OzJehV0+XBfg+WxPMt5wp+JBUKsVw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT", + "dependencies": { + "@lukeed/ms": "^2.0.2", + "escape-html": "~1.0.3", + "fast-decode-uri-component": "^1.0.1", + "http-errors": "^2.0.0", + "mime": "^3" + } + }, + "node_modules/@fastify/static": { + "version": "10.1.2", + "resolved": "https://registry.npmjs.org/@fastify/static/-/static-10.1.2.tgz", + "integrity": "sha512-G/g18cG9tLutT/OVyN1AIsHIl9L1UwmJ+S3dkyhVpplIx0nEMicd7RGQ+uJLyhKKF4a3tTcQydccn3Mop1fX+Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT", + "dependencies": { + "@fastify/accept-negotiator": "^2.0.0", + "@fastify/error": "^4.0.0", + "@fastify/send": "^4.0.0", + "content-disposition": "^2.0.1", + "fastify-plugin": "^6.0.0", + "fastq": "^1.17.1", + "glob": "^13.0.0" + } + }, + "node_modules/@fastify/static/node_modules/glob": { + "version": "13.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "minimatch": "^10.2.2", + "minipass": "^7.1.3", + "path-scurry": "^2.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/@ioredis/commands": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.10.0.tgz", @@ -2454,6 +2536,15 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@lukeed/ms": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@lukeed/ms/-/ms-2.0.2.tgz", + "integrity": "sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/@mapbox/jsonlint-lines-primitives": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@mapbox/jsonlint-lines-primitives/-/jsonlint-lines-primitives-2.0.3.tgz", @@ -3731,7 +3822,6 @@ "version": "4.0.4", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "dev": true, "license": "MIT", "engines": { "node": "18 || 20 || >=22" @@ -3754,7 +3844,6 @@ "version": "5.0.7", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz", "integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==", - "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" @@ -4004,15 +4093,15 @@ } }, "node_modules/concurrently": { - "version": "9.2.3", - "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-9.2.3.tgz", - "integrity": "sha512-ihjs0E2SxvDgq/MK418hX6YycQgKhsqxpbZuZbHo0yKfqDWdymWMjWYIpCIzqDDLLKClHlXev8whW/8WXmJ0BA==", + "version": "9.2.4", + "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-9.2.4.tgz", + "integrity": "sha512-TZ0CEhyzvFjgtAvHTusDMgj7wNdihCh7LLLrzdUOXIhdlnL2JBBGA9eJxR24rtqgmdjh3OA3hrN1rCHj6HM8qA==", "dev": true, "license": "MIT", "dependencies": { "chalk": "4.1.2", "rxjs": "7.8.2", - "shell-quote": "1.8.4", + "shell-quote": "1.9.0", "supports-color": "8.1.1", "tree-kill": "1.2.2", "yargs": "17.7.2" @@ -4028,6 +4117,19 @@ "url": "https://github.com/open-cli-tools/concurrently?sponsor=1" } }, + "node_modules/content-disposition": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-2.0.1.tgz", + "integrity": "sha512-e+H0ZXHSWYrENhQzw1LPuP4oF5MzVKmDU6d3hxlvaPEYLLg62MxtQNPRx4SYSuYJSBUgnQIG4HIN2tEtNv7Dog==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/convert-source-map": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", @@ -4272,6 +4374,15 @@ "node": ">=0.10" } }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/dequal": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", @@ -4570,6 +4681,12 @@ "node": ">=6" } }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, "node_modules/estree-walker": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", @@ -4666,9 +4783,9 @@ } }, "node_modules/fast-uri": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.3.tgz", - "integrity": "sha512-i70LwGWUduXqzicKXWshooq+sWL1K3WUU5rKZNG/0i3a1OSoX3HqhH5WbWwTmqWfor4urUakGPiRQcleRZTwOg==", + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.4.tgz", + "integrity": "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==", "funding": [ { "type": "github", @@ -4810,9 +4927,9 @@ } }, "node_modules/find-my-way": { - "version": "9.6.0", - "resolved": "https://registry.npmjs.org/find-my-way/-/find-my-way-9.6.0.tgz", - "integrity": "sha512-Zf4Xve4RymLl7NgaavNebZ01joJ8MfVerOG43wy7SHLO+r+K0C6d/SE0BiR7AV5V1VOCFlOP7ecdo+I4qmiHrQ==", + "version": "9.7.0", + "resolved": "https://registry.npmjs.org/find-my-way/-/find-my-way-9.7.0.tgz", + "integrity": "sha512-f2JHn75x2JlwUwLenZypgczR7YWMb/uO9BvUXtus+JMgkbIkLADd38cI4EiV+OQqrGo1Zlq6V8wnqMJ8e62wUQ==", "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.3", @@ -5200,6 +5317,26 @@ "node": ">=18" } }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/http-proxy-agent": { "version": "7.0.2", "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", @@ -5258,6 +5395,12 @@ "node": ">=8" } }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, "node_modules/internal-slot": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.1.0.tgz", @@ -6069,6 +6212,18 @@ "node": ">= 0.4" } }, + "node_modules/mime": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", + "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/min-indent": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/min-indent/-/min-indent-1.0.1.tgz", @@ -6083,7 +6238,6 @@ "version": "10.2.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", - "dev": true, "license": "BlueOak-1.0.0", "dependencies": { "brace-expansion": "^5.0.5" @@ -6108,7 +6262,6 @@ "version": "7.1.3", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", - "dev": true, "license": "BlueOak-1.0.0", "engines": { "node": ">=16 || 14 >=14.17" @@ -6274,7 +6427,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", - "dev": true, "license": "BlueOak-1.0.0", "dependencies": { "lru-cache": "^11.0.0", @@ -6291,7 +6443,6 @@ "version": "11.5.2", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.2.tgz", "integrity": "sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g==", - "dev": true, "license": "BlueOak-1.0.0", "engines": { "node": "20 || >=22" @@ -7215,6 +7366,12 @@ "node": ">= 0.4" } }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -7239,9 +7396,9 @@ } }, "node_modules/shell-quote": { - "version": "1.8.4", - "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.4.tgz", - "integrity": "sha512-VsC6n6vz1ihYYyZZwX7YZSF5l5x36ca17OC+a69h94YqB7X6XLwf+5MOgynYir2SLFUbl8gIYvBo8K8RoNQ6bQ==", + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.9.0.tgz", + "integrity": "sha512-Iov+JwFv/2HcTpcwNMKd8+IWNb8tboQJNQTkAY/LLVK7gGH9jy+LGkVqPxfekHl+yMmiqXszdGWXgkfml7hjqA==", "dev": true, "license": "MIT", "engines": { @@ -7462,6 +7619,15 @@ "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", "license": "MIT" }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/std-env": { "version": "3.10.0", "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", @@ -7848,6 +8014,15 @@ "node": ">=20" } }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, "node_modules/tough-cookie": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.1.2.tgz", @@ -7923,7 +8098,6 @@ "os": [ "aix" ], - "peer": true, "engines": { "node": ">=18" } @@ -7941,7 +8115,6 @@ "os": [ "android" ], - "peer": true, "engines": { "node": ">=18" } @@ -7959,7 +8132,6 @@ "os": [ "android" ], - "peer": true, "engines": { "node": ">=18" } @@ -7977,7 +8149,6 @@ "os": [ "android" ], - "peer": true, "engines": { "node": ">=18" } @@ -7995,7 +8166,6 @@ "os": [ "darwin" ], - "peer": true, "engines": { "node": ">=18" } @@ -8013,7 +8183,6 @@ "os": [ "darwin" ], - "peer": true, "engines": { "node": ">=18" } @@ -8031,7 +8200,6 @@ "os": [ "freebsd" ], - "peer": true, "engines": { "node": ">=18" } @@ -8049,7 +8217,6 @@ "os": [ "freebsd" ], - "peer": true, "engines": { "node": ">=18" } @@ -8067,7 +8234,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": ">=18" } @@ -8085,7 +8251,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": ">=18" } @@ -8103,7 +8268,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": ">=18" } @@ -8121,7 +8285,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": ">=18" } @@ -8139,7 +8302,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": ">=18" } @@ -8157,7 +8319,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": ">=18" } @@ -8175,7 +8336,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": ">=18" } @@ -8193,7 +8353,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": ">=18" } @@ -8211,7 +8370,6 @@ "os": [ "linux" ], - "peer": true, "engines": { "node": ">=18" } @@ -8229,7 +8387,6 @@ "os": [ "netbsd" ], - "peer": true, "engines": { "node": ">=18" } @@ -8247,7 +8404,6 @@ "os": [ "netbsd" ], - "peer": true, "engines": { "node": ">=18" } @@ -8265,7 +8421,6 @@ "os": [ "openbsd" ], - "peer": true, "engines": { "node": ">=18" } @@ -8283,7 +8438,6 @@ "os": [ "openbsd" ], - "peer": true, "engines": { "node": ">=18" } @@ -8301,7 +8455,6 @@ "os": [ "openharmony" ], - "peer": true, "engines": { "node": ">=18" } @@ -8319,7 +8472,6 @@ "os": [ "sunos" ], - "peer": true, "engines": { "node": ">=18" } @@ -8337,7 +8489,6 @@ "os": [ "win32" ], - "peer": true, "engines": { "node": ">=18" } @@ -8355,7 +8506,6 @@ "os": [ "win32" ], - "peer": true, "engines": { "node": ">=18" } @@ -8373,7 +8523,6 @@ "os": [ "win32" ], - "peer": true, "engines": { "node": ">=18" } diff --git a/package.json b/package.json index 233708e..5c0c347 100644 --- a/package.json +++ b/package.json @@ -14,15 +14,21 @@ "build": "npm run build --workspace @watermaps/shared && npm run build --workspace @watermaps/api && npm run build --workspace @watermaps/web", "dev": "npm run build --workspace @watermaps/shared && concurrently -n api,web -c cyan,green \"npm run dev --workspace @watermaps/api\" \"npm run dev --workspace @watermaps/web\"", "dev:https": "npm run build --workspace @watermaps/shared && concurrently -n api,web -c cyan,green \"npm run dev --workspace @watermaps/api\" \"npm run dev:https --workspace @watermaps/web\"", + "docker:down": "docker compose down", + "docker:up": "docker compose up -d --build watermaps", "enrich:marine-search": "node scripts/enrich-marine-search.mjs", "enrich:marine-websites": "node scripts/enrich-marine-websites.mjs", + "setup:local-germany": "./scripts/setup-local-germany.sh", + "setup:local-routing": "./scripts/setup-local-routing.sh", "sync:euris-locks": "node scripts/sync-euris-locks.mjs", - "test": "npm run build --workspace @watermaps/shared && npm run test --workspace @watermaps/shared && npm run test --workspace @watermaps/api && npm run test --workspace @watermaps/web", + "test": "npm run build --workspace @watermaps/shared && npm run test --workspace @watermaps/shared && npm run test --workspace @watermaps/api && npm run test --workspace @watermaps/web && npm run test:local-routing && npm run test:deployment", + "test:deployment": "bash deploy/scripts/tests/route-data-helpers.test.sh && bash deploy/scripts/tests/update-route-data.test.sh", "test:e2e": "npm run test:e2e --workspace @watermaps/web", + "test:local-routing": "PYTHONPATH=.tools/python python3 -m unittest discover -s scripts/tests -p 'test_*.py'", "typecheck": "npm run build --workspace @watermaps/shared && npm run typecheck --workspace @watermaps/shared && npm run typecheck --workspace @watermaps/api && npm run typecheck --workspace @watermaps/web" }, "devDependencies": { - "concurrently": "^9.1.2", + "concurrently": "^9.2.4", "typescript": "^5.8.3", "vitest": "^3.2.4" } diff --git a/packages/shared/src/fairway-routing.ts b/packages/shared/src/fairway-routing.ts index 1ed2f52..769eb7b 100644 --- a/packages/shared/src/fairway-routing.ts +++ b/packages/shared/src/fairway-routing.ts @@ -36,6 +36,12 @@ type EdgeSnap = { distanceNm: number; }; +type EdgeSnapPair = { + componentId: string; + start: EdgeSnap; + destination: EdgeSnap; +}; + export type FairwayGraph = { id: string; name: string; @@ -71,6 +77,7 @@ const ALTERNATIVE_EDGE_PENALTY = 8; const MAX_ALTERNATIVE_DISTANCE_FACTOR = 1.75; const MIN_DIFFERENT_DISTANCE_NM = 0.25; const MIN_DIFFERENT_DISTANCE_RATIO = 0.05; +const MAX_EDGE_SNAPS_PER_COMPONENT = 4; const EMS_BORKUM_GRAPH: FairwayGraph = { id: "ems-borkum-seed", @@ -211,12 +218,26 @@ export function buildFairwayRoutes( return []; } + const requestedPoints = [request.start, ...(request.waypoints ?? []), request.destination]; + const componentByNode = weaklyConnectedComponents(routableGraph); + const legSnapPairs = requestedPoints.slice(0, -1).map((start, index) => + findCompatibleEdgeSnapPairs( + routableGraph, + start, + requestedPoints[index + 1]!, + componentByNode + ) + ); + if (legSnapPairs.some((pairs) => pairs.length === 0)) { + return []; + } + const accepted: FairwayRouteCandidate[] = []; const penaltyCounts = new Map(); const maxAttempts = Math.max(8, routeLimit * 6); for (let attempt = 0; attempt < maxAttempts && accepted.length < routeLimit; attempt += 1) { - const candidate = buildFairwayRouteCandidate(request, routableGraph, penaltyCounts); + const candidate = buildFairwayRouteCandidate(request, routableGraph, penaltyCounts, legSnapPairs); if (!candidate) { break; } @@ -247,7 +268,8 @@ export function buildFairwayRoutes( function buildFairwayRouteCandidate( request: RouteRequest, routableGraph: FairwayGraph, - penaltyCounts: ReadonlyMap + penaltyCounts: ReadonlyMap, + legSnapPairs: EdgeSnapPair[][] ): FairwayRouteCandidate | null { const requestedPoints = [request.start, ...(request.waypoints ?? []), request.destination]; const routeCoordinates: Coordinate[] = []; @@ -257,7 +279,14 @@ function buildFairwayRouteCandidate( for (let index = 0; index < requestedPoints.length - 1; index += 1) { const legStart = requestedPoints[index]!; const legDestination = requestedPoints[index + 1]!; - const leg = buildFairwayLeg(routableGraph, legStart, legDestination, penaltyCounts, adjacency); + const leg = buildFairwayLeg( + routableGraph, + legStart, + legDestination, + penaltyCounts, + adjacency, + legSnapPairs[index]! + ); if (!leg) { return null; @@ -390,104 +419,153 @@ function buildFairwayLeg( legStart: Coordinate, legDestination: Coordinate, penaltyCounts: ReadonlyMap, - adjacency: Adjacency + adjacency: Adjacency, + snapPairs: EdgeSnapPair[] ): FairwayLeg | null { - const startSnap = findNearestEdgeSnap(graph, legStart); - const destinationSnap = findNearestEdgeSnap(graph, legDestination); - - if ( - !startSnap || - !destinationSnap || - startSnap.distanceNm > graph.maxSnapDistanceNm || - destinationSnap.distanceNm > graph.maxSnapDistanceNm - ) { - return null; - } - const candidates: FairwayLeg[] = []; + const routedComponents = new Set(); - if (startSnap.edge.id === destinationSnap.edge.id && canTraverseBetweenSnaps(startSnap, destinationSnap)) { - const directOnEdge = edgePathBetweenSnaps(startSnap, destinationSnap); - const coordinates: Coordinate[] = []; - appendCoordinate(coordinates, legStart); - for (const coordinate of directOnEdge) { - appendCoordinate(coordinates, coordinate); - } - appendCoordinate(coordinates, legDestination); - candidates.push({ - coordinates, - usedEdges: [startSnap.edge], - distanceNm: sumRouteDistanceNm(coordinates), - costNm: sumRouteDistanceNm(coordinates) * edgePenaltyMultiplier(startSnap.edge, penaltyCounts) - }); - } - - for (const startNodeId of [startSnap.edge.from, startSnap.edge.to]) { - if (!canTraverseFromSnapToNode(startSnap, startNodeId)) { + for (const { componentId, start: startSnap, destination: destinationSnap } of snapPairs) { + if (routedComponents.has(componentId)) { continue; } - for (const destinationNodeId of [destinationSnap.edge.from, destinationSnap.edge.to]) { - if (!canTraverseFromNodeToSnap(destinationSnap, destinationNodeId)) { - continue; - } - const path = shortestPath(graph, adjacency, startNodeId, destinationNodeId); - if (!path) { - continue; - } + const previousCandidateCount = candidates.length; + if (startSnap.edge.id === destinationSnap.edge.id && canTraverseBetweenSnaps(startSnap, destinationSnap)) { + const directOnEdge = edgePathBetweenSnaps(startSnap, destinationSnap); const coordinates: Coordinate[] = []; - const usedEdges = new Map(); - appendCoordinate(coordinates, legStart); - const startEdgeCoordinates = edgePathFromSnapToNode(startSnap, startNodeId); - for (const coordinate of startEdgeCoordinates) { + for (const coordinate of directOnEdge) { appendCoordinate(coordinates, coordinate); } - usedEdges.set(startSnap.edge.id, startSnap.edge); - - for (const step of path) { - usedEdges.set(step.edge.id, step.edge); - for (const coordinate of edgeCoordinates(step)) { - appendCoordinate(coordinates, coordinate); - } - } - - const destinationEdgeCoordinates = edgePathFromNodeToSnap(destinationSnap, destinationNodeId); - for (const coordinate of destinationEdgeCoordinates) { - appendCoordinate(coordinates, coordinate); - } - usedEdges.set(destinationSnap.edge.id, destinationSnap.edge); appendCoordinate(coordinates, legDestination); - - const distanceNm = sumRouteDistanceNm(coordinates); - const costNm = - sumRouteDistanceNm(startEdgeCoordinates) * edgePenaltyMultiplier(startSnap.edge, penaltyCounts) + - path.reduce((total, step) => total + step.weightNm, 0) + - sumRouteDistanceNm(destinationEdgeCoordinates) * edgePenaltyMultiplier(destinationSnap.edge, penaltyCounts); candidates.push({ coordinates, - usedEdges: [...usedEdges.values()], - distanceNm, - costNm + usedEdges: [startSnap.edge], + distanceNm: sumRouteDistanceNm(coordinates), + costNm: + startSnap.distanceNm + + sumRouteDistanceNm(directOnEdge) * edgePenaltyMultiplier(startSnap.edge, penaltyCounts) + + destinationSnap.distanceNm }); } + + for (const startNodeId of [startSnap.edge.from, startSnap.edge.to]) { + if (!canTraverseFromSnapToNode(startSnap, startNodeId)) { + continue; + } + for (const destinationNodeId of [destinationSnap.edge.from, destinationSnap.edge.to]) { + if (!canTraverseFromNodeToSnap(destinationSnap, destinationNodeId)) { + continue; + } + const path = shortestPath(graph, adjacency, startNodeId, destinationNodeId); + if (!path) { + continue; + } + + const coordinates: Coordinate[] = []; + const usedEdges = new Map(); + + appendCoordinate(coordinates, legStart); + const startEdgeCoordinates = edgePathFromSnapToNode(startSnap, startNodeId); + for (const coordinate of startEdgeCoordinates) { + appendCoordinate(coordinates, coordinate); + } + usedEdges.set(startSnap.edge.id, startSnap.edge); + + for (const step of path) { + usedEdges.set(step.edge.id, step.edge); + for (const coordinate of edgeCoordinates(step)) { + appendCoordinate(coordinates, coordinate); + } + } + + const destinationEdgeCoordinates = edgePathFromNodeToSnap(destinationSnap, destinationNodeId); + for (const coordinate of destinationEdgeCoordinates) { + appendCoordinate(coordinates, coordinate); + } + usedEdges.set(destinationSnap.edge.id, destinationSnap.edge); + appendCoordinate(coordinates, legDestination); + + const distanceNm = sumRouteDistanceNm(coordinates); + const costNm = + startSnap.distanceNm + + sumRouteDistanceNm(startEdgeCoordinates) * edgePenaltyMultiplier(startSnap.edge, penaltyCounts) + + path.reduce((total, step) => total + step.weightNm, 0) + + sumRouteDistanceNm(destinationEdgeCoordinates) * edgePenaltyMultiplier(destinationSnap.edge, penaltyCounts) + + destinationSnap.distanceNm; + candidates.push({ + coordinates, + usedEdges: [...usedEdges.values()], + distanceNm, + costNm + }); + } + } + + if (candidates.length > previousCandidateCount) { + routedComponents.add(componentId); + } } return candidates.sort((a, b) => a.costNm - b.costNm || a.distanceNm - b.distanceNm)[0] ?? null; } -function findNearestEdgeSnap(graph: FairwayGraph, coordinate: Coordinate): EdgeSnap | null { - let nearest: EdgeSnap | null = null; +function findCompatibleEdgeSnapPairs( + graph: FairwayGraph, + start: Coordinate, + destination: Coordinate, + componentByNode: ReadonlyMap +): EdgeSnapPair[] { + const startSnaps = findNearestEdgeSnapsByComponent(graph, start, componentByNode); + const destinationSnaps = findNearestEdgeSnapsByComponent(graph, destination, componentByNode); + const pairs: EdgeSnapPair[] = []; + + for (const [componentId, componentStartSnaps] of startSnaps) { + const componentDestinationSnaps = destinationSnaps.get(componentId); + if (!componentDestinationSnaps) { + continue; + } + + for (const startSnap of componentStartSnaps) { + for (const destinationSnap of componentDestinationSnaps) { + pairs.push({ componentId, start: startSnap, destination: destinationSnap }); + } + } + } + + return pairs.sort( + (a, b) => + a.start.distanceNm + a.destination.distanceNm - + (b.start.distanceNm + b.destination.distanceNm) + ); +} + +function findNearestEdgeSnapsByComponent( + graph: FairwayGraph, + coordinate: Coordinate, + componentByNode: ReadonlyMap +): Map { + const nearestByComponent = new Map(); for (const edge of graph.edges) { + const componentId = componentByNode.get(edge.from); + if (!componentId) { + continue; + } + + let nearestOnEdge: EdgeSnap | null = null; for (let index = 0; index < edge.coordinates.length - 1; index += 1) { const start = edge.coordinates[index]!; const end = edge.coordinates[index + 1]!; const snap = closestPointOnSegment(coordinate, start, end); const distanceNm = haversineDistanceNm(coordinate, snap.coordinate); - if (!nearest || distanceNm < nearest.distanceNm) { - nearest = { + if ( + distanceNm <= graph.maxSnapDistanceNm && + (!nearestOnEdge || distanceNm < nearestOnEdge.distanceNm) + ) { + nearestOnEdge = { edge, coordinate: snap.coordinate, segmentIndex: index, @@ -496,9 +574,51 @@ function findNearestEdgeSnap(graph: FairwayGraph, coordinate: Coordinate): EdgeS }; } } + + if (nearestOnEdge) { + const componentSnaps = nearestByComponent.get(componentId) ?? []; + componentSnaps.push(nearestOnEdge); + componentSnaps.sort((a, b) => a.distanceNm - b.distanceNm); + if (componentSnaps.length > MAX_EDGE_SNAPS_PER_COMPONENT) { + componentSnaps.length = MAX_EDGE_SNAPS_PER_COMPONENT; + } + nearestByComponent.set(componentId, componentSnaps); + } } - return nearest; + return nearestByComponent; +} + +function weaklyConnectedComponents(graph: FairwayGraph): Map { + const neighbours = new Map(); + + for (const edge of graph.edges) { + neighbours.set(edge.from, [...(neighbours.get(edge.from) ?? []), edge.to]); + neighbours.set(edge.to, [...(neighbours.get(edge.to) ?? []), edge.from]); + } + + const componentByNode = new Map(); + for (const startNodeId of neighbours.keys()) { + if (componentByNode.has(startNodeId)) { + continue; + } + + const componentId = startNodeId; + const pending = [startNodeId]; + componentByNode.set(startNodeId, componentId); + + while (pending.length > 0) { + const nodeId = pending.pop()!; + for (const neighbourId of neighbours.get(nodeId) ?? []) { + if (!componentByNode.has(neighbourId)) { + componentByNode.set(neighbourId, componentId); + pending.push(neighbourId); + } + } + } + } + + return componentByNode; } function shortestPath( diff --git a/packages/shared/src/inland-seed.ts b/packages/shared/src/inland-seed.ts index 50b86aa..db88561 100644 --- a/packages/shared/src/inland-seed.ts +++ b/packages/shared/src/inland-seed.ts @@ -517,3 +517,58 @@ export const EMDEN_HAMM_GRAPH: FairwayGraph = { } ] }; + +/** + * Offline fallback for the eastern Ems corridor from Emden Außenhafen towards + * the lower Ems. The geometry is a simplified extract of OSM/Geofabrik data + * (ODbL). Runtime PostGIS/OSM graphs take precedence whenever available. + */ +export const EMDEN_EAST_EMS_GRAPH: FairwayGraph = { + id: "emden-east-ems-seed", + name: "Emden Außenhafen – Unterems", + maxSnapDistanceNm: 0.6, + nodes: [ + { id: "emden-east-start", coordinate: { lat: 53.3422, lon: 7.1871 } }, + { id: "emden-east-destination", coordinate: { lat: 53.4650304, lon: 7.4733641 } } + ], + edges: [ + { + id: "emden-east-ems", + name: "Unterems östlich von Emden", + from: "emden-east-start", + to: "emden-east-destination", + minDepthM: null, + source: "openstreetmap-geofabrik-curated-seed", + coordinates: [ + { lat: 53.3422, lon: 7.1871 }, + { lat: 53.3473059, lon: 7.1911316 }, + { lat: 53.3606114, lon: 7.2037359 }, + { lat: 53.3644551, lon: 7.2082274 }, + { lat: 53.3661998, lon: 7.21042 }, + { lat: 53.3666164, lon: 7.2167288 }, + { lat: 53.3678346, lon: 7.2249094 }, + { lat: 53.3691393, lon: 7.2329392 }, + { lat: 53.3707491, lon: 7.2378392 }, + { lat: 53.373502, lon: 7.2424648 }, + { lat: 53.3759113, lon: 7.2504687 }, + { lat: 53.3764243, lon: 7.2562576 }, + { lat: 53.3799364, lon: 7.2603821 }, + { lat: 53.3834496, lon: 7.2598983 }, + { lat: 53.3874288, lon: 7.2662906 }, + { lat: 53.3920681, lon: 7.2704136 }, + { lat: 53.395175, lon: 7.2798374 }, + { lat: 53.3987926, lon: 7.2914857 }, + { lat: 53.4001184, lon: 7.3043088 }, + { lat: 53.4023614, lon: 7.3192701 }, + { lat: 53.4079972, lon: 7.3347423 }, + { lat: 53.414927, lon: 7.3450734 }, + { lat: 53.420342, lon: 7.3654351 }, + { lat: 53.4267192, lon: 7.3910633 }, + { lat: 53.4359646, lon: 7.420754 }, + { lat: 53.4504899, lon: 7.4518959 }, + { lat: 53.4630916, lon: 7.4715145 }, + { lat: 53.4650304, lon: 7.4733641 } + ] + } + ] +}; diff --git a/packages/shared/src/route.ts b/packages/shared/src/route.ts index fbfd46e..a2af4b1 100644 --- a/packages/shared/src/route.ts +++ b/packages/shared/src/route.ts @@ -1,6 +1,6 @@ import { coordinateToGeoJson, sumRouteDistanceNm } from "./geo.js"; import { buildFairwayRoute, type FairwayGraph } from "./fairway-routing.js"; -import { EMDEN_HAMM_GRAPH } from "./inland-seed.js"; +import { EMDEN_EAST_EMS_GRAPH, EMDEN_HAMM_GRAPH } from "./inland-seed.js"; import type { DepthSample, RouteRequest, @@ -16,7 +16,11 @@ export function buildRoute(request: RouteRequest, graph?: FairwayGraph): RouteRe return buildFairwayRoute(request, graph); } - return buildFairwayRoute(request) ?? buildFairwayRoute(request, EMDEN_HAMM_GRAPH); + return ( + buildFairwayRoute(request) ?? + buildFairwayRoute(request, EMDEN_EAST_EMS_GRAPH) ?? + buildFairwayRoute(request, EMDEN_HAMM_GRAPH) + ); } export function requiredDepthM(profile: VesselProfile): number { diff --git a/packages/shared/tests/route.test.ts b/packages/shared/tests/route.test.ts index 73abe6b..d415b9d 100644 --- a/packages/shared/tests/route.test.ts +++ b/packages/shared/tests/route.test.ts @@ -40,6 +40,44 @@ const ALTERNATIVE_GRAPH: FairwayGraph = { ] }; +const COMPONENT_AWARE_SNAP_GRAPH: FairwayGraph = { + id: "component-aware-snap-test", + name: "Komponentenbewusster Snap-Test", + maxSnapDistanceNm: 0.3, + nodes: [ + { id: "start-decoy-a", coordinate: { lat: 53.3416, lon: 7.186 } }, + { id: "start-decoy-b", coordinate: { lat: 53.342, lon: 7.187 } }, + { id: "destination-decoy-a", coordinate: { lat: 53.3282, lon: 6.9304 } }, + { id: "destination-decoy-b", coordinate: { lat: 53.3286, lon: 6.9294 } }, + { id: "shared-start", coordinate: { lat: 53.3395697, lon: 7.1848883 } }, + { id: "shared-east", coordinate: { lat: 53.3321722, lon: 7.1329034 } }, + { id: "shared-south", coordinate: { lat: 53.313849, lon: 7.0011017 } }, + { id: "shared-destination", coordinate: { lat: 53.3303531, lon: 6.9334715 } } + ], + edges: [ + edge("start-decoy", "start-decoy-a", "start-decoy-b", [ + { lat: 53.3416, lon: 7.186 }, + { lat: 53.342, lon: 7.187 } + ], { source: "closer-but-disconnected-start" }), + edge("destination-decoy", "destination-decoy-a", "destination-decoy-b", [ + { lat: 53.3282, lon: 6.9304 }, + { lat: 53.3286, lon: 6.9294 } + ], { source: "closer-but-disconnected-destination" }), + edge("shared-east", "shared-start", "shared-east", [ + { lat: 53.3395697, lon: 7.1848883 }, + { lat: 53.3321722, lon: 7.1329034 } + ], { source: "shared-local-component" }), + edge("shared-south", "shared-east", "shared-south", [ + { lat: 53.3321722, lon: 7.1329034 }, + { lat: 53.313849, lon: 7.0011017 } + ], { source: "shared-local-component" }), + edge("shared-west", "shared-south", "shared-destination", [ + { lat: 53.313849, lon: 7.0011017 }, + { lat: 53.3303531, lon: 6.9334715 } + ], { source: "shared-local-component" }) + ] +}; + function edge( id: string, from: string, @@ -162,6 +200,88 @@ describe("route assessment", () => { expect(result.dataSources).toContain("fairway-graph:ems-borkum-seed"); }); + it("uses a shared reachable component when the individually nearest edges are disconnected", () => { + const start = { lat: 53.3416, lon: 7.186 }; + const destination = { lat: 53.3282, lon: 6.9304 }; + const routes = buildFairwayRoutes( + { + start, + destination, + vesselProfile: { draughtM: 1, safetyReserveM: 0.3 } + }, + COMPONENT_AWARE_SNAP_GRAPH + ); + + expect(routes).toHaveLength(1); + expect(routes[0]?.geometry.coordinates[0]).toEqual([start.lon, start.lat]); + expect(routes[0]?.geometry.coordinates.at(-1)).toEqual([destination.lon, destination.lat]); + expect(routes[0]?.dataSources).toContain("shared-local-component"); + expect(routes[0]?.dataSources).not.toContain("closer-but-disconnected-start"); + expect(routes[0]?.dataSources).not.toContain("closer-but-disconnected-destination"); + expect(routes[0]?.distanceNm).toBeGreaterThan(9.5); + expect(routes[0]?.distanceNm).toBeLessThan(10.5); + }); + + it("returns no route when start and destination have no shared component inside the snap radius", () => { + const disconnectedGraph: FairwayGraph = { + ...COMPONENT_AWARE_SNAP_GRAPH, + nodes: COMPONENT_AWARE_SNAP_GRAPH.nodes.slice(0, 4), + edges: COMPONENT_AWARE_SNAP_GRAPH.edges.slice(0, 2) + }; + + expect( + buildFairwayRoute( + { + start: { lat: 53.3416, lon: 7.186 }, + destination: { lat: 53.3282, lon: 6.9304 }, + vesselProfile: { draughtM: 1, safetyReserveM: 0.3 } + }, + disconnectedGraph + ) + ).toBeNull(); + }); + + it("tries another snap in the same component when the nearest one-way branch cannot be exited", () => { + const graph: FairwayGraph = { + id: "oneway-snap-fallback", + name: "Einbahnstraßen-Snap-Fallback", + maxSnapDistanceNm: 0.2, + nodes: [ + { id: "junction", coordinate: { lat: 52, lon: 7 } }, + { id: "destination", coordinate: { lat: 52, lon: 7.04 } }, + { id: "oneway-dead-end", coordinate: { lat: 52.001, lon: 7 } } + ], + edges: [ + edge( + "oneway-trap", + "junction", + "oneway-dead-end", + [{ lat: 52, lon: 7 }, { lat: 52.001, lon: 7 }], + { oneway: true, source: "oneway-trap" } + ), + edge( + "main-route", + "junction", + "destination", + [{ lat: 52, lon: 7 }, { lat: 52, lon: 7.04 }], + { source: "routable-main-edge" } + ) + ] + }; + const route = buildFairwayRoute( + { + start: { lat: 52.001, lon: 7 }, + destination: { lat: 52, lon: 7.04 }, + vesselProfile: { draughtM: 1, safetyReserveM: 0.3 } + }, + graph + ); + + expect(route).not.toBeNull(); + expect(route?.dataSources).toContain("routable-main-edge"); + expect(route?.dataSources).not.toContain("oneway-trap"); + }); + it("does not fall back to a misleading straight line when no fairway graph matches", () => { const result = buildRoute({ start: { lat: 54.1749, lon: 12.0731 }, diff --git a/scripts/build-local-fairways.py b/scripts/build-local-fairways.py new file mode 100755 index 0000000..f8eb145 --- /dev/null +++ b/scripts/build-local-fairways.py @@ -0,0 +1,392 @@ +#!/usr/bin/env python3 +"""Build a compact, file-backed fairway dataset from Geofabrik OSM PBFs. + +The importer intentionally uses two streaming passes over every source. The +first pass retains only routable marine/inland ways and their node IDs; the +second retains only coordinates referenced by those ways. Coordinates and +ways shared by neighbouring extracts are merged by their globally unique OSM +IDs. This avoids loading complete OSM node tables into memory and does not +require PostGIS or Docker. +""" + +from __future__ import annotations + +import argparse +from collections import defaultdict +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import sys +import tempfile +from typing import Any + +try: + import osmium +except ImportError as error: + raise SystemExit( + "pyosmium fehlt. Installiere es mit " + "`python3 -m pip install --target .tools/python 'osmium>=4,<5'` " + "und starte mit `PYTHONPATH=.tools/python`." + ) from error + + +KEPT_TAGS = { + "access", + "boat", + "construction", + "depth", + "disused", + "maxdraft", + "maxdraught", + "maxheight", + "maxheight:physical", + "maxwidth", + "maxwidth:physical", + "min_depth", + "motor_vehicle", + "motorboat", + "name", + "oneway", + "proposed", + "ref", + "route", + "seamark:bridge:clearance_height", + "seamark:bridge:clearance_height_safe", + "seamark:fairway:minimum_depth", + "seamark:lock:chamber_width", + "seamark:navigation_line:minimum_depth", + "seamark:recommended_track:minimum_depth", + "seamark:restriction:max_draught", + "seamark:type", + "ship", + "waterway", +} +GENERATOR_VERSION = 2 + + +def is_routable(tags: dict[str, str]) -> bool: + if ( + tags.get("access") in {"no", "private"} + or tags.get("boat") == "no" + or tags.get("ship") == "no" + or tags.get("motorboat") == "no" + or tags.get("disused") == "yes" + or "construction" in tags + or "proposed" in tags + ): + return False + + seamark_type = tags.get("seamark:type") + if seamark_type in {"navigation_line", "recommended_track"}: + return True + if tags.get("waterway") in {"fairway", "canal"}: + return True + if tags.get("waterway") == "river" and any( + tags.get(key) in {"yes", "designated", "permissive"} + for key in ("boat", "ship", "motorboat") + ): + return True + if ( + tags.get("route") == "ferry" + and tags.get("ship") != "no" + and tags.get("motor_vehicle") != "no" + ): + return True + return seamark_type == "fairway" + + +class WayCollector(osmium.SimpleHandler): + def __init__(self) -> None: + super().__init__() + self.ways: list[dict[str, Any]] = [] + self.node_ids: set[int] = set() + + def way(self, way: Any) -> None: + tags = {tag.k: tag.v for tag in way.tags} + if not is_routable(tags): + return + + node_ids = [node.ref for node in way.nodes] + if len(node_ids) < 2: + return + + self.node_ids.update(node_ids) + self.ways.append( + { + "id": str(way.id), + "version": int(getattr(way, "version", 0) or 0), + "nodes": node_ids, + "tags": {key: value for key, value in tags.items() if key in KEPT_TAGS}, + } + ) + + +class NodeCollector(osmium.SimpleHandler): + def __init__(self, wanted: set[int]) -> None: + super().__init__() + self.wanted = wanted + self.coordinates: dict[int, tuple[int, float, float]] = {} + + def node(self, node: Any) -> None: + if node.id not in self.wanted or not node.location.valid(): + return + candidate = ( + int(getattr(node, "version", 0) or 0), + node.location.lat, + node.location.lon, + ) + current = self.coordinates.get(node.id) + if current is None or candidate > current: + self.coordinates[node.id] = candidate + + +def way_bbox(coordinates: list[list[float]]) -> list[float]: + lats = [coordinate[0] for coordinate in coordinates] + lons = [coordinate[1] for coordinate in coordinates] + return [min(lons), min(lats), max(lons), max(lats)] + + +def source_region(path: Path) -> str: + name = path.name.lower() + for suffix in ("-latest.osm.pbf", ".osm.pbf", ".pbf"): + if name.endswith(suffix): + name = name[: -len(suffix)] + break + region = re.sub(r"[^a-z0-9]+", "-", name).strip("-") + return region or "unknown" + + +def adjacent_md5(path: Path) -> str | None: + checksum_path = Path(f"{path}.md5") + if not checksum_path.is_file(): + return None + fields = checksum_path.read_text(encoding="utf-8").split(maxsplit=1) + if not fields: + return None + checksum = fields[0].lower() + return checksum if re.fullmatch(r"[0-9a-f]{32}", checksum) else None + + +def candidate_rank(candidate: dict[str, Any]) -> tuple[int, int, str]: + """Return a deterministic preference for duplicate versions of an OSM way.""" + + fingerprint = hashlib.sha256( + json.dumps( + { + "nodes": candidate["nodes"], + "tags": candidate["tags"], + "coordinates": candidate["coordinates"], + }, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + ).hexdigest() + return candidate["version"], len(candidate["nodes"]), fingerprint + + +def build_document(pbf_paths: list[Path]) -> dict[str, Any]: + sources: list[dict[str, Any]] = [] + wanted_node_ids: set[int] = set() + + for index, path in enumerate(pbf_paths, start=1): + print( + f"Pass 1/2 [{index}/{len(pbf_paths)}]: routbare Wege aus {path} lesen", + flush=True, + ) + collector = WayCollector() + processor = osmium.FileProcessor( + str(path), osmium.osm.WAY + ).with_filter(osmium.filter.KeyFilter("seamark:type", "waterway", "route")) + for way in processor: + collector.way(way) + wanted_node_ids.update(collector.node_ids) + stat = path.stat() + sources.append( + { + "path": path, + "region": source_region(path), + "collector": collector, + "metadata": { + "region": source_region(path), + "file": path.name, + "sizeBytes": stat.st_size, + "modifiedAt": datetime.fromtimestamp( + stat.st_mtime, timezone.utc + ).isoformat(), + "checksumMd5": adjacent_md5(path), + "routableWaysFound": len(collector.ways), + "referencedNodes": len(collector.node_ids), + "nodeCoordinatesFound": 0, + "exportedCandidateWays": 0, + "incompleteWaysSkipped": 0, + "closedFairwaysSkipped": 0, + }, + } + ) + print( + f"{len(collector.ways)} Wege mit " + f"{len(collector.node_ids)} referenzierten Knoten gefunden", + flush=True, + ) + + global_coordinates: dict[int, tuple[int, float, float]] = {} + coordinate_conflicts = 0 + for index, source in enumerate(sources, start=1): + print( + f"Pass 2/2 [{index}/{len(sources)}]: benötigte Knotenkoordinaten " + f"aus {source['path']} lesen", + flush=True, + ) + nodes = NodeCollector(wanted_node_ids) + if wanted_node_ids: + processor = osmium.FileProcessor( + str(source["path"]), osmium.osm.NODE + ).with_filter(osmium.filter.IdFilter(wanted_node_ids)) + for node in processor: + nodes.node(node) + + source["metadata"]["nodeCoordinatesFound"] = len(nodes.coordinates) + for node_id, candidate in nodes.coordinates.items(): + current = global_coordinates.get(node_id) + if current is not None and current[1:] != candidate[1:]: + coordinate_conflicts += 1 + if current is None or candidate > current: + global_coordinates[node_id] = candidate + + candidates_by_id: dict[str, list[dict[str, Any]]] = defaultdict(list) + for source in sources: + for way in source["collector"].ways: + if any(node_id not in global_coordinates for node_id in way["nodes"]): + source["metadata"]["incompleteWaysSkipped"] += 1 + continue + coordinates = [ + [global_coordinates[node_id][1], global_coordinates[node_id][2]] + for node_id in way["nodes"] + ] + if ( + way["tags"].get("seamark:type") == "fairway" + and coordinates[0] == coordinates[-1] + ): + source["metadata"]["closedFairwaysSkipped"] += 1 + continue + + source["metadata"]["exportedCandidateWays"] += 1 + candidates_by_id[way["id"]].append( + { + **way, + "coordinates": coordinates, + "region": source["region"], + "sourceFile": source["path"].name, + } + ) + + exported: list[dict[str, Any]] = [] + duplicate_ways_merged = 0 + for way_id, candidates in candidates_by_id.items(): + winner = max(candidates, key=candidate_rank) + duplicate_ways_merged += len(candidates) - 1 + exported.append( + { + "id": way_id, + "osmVersion": winner["version"], + "regions": sorted({candidate["region"] for candidate in candidates}), + "sourceFiles": sorted( + {candidate["sourceFile"] for candidate in candidates} + ), + "bbox": way_bbox(winner["coordinates"]), + "tags": winner["tags"], + "coordinates": winner["coordinates"], + } + ) + exported.sort(key=lambda way: int(way["id"])) + + source_metadata = [source["metadata"] for source in sources] + regions = list(dict.fromkeys(source["region"] for source in sources)) + source_label = ( + pbf_paths[0].name if len(pbf_paths) == 1 else "+".join(regions) + ) + modified_at = max(path.stat().st_mtime for path in pbf_paths) + return { + "version": 1, + "generatorVersion": GENERATOR_VERSION, + "source": source_label, + "sources": source_metadata, + "sourceSizeBytes": sum(path.stat().st_size for path in pbf_paths), + "sourceModifiedAt": datetime.fromtimestamp( + modified_at, timezone.utc + ).isoformat(), + "generatedAt": datetime.now(timezone.utc).isoformat(), + "incompleteWaysSkipped": sum( + source["metadata"]["incompleteWaysSkipped"] for source in sources + ), + "closedFairwaysSkipped": sum( + source["metadata"]["closedFairwaysSkipped"] for source in sources + ), + "duplicateWaysMerged": duplicate_ways_merged, + "nodeCoordinateConflicts": coordinate_conflicts, + "ways": exported, + } + + +def write_document(document: dict[str, Any], output: Path) -> None: + output.parent.mkdir(parents=True, exist_ok=True) + temporary_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + "w", + encoding="utf-8", + dir=output.parent, + prefix=f".{output.name}.", + suffix=".tmp", + delete=False, + ) as temporary: + json.dump(document, temporary, ensure_ascii=False, separators=(",", ":")) + temporary.write("\n") + temporary_path = Path(temporary.name) + temporary_path.chmod(0o644) + os.replace(temporary_path, output) + except BaseException: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + raise + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("pbf", type=Path, nargs="+") + parser.add_argument( + "--output", + type=Path, + default=Path("data/local/germany-netherlands-fairways.json"), + ) + args = parser.parse_args() + + pbf_paths: list[Path] = [] + seen_paths: set[Path] = set() + for path in args.pbf: + if not path.is_file(): + parser.error(f"PBF nicht gefunden: {path}") + resolved = path.resolve() + if resolved in seen_paths: + parser.error(f"PBF doppelt angegeben: {path}") + seen_paths.add(resolved) + pbf_paths.append(path) + + document = build_document(pbf_paths) + write_document(document, args.output) + print( + f"{len(document['ways'])} Wege nach {args.output} geschrieben " + f"({document['duplicateWaysMerged']} Duplikate zusammengeführt; " + f"{document['incompleteWaysSkipped']} unvollständige Wege verworfen; " + f"{args.output.stat().st_size / 1024 / 1024:.1f} MiB)", + flush=True, + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/download-geofabrik.sh b/scripts/download-geofabrik.sh index 4978248..32f1ed8 100755 --- a/scripts/download-geofabrik.sh +++ b/scripts/download-geofabrik.sh @@ -29,23 +29,119 @@ for region in "${regions[@]}"; do esac target="${OUT_DIR}/${file_name}" checksum_target="${target}.md5" + checksum_download="${checksum_target}.part" + download_part="${target}.part" + download_part_checksum="${download_part}.expected-md5" - echo "Downloading $url.md5" - curl --fail --location --retry 5 --retry-delay 5 --output "$checksum_target" "${url}.md5" + # Resolve the `latest` PBF redirect first and fetch the checksum belonging to + # that exact dated snapshot. This prevents transparent download proxies from + # combining a fresh PBF redirect with a stale `latest` checksum. + resolved_url="$( + curl \ + --fail \ + --head \ + --location \ + --silent \ + --show-error \ + --retry 5 \ + --retry-all-errors \ + --retry-delay 5 \ + --output /dev/null \ + --write-out '%{url_effective}' \ + "$url" + )" + resolved_url="${resolved_url%%\?*}" + resolved_file_name="${resolved_url##*/}" + if [[ "$resolved_file_name" =~ ^[a-z0-9-]+-[0-9]{6}\.osm\.pbf$ ]]; then + download_url="$resolved_url" + else + download_url="$url" + fi + checksum_url="${download_url}.md5" + + echo "Downloading $checksum_url" + curl \ + --fail \ + --location \ + --silent \ + --show-error \ + --retry 5 \ + --retry-all-errors \ + --retry-delay 5 \ + --output "$checksum_download" \ + "$checksum_url" + + expected_checksum="$(awk 'NR == 1 { print tolower($1) }' "$checksum_download")" + if [[ ! "$expected_checksum" =~ ^[0-9a-f]{32}$ ]]; then + echo "Invalid checksum response for $url" >&2 + exit 1 + fi + + checksum_file_name="$(awk 'NR == 1 { name = $2; sub(/^\*/, "", name); print name }' "$checksum_download")" + if [[ "$checksum_file_name" =~ ^[a-z0-9-]+-[0-9]{6}\.osm\.pbf$ ]] && + [[ "$checksum_file_name" != "${download_url##*/}" ]]; then + echo "Checksum file does not match resolved snapshot: $checksum_file_name" >&2 + exit 1 + fi - expected_checksum="$(awk 'NR == 1 { print $1 }' "$checksum_target")" if [[ -f "$target" ]] && [[ "$(md5sum "$target" | awk '{ print $1 }')" == "$expected_checksum" ]]; then + mv -f "$checksum_download" "$checksum_target" + rm -f "$download_part" "$download_part_checksum" echo "$(basename "$target"): already current" continue fi - echo "Downloading $url" - curl --fail --location --continue-at - --retry 5 --retry-delay 5 --output "$target" "$url" + previous_part_checksum="" + if [[ -f "$download_part_checksum" ]]; then + previous_part_checksum="$(awk 'NR == 1 { print tolower($1) }' "$download_part_checksum")" + fi + if [[ "$previous_part_checksum" != "$expected_checksum" ]]; then + rm -f "$download_part" + fi + printf '%s\n' "$expected_checksum" >"$download_part_checksum" - actual_checksum="$(md5sum "$target" | awk '{ print $1 }')" - if [[ -z "$expected_checksum" || "$actual_checksum" != "$expected_checksum" ]]; then - echo "Checksum verification failed for $target" >&2 + part_is_complete=false + if [[ -f "$download_part" ]] && [[ "$(md5sum "$download_part" | awk '{ print $1 }')" == "$expected_checksum" ]]; then + part_is_complete=true + fi + + if [[ "$part_is_complete" == false ]]; then + echo "Downloading $download_url" + if ! curl \ + --fail \ + --location \ + --silent \ + --show-error \ + --continue-at - \ + --retry 5 \ + --retry-all-errors \ + --retry-delay 5 \ + --output "$download_part" \ + "$download_url"; then + echo "Resuming failed; retrying $download_url from the beginning" >&2 + rm -f "$download_part" + curl \ + --fail \ + --location \ + --silent \ + --show-error \ + --retry 5 \ + --retry-all-errors \ + --retry-delay 5 \ + --output "$download_part" \ + "$download_url" + fi + fi + + actual_checksum="$(md5sum "$download_part" | awk '{ print $1 }')" + if [[ "$actual_checksum" != "$expected_checksum" ]]; then + rm -f "$download_part" "$download_part_checksum" + echo "Checksum verification failed for $url; previous snapshot retained" >&2 exit 1 fi + + mv -f "$download_part" "$target" + mv -f "$checksum_download" "$checksum_target" + rm -f "$download_part_checksum" echo "$(basename "$target"): OK" done diff --git a/scripts/setup-local-germany.sh b/scripts/setup-local-germany.sh new file mode 100755 index 0000000..af3558a --- /dev/null +++ b/scripts/setup-local-germany.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +echo "Hinweis: setup-local-germany.sh ist veraltet; richte Deutschland und Niederlande gemeinsam ein." >&2 +exec "$ROOT_DIR/scripts/setup-local-routing.sh" "$@" diff --git a/scripts/setup-local-routing.sh b/scripts/setup-local-routing.sh new file mode 100755 index 0000000..2217a62 --- /dev/null +++ b/scripts/setup-local-routing.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +GEOFABRIK_DIR="${WATERMAPS_GEOFABRIK_DIR:-${SEA_COMPASS_GEOFABRIK_DIR:-data/geofabrik}}" +PYTHON_TARGET="$ROOT_DIR/.tools/python" +OUTPUT_PATH="${WATERMAPS_LOCAL_FAIRWAYS_PATH:-$ROOT_DIR/data/local/germany-netherlands-fairways.json}" + +if [[ "$GEOFABRIK_DIR" != /* ]]; then + GEOFABRIK_DIR="$ROOT_DIR/$GEOFABRIK_DIR" +fi +if [[ "$OUTPUT_PATH" != /* ]]; then + OUTPUT_PATH="$ROOT_DIR/$OUTPUT_PATH" +fi + +PBF_PATHS=( + "$GEOFABRIK_DIR/germany-latest.osm.pbf" + "$GEOFABRIK_DIR/netherlands-latest.osm.pbf" +) + +cd "$ROOT_DIR" + +# This validates existing snapshots and only replaces them after a complete, +# checksum-verified download. +WATERMAPS_GEOFABRIK_DIR="$GEOFABRIK_DIR" \ + ./scripts/download-geofabrik.sh germany netherlands + +# A successful previous build can be reused when both downloaded snapshots +# still match the checksums recorded in its per-source metadata. +if python3 - "$OUTPUT_PATH" "${PBF_PATHS[@]}" <<'PY' +import json +from pathlib import Path +import re +import sys + +output = Path(sys.argv[1]) +pbf_paths = [Path(value) for value in sys.argv[2:]] +try: + document = json.loads(output.read_text(encoding="utf-8")) + sources = { + source["file"]: source + for source in document["sources"] + if isinstance(source, dict) and isinstance(source.get("file"), str) + } + if ( + document.get("version") != 1 + or document.get("generatorVersion") != 2 + or not isinstance(document.get("ways"), list) + ): + raise ValueError("unsupported index format") + for pbf_path in pbf_paths: + checksum_text = Path(f"{pbf_path}.md5").read_text(encoding="utf-8") + checksum = checksum_text.split(maxsplit=1)[0].lower() + metadata = sources[pbf_path.name] + if not re.fullmatch(r"[0-9a-f]{32}", checksum): + raise ValueError("invalid checksum") + if metadata.get("checksumMd5") != checksum: + raise ValueError("snapshot changed") + if metadata.get("sizeBytes") != pbf_path.stat().st_size: + raise ValueError("snapshot size changed") +except ( + FileNotFoundError, + IndexError, + KeyError, + TypeError, + ValueError, + json.JSONDecodeError, +): + raise SystemExit(1) +PY +then + echo "Lokaler Deutschland-/Niederlande-Routingindex ist bereits aktuell: $OUTPUT_PATH" + exit 0 +fi + +mkdir -p "$PYTHON_TARGET" +if ! PYTHONPATH="$PYTHON_TARGET" python3 - <<'PY' +from importlib.metadata import version +import osmium + +raise SystemExit(0 if version("osmium") == "4.3.1" else 1) +PY +then + python3 -m pip install \ + --disable-pip-version-check \ + --target "$PYTHON_TARGET" \ + --upgrade \ + "osmium==4.3.1" +fi + +PYTHONPATH="$PYTHON_TARGET" python3 scripts/build-local-fairways.py \ + "${PBF_PATHS[@]}" \ + --output "$OUTPUT_PATH" + +echo "Lokale Deutschland-/Niederlande-Fahrrouten sind bereit: $OUTPUT_PATH" diff --git a/scripts/tests/test_build_local_fairways.py b/scripts/tests/test_build_local_fairways.py new file mode 100644 index 0000000..a0db137 --- /dev/null +++ b/scripts/tests/test_build_local_fairways.py @@ -0,0 +1,194 @@ +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import stat +import subprocess +import sys +import tempfile +import unittest + +try: + import osmium +except ImportError: + osmium = None + + +ROOT_DIR = Path(__file__).resolve().parents[2] +BUILDER = ROOT_DIR / "scripts" / "build-local-fairways.py" + + +def write_pbf( + path: Path, + nodes: list[tuple[int, float, float]], + ways: list[tuple[int, int, list[int], dict[str, str]]], +) -> None: + assert osmium is not None + with osmium.SimpleWriter(str(path)) as writer: + for node_id, lon, lat in nodes: + writer.add_node( + osmium.osm.mutable.Node( + id=node_id, + version=1, + location=(lon, lat), + ) + ) + for way_id, version, node_ids, tags in ways: + writer.add_way( + osmium.osm.mutable.Way( + id=way_id, + version=version, + nodes=node_ids, + tags=tags, + ) + ) + checksum = hashlib.md5(path.read_bytes()).hexdigest() + Path(f"{path}.md5").write_text( + f"{checksum} {path.name}\n", + encoding="utf-8", + ) + + +@unittest.skipIf(osmium is None, "pyosmium/osmium is not installed") +class BuildLocalFairwaysTest(unittest.TestCase): + def test_merges_two_extracts_and_deduplicates_shared_osm_ways(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + work_dir = Path(temporary_directory) + germany = work_dir / "germany-latest.osm.pbf" + netherlands = work_dir / "netherlands-latest.osm.pbf" + output = work_dir / "germany-netherlands-fairways.json" + + write_pbf( + germany, + [ + (1, 7.0, 53.0), + (2, 7.1, 53.1), + (3, 7.2, 53.2), + ], + [ + (100, 1, [1, 2], {"waterway": "canal", "name": "DE Kanal"}), + ( + 200, + 1, + [2, 3], + {"seamark:type": "navigation_line", "name": "Alte Linie"}, + ), + (300, 1, [1, 2, 1], {"seamark:type": "fairway"}), + (500, 1, [1, 9], {"waterway": "canal", "name": "Grenzkanal"}), + (600, 1, [1, 9999], {"waterway": "canal"}), + ], + ) + write_pbf( + netherlands, + [ + (2, 7.1, 53.1), + (3, 7.2, 53.2), + (4, 7.3, 53.3), + (9, 7.05, 53.05), + ], + [ + ( + 200, + 2, + [2, 3, 4], + {"seamark:type": "navigation_line", "name": "Nieuwe lijn"}, + ), + (400, 1, [3, 4], {"route": "ferry", "name": "Veerboot"}), + ], + ) + + result = subprocess.run( + [ + sys.executable, + str(BUILDER), + str(germany), + str(netherlands), + "--output", + str(output), + ], + cwd=ROOT_DIR, + env={ + **os.environ, + "PYTHONPATH": str(ROOT_DIR / ".tools" / "python"), + }, + check=False, + capture_output=True, + text=True, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + document = json.loads(output.read_text(encoding="utf-8")) + self.assertEqual( + stat.S_IMODE(output.stat().st_mode), + 0o644, + "the unprivileged application container must be able to read the index", + ) + self.assertEqual(document["version"], 1) + self.assertEqual(document["generatorVersion"], 2) + self.assertEqual(document["source"], "germany+netherlands") + self.assertEqual(document["duplicateWaysMerged"], 1) + self.assertEqual(document["incompleteWaysSkipped"], 1) + self.assertEqual(document["closedFairwaysSkipped"], 1) + self.assertEqual([way["id"] for way in document["ways"]], ["100", "200", "400", "500"]) + + shared_way = next(way for way in document["ways"] if way["id"] == "200") + self.assertEqual(shared_way["osmVersion"], 2) + self.assertEqual(shared_way["regions"], ["germany", "netherlands"]) + self.assertEqual( + shared_way["sourceFiles"], + ["germany-latest.osm.pbf", "netherlands-latest.osm.pbf"], + ) + self.assertEqual(shared_way["tags"]["name"], "Nieuwe lijn") + self.assertEqual(len(shared_way["coordinates"]), 3) + + cross_border_way = next( + way for way in document["ways"] if way["id"] == "500" + ) + self.assertEqual( + cross_border_way["coordinates"], + [[53.0, 7.0], [53.05, 7.05]], + ) + + self.assertEqual( + [source["region"] for source in document["sources"]], + ["germany", "netherlands"], + ) + self.assertTrue( + all(source["checksumMd5"] for source in document["sources"]) + ) + + def test_failed_source_read_keeps_existing_output(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + work_dir = Path(temporary_directory) + broken_pbf = work_dir / "germany-latest.osm.pbf" + output = work_dir / "germany-netherlands-fairways.json" + broken_pbf.write_bytes(b"not an OSM PBF") + previous_content = '{"version":1,"source":"previous","ways":[]}\n' + output.write_text(previous_content, encoding="utf-8") + + result = subprocess.run( + [ + sys.executable, + str(BUILDER), + str(broken_pbf), + "--output", + str(output), + ], + cwd=ROOT_DIR, + env={ + **os.environ, + "PYTHONPATH": str(ROOT_DIR / ".tools" / "python"), + }, + check=False, + capture_output=True, + text=True, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(output.read_text(encoding="utf-8"), previous_content) + + +if __name__ == "__main__": + unittest.main()