Bind Geofabrik downloads to resolved mirrors

deploy/scripts/tests/download-geofabrik.test.sh

@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
+DOWNLOAD_SCRIPT="$ROOT_DIR/scripts/download-geofabrik.sh"
+TEST_ROOT="$(mktemp -d)"
+FAKE_BIN="$TEST_ROOT/bin"
+CALL_LOG="$TEST_ROOT/curl.log"
+PBF_FIXTURE="$TEST_ROOT/snapshot.osm.pbf"
+CHECKSUM_FIXTURE="$TEST_ROOT/snapshot.osm.pbf.md5"
+trap 'rm -rf -- "$TEST_ROOT"' EXIT
+
+mkdir -p "$FAKE_BIN"
+printf 'verified geofabrik snapshot\n' >"$PBF_FIXTURE"
+
+fail() {
+  printf 'Fehler: %s\n' "$*" >&2
+  exit 1
+}
+
+cat >"$FAKE_BIN/curl" <<'SH'
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+is_head=false
+output_path=""
+url="${!#}"
+for ((index = 1; index <= $#; index += 1)); do
+  argument="${!index}"
+  case "$argument" in
+    --head)
+      is_head=true
+      ;;
+    --output)
+      index=$((index + 1))
+      output_path="${!index}"
+      ;;
+  esac
+done
+
+if [[ "$is_head" == "true" ]]; then
+  printf 'HEAD|%s\n' "$url" >>"$FAKE_CALL_LOG"
+  printf '%s' "$FAKE_RESOLVED_URL"
+  exit 0
+fi
+
+printf 'GET|%s\n' "$url" >>"$FAKE_CALL_LOG"
+if [[ "$url" == "$FAKE_CHECKSUM_URL" ]]; then
+  cp "$FAKE_CHECKSUM_FIXTURE" "$output_path"
+  exit 0
+fi
+if [[ -n "${FAKE_PBF_URL:-}" && "$url" == "$FAKE_PBF_URL" ]]; then
+  cp "$FAKE_PBF_FIXTURE" "$output_path"
+  exit 0
+fi
+
+printf 'Unerwarteter Download: %s\n' "$url" >&2
+exit 90
+SH
+chmod 0755 "$FAKE_BIN/curl"
+
+snapshot_checksum="$(md5sum "$PBF_FIXTURE" | awk '{ print $1 }')"
+printf '%s  germany-260728.osm.pbf\n' \
+  "$snapshot_checksum" \
+  >"$CHECKSUM_FIXTURE"
+
+germany_output="$TEST_ROOT/germany"
+germany_latest='https://download.example/europe/germany-latest.osm.pbf'
+germany_mirror='https://mirror.example/geofabrik/germany-latest.osm.pbf'
+PATH="$FAKE_BIN:$PATH" \
+FAKE_CALL_LOG="$CALL_LOG" \
+FAKE_RESOLVED_URL="$germany_mirror" \
+FAKE_CHECKSUM_URL="$germany_mirror.md5" \
+FAKE_CHECKSUM_FIXTURE="$CHECKSUM_FIXTURE" \
+FAKE_PBF_URL="$germany_mirror" \
+FAKE_PBF_FIXTURE="$PBF_FIXTURE" \
+WATERMAPS_GEOFABRIK_DIR="$germany_output" \
+WATERMAPS_GEOFABRIK_EUROPE_BASE_URL='https://download.example/europe' \
+  "$DOWNLOAD_SCRIPT" germany
+
+cmp --silent "$PBF_FIXTURE" "$germany_output/germany-latest.osm.pbf" ||
+  fail "Der prüfsummengebundene Snapshot wurde nicht aktiviert."
+grep -Fxq "GET|$germany_mirror.md5" "$CALL_LOG" ||
+  fail "Die Prüfsumme wurde nicht vom effektiven Mirror gelesen."
+grep -Fxq "GET|$germany_mirror" "$CALL_LOG" ||
+  fail "Der PBF wurde nicht vom selben effektiven Mirror gelesen."
+if grep -Fxq "GET|$germany_latest.md5" "$CALL_LOG"; then
+  fail "Die potenziell veraltete Prüfsumme der ursprünglichen URL wurde verwendet."
+fi
+
+printf '%s  netherlands-260727.osm.pbf\n' \
+  "$snapshot_checksum" \
+  >"$CHECKSUM_FIXTURE"
+: >"$CALL_LOG"
+netherlands_output="$TEST_ROOT/netherlands"
+mkdir -p "$netherlands_output"
+printf 'previous valid snapshot\n' \
+  >"$netherlands_output/netherlands-latest.osm.pbf"
+netherlands_dated='https://mirror.example/geofabrik/netherlands-260728.osm.pbf'
+
+set +e
+PATH="$FAKE_BIN:$PATH" \
+FAKE_CALL_LOG="$CALL_LOG" \
+FAKE_RESOLVED_URL="$netherlands_dated" \
+FAKE_CHECKSUM_URL="$netherlands_dated.md5" \
+FAKE_CHECKSUM_FIXTURE="$CHECKSUM_FIXTURE" \
+FAKE_PBF_URL='' \
+FAKE_PBF_FIXTURE="$PBF_FIXTURE" \
+WATERMAPS_GEOFABRIK_DIR="$netherlands_output" \
+WATERMAPS_GEOFABRIK_EUROPE_BASE_URL='https://download.example/europe' \
+  "$DOWNLOAD_SCRIPT" netherlands \
+  >"$TEST_ROOT/mismatch.out" \
+  2>&1
+mismatch_status=$?
+set -e
+
+[[ "$mismatch_status" -ne 0 ]] ||
+  fail "Eine Prüfsumme für einen anderen datierten Snapshot wurde akzeptiert."
+grep -Fxq 'previous valid snapshot' \
+  "$netherlands_output/netherlands-latest.osm.pbf" ||
+  fail "Der bisherige Snapshot wurde nach einem Mismatch verändert."
+if grep -Fxq "GET|$netherlands_dated" "$CALL_LOG"; then
+  fail "Der PBF-Download begann trotz abweichendem Prüfsummendateinamen."
+fi
+
+printf 'Geofabrik-Download: Mirror-Bindung und datierter Mismatch: OK\n'

package.json

@@ -22,7 +22,7 @@
     "setup:local-routing": "./scripts/setup-local-routing.sh",
     "sync:euris-locks": "node scripts/sync-euris-locks.mjs",
     "test": "npm run build --workspace @watermaps/shared && npm run test --workspace @watermaps/shared && npm run test --workspace @watermaps/api && npm run test --workspace @watermaps/web && npm run test:local-routing && npm run test:deployment",
-    "test:deployment": "bash deploy/scripts/tests/image-references.test.sh && bash deploy/scripts/tests/auto-deploy.test.sh && bash deploy/scripts/tests/upload-and-deploy.test.sh && bash deploy/scripts/tests/route-data-helpers.test.sh && bash deploy/scripts/tests/update-route-data.test.sh",
+    "test:deployment": "bash deploy/scripts/tests/image-references.test.sh && bash deploy/scripts/tests/download-geofabrik.test.sh && bash deploy/scripts/tests/auto-deploy.test.sh && bash deploy/scripts/tests/upload-and-deploy.test.sh && bash deploy/scripts/tests/route-data-helpers.test.sh && bash deploy/scripts/tests/update-route-data.test.sh",
     "test:e2e": "npm run test:e2e --workspace @watermaps/web",
     "test:local-routing": "PYTHONPATH=.tools/python python3 -m unittest discover -s scripts/tests -p 'test_*.py'",
     "typecheck": "npm run build --workspace @watermaps/shared && npm run typecheck --workspace @watermaps/shared && npm run typecheck --workspace @watermaps/api && npm run typecheck --workspace @watermaps/web"

scripts/download-geofabrik.sh

@@ -18,6 +18,10 @@ fi
 mkdir -p "$OUT_DIR"
 
 for region in "${regions[@]}"; do
+  if [[ ! "$region" =~ ^[a-z0-9-]+$ ]]; then
+    echo "Invalid Geofabrik region: $region" >&2
+    exit 1
+  fi
   file_name="${region}-latest.osm.pbf"
   case "$region" in
     germany|netherlands)
@@ -33,9 +37,10 @@ for region in "${regions[@]}"; do
   download_part="${target}.part"
   download_part_checksum="${download_part}.expected-md5"
 
-  # Resolve the `latest` PBF redirect first and fetch the checksum belonging to
+  # Resolve the effective `latest` endpoint first and fetch its checksum from
-  # that exact dated snapshot. This prevents transparent download proxies from
+  # the same mirror. Some Geofabrik mirrors keep `latest` in the effective URL
-  # combining a fresh PBF redirect with a stale `latest` checksum.
+  # while the checksum names the dated snapshot; both forms remain bound by
+  # the checksum verification before the active PBF is replaced.
   resolved_url="$(
     curl \
       --fail \
@@ -52,7 +57,8 @@ for region in "${regions[@]}"; do
   )"
   resolved_url="${resolved_url%%\?*}"
   resolved_file_name="${resolved_url##*/}"
-  if [[ "$resolved_file_name" =~ ^[a-z0-9-]+-[0-9]{6}\.osm\.pbf$ ]]; then
+  if [[ "$resolved_file_name" == "$file_name" ||
+        "$resolved_file_name" =~ ^${region}-[0-9]{6}\.osm\.pbf$ ]]; then
     download_url="$resolved_url"
   else
     download_url="$url"
@@ -78,11 +84,18 @@ for region in "${regions[@]}"; do
   fi
 
   checksum_file_name="$(awk 'NR == 1 { name = $2; sub(/^\*/, "", name); print name }' "$checksum_download")"
-  if [[ "$checksum_file_name" =~ ^[a-z0-9-]+-[0-9]{6}\.osm\.pbf$ ]] &&
+  download_file_name="${download_url##*/}"
-    [[ "$checksum_file_name" != "${download_url##*/}" ]]; then
+  if [[ "$download_file_name" =~ ^${region}-[0-9]{6}\.osm\.pbf$ ]]; then
+    if [[ "$checksum_file_name" != "$download_file_name" ]]; then
       echo "Checksum file does not match resolved snapshot: $checksum_file_name" >&2
       exit 1
     fi
+  elif [[ "$download_file_name" == "$file_name" ]] &&
+    [[ "$checksum_file_name" != "$file_name" &&
+      ! "$checksum_file_name" =~ ^${region}-[0-9]{6}\.osm\.pbf$ ]]; then
+    echo "Checksum does not identify requested region: $checksum_file_name" >&2
+    exit 1
+  fi
 
   if [[ -f "$target" ]] && [[ "$(md5sum "$target" | awk '{ print $1 }')" == "$expected_checksum" ]]; then
     mv -f "$checksum_download" "$checksum_target"