optimized events

2026-07-28 14:36:47 +02:00
parent b98ec8bc6f
commit 593dbd5f85
42 changed files with 2405 additions and 283 deletions
@@ -0,0 +1,158 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+cd /workspace
+
+database_url="${DATABASE_URL:-}"
+geofabrik_dir="${WATERMAPS_GEOFABRIK_DIR:-/workspace/data/geofabrik}"
+marker_file="${WATERMAPS_MARINE_MARKER_PATH:-/workspace/data/local/.marine-features.ready}"
+temporary_root="${TMPDIR:-/tmp}"
+pbf_paths=(
+  "$geofabrik_dir/germany-latest.osm.pbf"
+  "$geofabrik_dir/netherlands-latest.osm.pbf"
+)
+
+if [[ -z "$database_url" ]]; then
+  printf 'Fehler: DATABASE_URL ist für den Marine-Feature-Import erforderlich.\n' >&2
+  exit 1
+fi
+
+if [[ ! -x /workspace/scripts/import-geofabrik.sh ]]; then
+  printf 'Fehler: scripts/import-geofabrik.sh fehlt oder ist nicht ausführbar.\n' >&2
+  exit 1
+fi
+
+mkdir -p "$(dirname "$marker_file")" "$temporary_root"
+export TMPDIR="$temporary_root"
+
+declare -A source_checksums
+for pbf_path in "${pbf_paths[@]}"; do
+  if [[ ! -s "$pbf_path" ]]; then
+    printf 'Fehler: Geofabrik-Snapshot fehlt oder ist leer: %s\n' "$pbf_path" >&2
+    exit 1
+  fi
+
+  checksum_file="${pbf_path}.md5"
+  if [[ ! -s "$checksum_file" ]]; then
+    printf 'Fehler: Geofabrik-Prüfsumme fehlt: %s\n' "$checksum_file" >&2
+    exit 1
+  fi
+
+  expected_checksum="$(awk 'NR == 1 { print tolower($1) }' "$checksum_file")"
+  if [[ ! "$expected_checksum" =~ ^[0-9a-f]{32}$ ]]; then
+    printf 'Fehler: Ungültige Geofabrik-Prüfsumme in %s\n' "$checksum_file" >&2
+    exit 1
+  fi
+
+  actual_checksum="$(md5sum "$pbf_path" | awk '{ print $1 }')"
+  if [[ "$actual_checksum" != "$expected_checksum" ]]; then
+    printf 'Fehler: Geofabrik-Snapshot stimmt nicht mit seiner Prüfsumme überein: %s\n' "$pbf_path" >&2
+    exit 1
+  fi
+  source_checksums["$(basename "$pbf_path" -latest.osm.pbf)"]="$actual_checksum"
+done
+
+# Every imported OSM row receives updated_at=now(). The shared database
+# timestamp lets us remove disappeared OSM objects only after both regional
+# imports completed and passed the sanity checks. Other sources such as EuRIS
+# and facility-website enrichments are deliberately preserved.
+import_started_epoch="$(
+  psql "$database_url" \
+    --no-psqlrc \
+    --tuples-only \
+    --no-align \
+    --set ON_ERROR_STOP=1 \
+    --command "SELECT extract(epoch FROM clock_timestamp());"
+)"
+import_started_epoch="${import_started_epoch//[[:space:]]/}"
+if [[ ! "$import_started_epoch" =~ ^[0-9]+([.][0-9]+)?$ ]]; then
+  printf 'Fehler: Datenbank lieferte keinen gültigen Importzeitpunkt.\n' >&2
+  exit 1
+fi
+
+for pbf_path in "${pbf_paths[@]}"; do
+  printf 'Importiere Marine-Features aus %s\n' "$(basename "$pbf_path")"
+  DATABASE_URL="$database_url" /workspace/scripts/import-geofabrik.sh "$pbf_path"
+done
+
+fresh_counts="$(
+  psql "$database_url" \
+    --no-psqlrc \
+    --tuples-only \
+    --no-align \
+    --field-separator '|' \
+    --set ON_ERROR_STOP=1 \
+    --command "
+      SELECT
+        count(*)::bigint,
+        count(*) FILTER (WHERE layer = 'harbours')::bigint,
+        count(*) FILTER (WHERE layer = 'locks')::bigint,
+        count(*) FILTER (WHERE layer = 'bridges')::bigint
+      FROM marine_features
+      WHERE source = 'osm'
+        AND updated_at >= to_timestamp($import_started_epoch);
+    "
+)"
+fresh_counts="${fresh_counts//[[:space:]]/}"
+IFS='|' read -r fresh_total fresh_harbours fresh_locks fresh_bridges <<<"$fresh_counts"
+for count in "$fresh_total" "$fresh_harbours" "$fresh_locks" "$fresh_bridges"; do
+  if [[ ! "$count" =~ ^[1-9][0-9]*$ ]]; then
+    printf 'Fehler: Der neue OSM-Import enthält keine vollständigen Hafen-/Schleusen-/Brückendaten.\n' >&2
+    exit 1
+  fi
+done
+
+psql "$database_url" \
+  --no-psqlrc \
+  --set ON_ERROR_STOP=1 \
+  --command "
+    BEGIN;
+    DELETE FROM marine_features
+    WHERE source = 'osm'
+      AND updated_at < to_timestamp($import_started_epoch);
+    DELETE FROM marine_fairway_edges
+    WHERE source = 'osm'
+      AND updated_at < to_timestamp($import_started_epoch);
+    COMMIT;
+    ANALYZE marine_features;
+    ANALYZE marine_fairway_edges;
+  "
+
+final_counts="$(
+  psql "$database_url" \
+    --no-psqlrc \
+    --tuples-only \
+    --no-align \
+    --field-separator '|' \
+    --set ON_ERROR_STOP=1 \
+    --command "
+      SELECT
+        count(*)::bigint,
+        count(*) FILTER (WHERE layer = 'harbours')::bigint,
+        count(*) FILTER (WHERE layer = 'locks')::bigint,
+        count(*) FILTER (WHERE layer = 'bridges')::bigint
+      FROM marine_features
+      WHERE source = 'osm';
+    "
+)"
+final_counts="${final_counts//[[:space:]]/}"
+IFS='|' read -r total_osm_features harbour_count lock_count bridge_count <<<"$final_counts"
+
+temporary_marker="$(mktemp "$(dirname "$marker_file")/.marine-features-ready.XXXXXX")"
+{
+  printf 'format_version=1\n'
+  printf 'generated_at=%s\n' "$(date --utc +%Y-%m-%dT%H:%M:%SZ)"
+  printf 'source=germany+netherlands\n'
+  printf 'germany_md5=%s\n' "${source_checksums[germany]}"
+  printf 'netherlands_md5=%s\n' "${source_checksums[netherlands]}"
+  printf 'total_osm_features=%s\n' "$total_osm_features"
+  printf 'harbour_count=%s\n' "$harbour_count"
+  printf 'lock_count=%s\n' "$lock_count"
+  printf 'bridge_count=%s\n' "$bridge_count"
+} >"$temporary_marker"
+chmod 0644 "$temporary_marker"
+mv -f "$temporary_marker" "$marker_file"
+
+printf 'Marine-Features sind bereit: %s OSM-Objekte (%s Häfen, %s Schleusen, %s Brücken)\n' \
+  "$total_osm_features" "$harbour_count" "$lock_count" "$bridge_count"