#!/usr/bin/env bash set -euo pipefail OUT_DIR="${WATERMAPS_GEOFABRIK_DIR:-${SEA_COMPASS_GEOFABRIK_DIR:-data/geofabrik}}" GERMANY_BASE_URL="${WATERMAPS_GEOFABRIK_GERMANY_BASE_URL:-${SEA_COMPASS_GEOFABRIK_GERMANY_BASE_URL:-https://download.geofabrik.de/europe/germany}}" EUROPE_BASE_URL="${WATERMAPS_GEOFABRIK_EUROPE_BASE_URL:-${SEA_COMPASS_GEOFABRIK_EUROPE_BASE_URL:-https://download.geofabrik.de/europe}}" DEFAULT_REGIONS=( germany netherlands ) regions=("$@") if [[ "${#regions[@]}" -eq 0 ]]; then regions=("${DEFAULT_REGIONS[@]}") fi mkdir -p "$OUT_DIR" for region in "${regions[@]}"; do if [[ ! "$region" =~ ^[a-z0-9-]+$ ]]; then echo "Invalid Geofabrik region: $region" >&2 exit 1 fi file_name="${region}-latest.osm.pbf" case "$region" in germany|netherlands) url="${EUROPE_BASE_URL}/${file_name}" ;; *) url="${GERMANY_BASE_URL}/${file_name}" ;; esac target="${OUT_DIR}/${file_name}" checksum_target="${target}.md5" checksum_download="${checksum_target}.part" download_part="${target}.part" download_part_checksum="${download_part}.expected-md5" # Resolve the effective `latest` endpoint first and fetch its checksum from # the same mirror. Some Geofabrik mirrors keep `latest` in the effective URL # while the checksum names the dated snapshot; both forms remain bound by # the checksum verification before the active PBF is replaced. resolved_url="$( curl \ --fail \ --head \ --location \ --silent \ --show-error \ --retry 5 \ --retry-all-errors \ --retry-delay 5 \ --output /dev/null \ --write-out '%{url_effective}' \ "$url" )" resolved_url="${resolved_url%%\?*}" resolved_file_name="${resolved_url##*/}" if [[ "$resolved_file_name" == "$file_name" || "$resolved_file_name" =~ ^${region}-[0-9]{6}\.osm\.pbf$ ]]; then download_url="$resolved_url" else download_url="$url" fi checksum_url="${download_url}.md5" echo "Downloading $checksum_url" curl \ --fail \ --location \ --silent \ --show-error \ --retry 5 \ --retry-all-errors \ --retry-delay 5 \ --output "$checksum_download" \ "$checksum_url" expected_checksum="$(awk 'NR == 1 { print tolower($1) }' "$checksum_download")" if [[ ! "$expected_checksum" =~ ^[0-9a-f]{32}$ ]]; then echo "Invalid checksum response for $url" >&2 exit 1 fi checksum_file_name="$(awk 'NR == 1 { name = $2; sub(/^\*/, "", name); print name }' "$checksum_download")" download_file_name="${download_url##*/}" if [[ "$download_file_name" =~ ^${region}-[0-9]{6}\.osm\.pbf$ ]]; then if [[ "$checksum_file_name" != "$download_file_name" ]]; then echo "Checksum file does not match resolved snapshot: $checksum_file_name" >&2 exit 1 fi elif [[ "$download_file_name" == "$file_name" ]] && [[ "$checksum_file_name" != "$file_name" && ! "$checksum_file_name" =~ ^${region}-[0-9]{6}\.osm\.pbf$ ]]; then echo "Checksum does not identify requested region: $checksum_file_name" >&2 exit 1 fi if [[ -f "$target" ]] && [[ "$(md5sum "$target" | awk '{ print $1 }')" == "$expected_checksum" ]]; then mv -f "$checksum_download" "$checksum_target" rm -f "$download_part" "$download_part_checksum" echo "$(basename "$target"): already current" continue fi previous_part_checksum="" if [[ -f "$download_part_checksum" ]]; then previous_part_checksum="$(awk 'NR == 1 { print tolower($1) }' "$download_part_checksum")" fi if [[ "$previous_part_checksum" != "$expected_checksum" ]]; then rm -f "$download_part" fi printf '%s\n' "$expected_checksum" >"$download_part_checksum" part_is_complete=false if [[ -f "$download_part" ]] && [[ "$(md5sum "$download_part" | awk '{ print $1 }')" == "$expected_checksum" ]]; then part_is_complete=true fi if [[ "$part_is_complete" == false ]]; then echo "Downloading $download_url" if ! curl \ --fail \ --location \ --silent \ --show-error \ --continue-at - \ --retry 5 \ --retry-all-errors \ --retry-delay 5 \ --output "$download_part" \ "$download_url"; then echo "Resuming failed; retrying $download_url from the beginning" >&2 rm -f "$download_part" curl \ --fail \ --location \ --silent \ --show-error \ --retry 5 \ --retry-all-errors \ --retry-delay 5 \ --output "$download_part" \ "$download_url" fi fi actual_checksum="$(md5sum "$download_part" | awk '{ print $1 }')" if [[ "$actual_checksum" != "$expected_checksum" ]]; then rm -f "$download_part" "$download_part_checksum" echo "Checksum verification failed for $url; previous snapshot retained" >&2 exit 1 fi mv -f "$download_part" "$target" mv -f "$checksum_download" "$checksum_target" rm -f "$download_part_checksum" echo "$(basename "$target"): OK" done