#!/usr/bin/env bash set -euo pipefail OUT_DIR="${WATERMAPS_GEOFABRIK_DIR:-${SEA_COMPASS_GEOFABRIK_DIR:-data/geofabrik}}" GERMANY_BASE_URL="${WATERMAPS_GEOFABRIK_GERMANY_BASE_URL:-${SEA_COMPASS_GEOFABRIK_GERMANY_BASE_URL:-https://download.geofabrik.de/europe/germany}}" EUROPE_BASE_URL="${WATERMAPS_GEOFABRIK_EUROPE_BASE_URL:-${SEA_COMPASS_GEOFABRIK_EUROPE_BASE_URL:-https://download.geofabrik.de/europe}}" DEFAULT_REGIONS=( germany netherlands ) regions=("$@") if [[ "${#regions[@]}" -eq 0 ]]; then regions=("${DEFAULT_REGIONS[@]}") fi mkdir -p "$OUT_DIR" for region in "${regions[@]}"; do file_name="${region}-latest.osm.pbf" case "$region" in germany|netherlands) url="${EUROPE_BASE_URL}/${file_name}" ;; *) url="${GERMANY_BASE_URL}/${file_name}" ;; esac target="${OUT_DIR}/${file_name}" checksum_target="${target}.md5" checksum_download="${checksum_target}.part" download_part="${target}.part" download_part_checksum="${download_part}.expected-md5" # Resolve the `latest` PBF redirect first and fetch the checksum belonging to # that exact dated snapshot. This prevents transparent download proxies from # combining a fresh PBF redirect with a stale `latest` checksum. resolved_url="$( curl \ --fail \ --head \ --location \ --silent \ --show-error \ --retry 5 \ --retry-all-errors \ --retry-delay 5 \ --output /dev/null \ --write-out '%{url_effective}' \ "$url" )" resolved_url="${resolved_url%%\?*}" resolved_file_name="${resolved_url##*/}" if [[ "$resolved_file_name" =~ ^[a-z0-9-]+-[0-9]{6}\.osm\.pbf$ ]]; then download_url="$resolved_url" else download_url="$url" fi checksum_url="${download_url}.md5" echo "Downloading $checksum_url" curl \ --fail \ --location \ --silent \ --show-error \ --retry 5 \ --retry-all-errors \ --retry-delay 5 \ --output "$checksum_download" \ "$checksum_url" expected_checksum="$(awk 'NR == 1 { print tolower($1) }' "$checksum_download")" if [[ ! "$expected_checksum" =~ ^[0-9a-f]{32}$ ]]; then echo "Invalid checksum response for $url" >&2 exit 1 fi checksum_file_name="$(awk 'NR == 1 { name = $2; sub(/^\*/, "", name); print name }' "$checksum_download")" if [[ "$checksum_file_name" =~ ^[a-z0-9-]+-[0-9]{6}\.osm\.pbf$ ]] && [[ "$checksum_file_name" != "${download_url##*/}" ]]; then echo "Checksum file does not match resolved snapshot: $checksum_file_name" >&2 exit 1 fi if [[ -f "$target" ]] && [[ "$(md5sum "$target" | awk '{ print $1 }')" == "$expected_checksum" ]]; then mv -f "$checksum_download" "$checksum_target" rm -f "$download_part" "$download_part_checksum" echo "$(basename "$target"): already current" continue fi previous_part_checksum="" if [[ -f "$download_part_checksum" ]]; then previous_part_checksum="$(awk 'NR == 1 { print tolower($1) }' "$download_part_checksum")" fi if [[ "$previous_part_checksum" != "$expected_checksum" ]]; then rm -f "$download_part" fi printf '%s\n' "$expected_checksum" >"$download_part_checksum" part_is_complete=false if [[ -f "$download_part" ]] && [[ "$(md5sum "$download_part" | awk '{ print $1 }')" == "$expected_checksum" ]]; then part_is_complete=true fi if [[ "$part_is_complete" == false ]]; then echo "Downloading $download_url" if ! curl \ --fail \ --location \ --silent \ --show-error \ --continue-at - \ --retry 5 \ --retry-all-errors \ --retry-delay 5 \ --output "$download_part" \ "$download_url"; then echo "Resuming failed; retrying $download_url from the beginning" >&2 rm -f "$download_part" curl \ --fail \ --location \ --silent \ --show-error \ --retry 5 \ --retry-all-errors \ --retry-delay 5 \ --output "$download_part" \ "$download_url" fi fi actual_checksum="$(md5sum "$download_part" | awk '{ print $1 }')" if [[ "$actual_checksum" != "$expected_checksum" ]]; then rm -f "$download_part" "$download_part_checksum" echo "Checksum verification failed for $url; previous snapshot retained" >&2 exit 1 fi mv -f "$download_part" "$target" mv -f "$checksum_download" "$checksum_target" rm -f "$download_part_checksum" echo "$(basename "$target"): OK" done