watermaps/scripts/download-geofabrik.sh

#!/usr/bin/env bash
set -euo pipefail

OUT_DIR="${WATERMAPS_GEOFABRIK_DIR:-${SEA_COMPASS_GEOFABRIK_DIR:-data/geofabrik}}"
GERMANY_BASE_URL="${WATERMAPS_GEOFABRIK_GERMANY_BASE_URL:-${SEA_COMPASS_GEOFABRIK_GERMANY_BASE_URL:-https://download.geofabrik.de/europe/germany}}"
EUROPE_BASE_URL="${WATERMAPS_GEOFABRIK_EUROPE_BASE_URL:-${SEA_COMPASS_GEOFABRIK_EUROPE_BASE_URL:-https://download.geofabrik.de/europe}}"

DEFAULT_REGIONS=(
  germany
  netherlands
)

regions=("$@")
if [[ "${#regions[@]}" -eq 0 ]]; then
  regions=("${DEFAULT_REGIONS[@]}")
fi

mkdir -p "$OUT_DIR"

for region in "${regions[@]}"; do
  file_name="${region}-latest.osm.pbf"
  case "$region" in
    germany|netherlands)
      url="${EUROPE_BASE_URL}/${file_name}"
      ;;
    *)
      url="${GERMANY_BASE_URL}/${file_name}"
      ;;
  esac
  target="${OUT_DIR}/${file_name}"
  checksum_target="${target}.md5"
  checksum_download="${checksum_target}.part"
  download_part="${target}.part"
  download_part_checksum="${download_part}.expected-md5"

  # Resolve the `latest` PBF redirect first and fetch the checksum belonging to
  # that exact dated snapshot. This prevents transparent download proxies from
  # combining a fresh PBF redirect with a stale `latest` checksum.
  resolved_url="$(
    curl \
      --fail \
      --head \
      --location \
      --silent \
      --show-error \
      --retry 5 \
      --retry-all-errors \
      --retry-delay 5 \
      --output /dev/null \
      --write-out '%{url_effective}' \
      "$url"
  )"
  resolved_url="${resolved_url%%\?*}"
  resolved_file_name="${resolved_url##*/}"
  if [[ "$resolved_file_name" =~ ^[a-z0-9-]+-[0-9]{6}\.osm\.pbf$ ]]; then
    download_url="$resolved_url"
  else
    download_url="$url"
  fi
  checksum_url="${download_url}.md5"

  echo "Downloading $checksum_url"
  curl \
    --fail \
    --location \
    --silent \
    --show-error \
    --retry 5 \
    --retry-all-errors \
    --retry-delay 5 \
    --output "$checksum_download" \
    "$checksum_url"

  expected_checksum="$(awk 'NR == 1 { print tolower($1) }' "$checksum_download")"
  if [[ ! "$expected_checksum" =~ ^[0-9a-f]{32}$ ]]; then
    echo "Invalid checksum response for $url" >&2
    exit 1
  fi

  checksum_file_name="$(awk 'NR == 1 { name = $2; sub(/^\*/, "", name); print name }' "$checksum_download")"
  if [[ "$checksum_file_name" =~ ^[a-z0-9-]+-[0-9]{6}\.osm\.pbf$ ]] &&
    [[ "$checksum_file_name" != "${download_url##*/}" ]]; then
    echo "Checksum file does not match resolved snapshot: $checksum_file_name" >&2
    exit 1
  fi

  if [[ -f "$target" ]] && [[ "$(md5sum "$target" | awk '{ print $1 }')" == "$expected_checksum" ]]; then
    mv -f "$checksum_download" "$checksum_target"
    rm -f "$download_part" "$download_part_checksum"
    echo "$(basename "$target"): already current"
    continue
  fi

  previous_part_checksum=""
  if [[ -f "$download_part_checksum" ]]; then
    previous_part_checksum="$(awk 'NR == 1 { print tolower($1) }' "$download_part_checksum")"
  fi
  if [[ "$previous_part_checksum" != "$expected_checksum" ]]; then
    rm -f "$download_part"
  fi
  printf '%s\n' "$expected_checksum" >"$download_part_checksum"

  part_is_complete=false
  if [[ -f "$download_part" ]] && [[ "$(md5sum "$download_part" | awk '{ print $1 }')" == "$expected_checksum" ]]; then
    part_is_complete=true
  fi

  if [[ "$part_is_complete" == false ]]; then
    echo "Downloading $download_url"
    if ! curl \
      --fail \
      --location \
      --silent \
      --show-error \
      --continue-at - \
      --retry 5 \
      --retry-all-errors \
      --retry-delay 5 \
      --output "$download_part" \
      "$download_url"; then
      echo "Resuming failed; retrying $download_url from the beginning" >&2
      rm -f "$download_part"
      curl \
        --fail \
        --location \
        --silent \
        --show-error \
        --retry 5 \
        --retry-all-errors \
        --retry-delay 5 \
        --output "$download_part" \
        "$download_url"
    fi
  fi

  actual_checksum="$(md5sum "$download_part" | awk '{ print $1 }')"
  if [[ "$actual_checksum" != "$expected_checksum" ]]; then
    rm -f "$download_part" "$download_part_checksum"
    echo "Checksum verification failed for $url; previous snapshot retained" >&2
    exit 1
  fi

  mv -f "$download_part" "$target"
  mv -f "$checksum_download" "$checksum_target"
  rm -f "$download_part_checksum"
  echo "$(basename "$target"): OK"
done