feat: add Docker/OpenTofu deployment and DE/NL routing
This commit is contained in:
BuTzZ
Executable
+392
@@ -0,0 +1,392 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build a compact, file-backed fairway dataset from Geofabrik OSM PBFs.
|
||||
|
||||
The importer intentionally uses two streaming passes over every source. The
|
||||
first pass retains only routable marine/inland ways and their node IDs; the
|
||||
second retains only coordinates referenced by those ways. Coordinates and
|
||||
ways shared by neighbouring extracts are merged by their globally unique OSM
|
||||
IDs. This avoids loading complete OSM node tables into memory and does not
|
||||
require PostGIS or Docker.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
import osmium
|
||||
except ImportError as error:
|
||||
raise SystemExit(
|
||||
"pyosmium fehlt. Installiere es mit "
|
||||
"`python3 -m pip install --target .tools/python 'osmium>=4,<5'` "
|
||||
"und starte mit `PYTHONPATH=.tools/python`."
|
||||
) from error
|
||||
|
||||
|
||||
KEPT_TAGS = {
|
||||
"access",
|
||||
"boat",
|
||||
"construction",
|
||||
"depth",
|
||||
"disused",
|
||||
"maxdraft",
|
||||
"maxdraught",
|
||||
"maxheight",
|
||||
"maxheight:physical",
|
||||
"maxwidth",
|
||||
"maxwidth:physical",
|
||||
"min_depth",
|
||||
"motor_vehicle",
|
||||
"motorboat",
|
||||
"name",
|
||||
"oneway",
|
||||
"proposed",
|
||||
"ref",
|
||||
"route",
|
||||
"seamark:bridge:clearance_height",
|
||||
"seamark:bridge:clearance_height_safe",
|
||||
"seamark:fairway:minimum_depth",
|
||||
"seamark:lock:chamber_width",
|
||||
"seamark:navigation_line:minimum_depth",
|
||||
"seamark:recommended_track:minimum_depth",
|
||||
"seamark:restriction:max_draught",
|
||||
"seamark:type",
|
||||
"ship",
|
||||
"waterway",
|
||||
}
|
||||
GENERATOR_VERSION = 2
|
||||
|
||||
|
||||
def is_routable(tags: dict[str, str]) -> bool:
|
||||
if (
|
||||
tags.get("access") in {"no", "private"}
|
||||
or tags.get("boat") == "no"
|
||||
or tags.get("ship") == "no"
|
||||
or tags.get("motorboat") == "no"
|
||||
or tags.get("disused") == "yes"
|
||||
or "construction" in tags
|
||||
or "proposed" in tags
|
||||
):
|
||||
return False
|
||||
|
||||
seamark_type = tags.get("seamark:type")
|
||||
if seamark_type in {"navigation_line", "recommended_track"}:
|
||||
return True
|
||||
if tags.get("waterway") in {"fairway", "canal"}:
|
||||
return True
|
||||
if tags.get("waterway") == "river" and any(
|
||||
tags.get(key) in {"yes", "designated", "permissive"}
|
||||
for key in ("boat", "ship", "motorboat")
|
||||
):
|
||||
return True
|
||||
if (
|
||||
tags.get("route") == "ferry"
|
||||
and tags.get("ship") != "no"
|
||||
and tags.get("motor_vehicle") != "no"
|
||||
):
|
||||
return True
|
||||
return seamark_type == "fairway"
|
||||
|
||||
|
||||
class WayCollector(osmium.SimpleHandler):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.ways: list[dict[str, Any]] = []
|
||||
self.node_ids: set[int] = set()
|
||||
|
||||
def way(self, way: Any) -> None:
|
||||
tags = {tag.k: tag.v for tag in way.tags}
|
||||
if not is_routable(tags):
|
||||
return
|
||||
|
||||
node_ids = [node.ref for node in way.nodes]
|
||||
if len(node_ids) < 2:
|
||||
return
|
||||
|
||||
self.node_ids.update(node_ids)
|
||||
self.ways.append(
|
||||
{
|
||||
"id": str(way.id),
|
||||
"version": int(getattr(way, "version", 0) or 0),
|
||||
"nodes": node_ids,
|
||||
"tags": {key: value for key, value in tags.items() if key in KEPT_TAGS},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class NodeCollector(osmium.SimpleHandler):
|
||||
def __init__(self, wanted: set[int]) -> None:
|
||||
super().__init__()
|
||||
self.wanted = wanted
|
||||
self.coordinates: dict[int, tuple[int, float, float]] = {}
|
||||
|
||||
def node(self, node: Any) -> None:
|
||||
if node.id not in self.wanted or not node.location.valid():
|
||||
return
|
||||
candidate = (
|
||||
int(getattr(node, "version", 0) or 0),
|
||||
node.location.lat,
|
||||
node.location.lon,
|
||||
)
|
||||
current = self.coordinates.get(node.id)
|
||||
if current is None or candidate > current:
|
||||
self.coordinates[node.id] = candidate
|
||||
|
||||
|
||||
def way_bbox(coordinates: list[list[float]]) -> list[float]:
|
||||
lats = [coordinate[0] for coordinate in coordinates]
|
||||
lons = [coordinate[1] for coordinate in coordinates]
|
||||
return [min(lons), min(lats), max(lons), max(lats)]
|
||||
|
||||
|
||||
def source_region(path: Path) -> str:
|
||||
name = path.name.lower()
|
||||
for suffix in ("-latest.osm.pbf", ".osm.pbf", ".pbf"):
|
||||
if name.endswith(suffix):
|
||||
name = name[: -len(suffix)]
|
||||
break
|
||||
region = re.sub(r"[^a-z0-9]+", "-", name).strip("-")
|
||||
return region or "unknown"
|
||||
|
||||
|
||||
def adjacent_md5(path: Path) -> str | None:
|
||||
checksum_path = Path(f"{path}.md5")
|
||||
if not checksum_path.is_file():
|
||||
return None
|
||||
fields = checksum_path.read_text(encoding="utf-8").split(maxsplit=1)
|
||||
if not fields:
|
||||
return None
|
||||
checksum = fields[0].lower()
|
||||
return checksum if re.fullmatch(r"[0-9a-f]{32}", checksum) else None
|
||||
|
||||
|
||||
def candidate_rank(candidate: dict[str, Any]) -> tuple[int, int, str]:
|
||||
"""Return a deterministic preference for duplicate versions of an OSM way."""
|
||||
|
||||
fingerprint = hashlib.sha256(
|
||||
json.dumps(
|
||||
{
|
||||
"nodes": candidate["nodes"],
|
||||
"tags": candidate["tags"],
|
||||
"coordinates": candidate["coordinates"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
).hexdigest()
|
||||
return candidate["version"], len(candidate["nodes"]), fingerprint
|
||||
|
||||
|
||||
def build_document(pbf_paths: list[Path]) -> dict[str, Any]:
|
||||
sources: list[dict[str, Any]] = []
|
||||
wanted_node_ids: set[int] = set()
|
||||
|
||||
for index, path in enumerate(pbf_paths, start=1):
|
||||
print(
|
||||
f"Pass 1/2 [{index}/{len(pbf_paths)}]: routbare Wege aus {path} lesen",
|
||||
flush=True,
|
||||
)
|
||||
collector = WayCollector()
|
||||
processor = osmium.FileProcessor(
|
||||
str(path), osmium.osm.WAY
|
||||
).with_filter(osmium.filter.KeyFilter("seamark:type", "waterway", "route"))
|
||||
for way in processor:
|
||||
collector.way(way)
|
||||
wanted_node_ids.update(collector.node_ids)
|
||||
stat = path.stat()
|
||||
sources.append(
|
||||
{
|
||||
"path": path,
|
||||
"region": source_region(path),
|
||||
"collector": collector,
|
||||
"metadata": {
|
||||
"region": source_region(path),
|
||||
"file": path.name,
|
||||
"sizeBytes": stat.st_size,
|
||||
"modifiedAt": datetime.fromtimestamp(
|
||||
stat.st_mtime, timezone.utc
|
||||
).isoformat(),
|
||||
"checksumMd5": adjacent_md5(path),
|
||||
"routableWaysFound": len(collector.ways),
|
||||
"referencedNodes": len(collector.node_ids),
|
||||
"nodeCoordinatesFound": 0,
|
||||
"exportedCandidateWays": 0,
|
||||
"incompleteWaysSkipped": 0,
|
||||
"closedFairwaysSkipped": 0,
|
||||
},
|
||||
}
|
||||
)
|
||||
print(
|
||||
f"{len(collector.ways)} Wege mit "
|
||||
f"{len(collector.node_ids)} referenzierten Knoten gefunden",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
global_coordinates: dict[int, tuple[int, float, float]] = {}
|
||||
coordinate_conflicts = 0
|
||||
for index, source in enumerate(sources, start=1):
|
||||
print(
|
||||
f"Pass 2/2 [{index}/{len(sources)}]: benötigte Knotenkoordinaten "
|
||||
f"aus {source['path']} lesen",
|
||||
flush=True,
|
||||
)
|
||||
nodes = NodeCollector(wanted_node_ids)
|
||||
if wanted_node_ids:
|
||||
processor = osmium.FileProcessor(
|
||||
str(source["path"]), osmium.osm.NODE
|
||||
).with_filter(osmium.filter.IdFilter(wanted_node_ids))
|
||||
for node in processor:
|
||||
nodes.node(node)
|
||||
|
||||
source["metadata"]["nodeCoordinatesFound"] = len(nodes.coordinates)
|
||||
for node_id, candidate in nodes.coordinates.items():
|
||||
current = global_coordinates.get(node_id)
|
||||
if current is not None and current[1:] != candidate[1:]:
|
||||
coordinate_conflicts += 1
|
||||
if current is None or candidate > current:
|
||||
global_coordinates[node_id] = candidate
|
||||
|
||||
candidates_by_id: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for source in sources:
|
||||
for way in source["collector"].ways:
|
||||
if any(node_id not in global_coordinates for node_id in way["nodes"]):
|
||||
source["metadata"]["incompleteWaysSkipped"] += 1
|
||||
continue
|
||||
coordinates = [
|
||||
[global_coordinates[node_id][1], global_coordinates[node_id][2]]
|
||||
for node_id in way["nodes"]
|
||||
]
|
||||
if (
|
||||
way["tags"].get("seamark:type") == "fairway"
|
||||
and coordinates[0] == coordinates[-1]
|
||||
):
|
||||
source["metadata"]["closedFairwaysSkipped"] += 1
|
||||
continue
|
||||
|
||||
source["metadata"]["exportedCandidateWays"] += 1
|
||||
candidates_by_id[way["id"]].append(
|
||||
{
|
||||
**way,
|
||||
"coordinates": coordinates,
|
||||
"region": source["region"],
|
||||
"sourceFile": source["path"].name,
|
||||
}
|
||||
)
|
||||
|
||||
exported: list[dict[str, Any]] = []
|
||||
duplicate_ways_merged = 0
|
||||
for way_id, candidates in candidates_by_id.items():
|
||||
winner = max(candidates, key=candidate_rank)
|
||||
duplicate_ways_merged += len(candidates) - 1
|
||||
exported.append(
|
||||
{
|
||||
"id": way_id,
|
||||
"osmVersion": winner["version"],
|
||||
"regions": sorted({candidate["region"] for candidate in candidates}),
|
||||
"sourceFiles": sorted(
|
||||
{candidate["sourceFile"] for candidate in candidates}
|
||||
),
|
||||
"bbox": way_bbox(winner["coordinates"]),
|
||||
"tags": winner["tags"],
|
||||
"coordinates": winner["coordinates"],
|
||||
}
|
||||
)
|
||||
exported.sort(key=lambda way: int(way["id"]))
|
||||
|
||||
source_metadata = [source["metadata"] for source in sources]
|
||||
regions = list(dict.fromkeys(source["region"] for source in sources))
|
||||
source_label = (
|
||||
pbf_paths[0].name if len(pbf_paths) == 1 else "+".join(regions)
|
||||
)
|
||||
modified_at = max(path.stat().st_mtime for path in pbf_paths)
|
||||
return {
|
||||
"version": 1,
|
||||
"generatorVersion": GENERATOR_VERSION,
|
||||
"source": source_label,
|
||||
"sources": source_metadata,
|
||||
"sourceSizeBytes": sum(path.stat().st_size for path in pbf_paths),
|
||||
"sourceModifiedAt": datetime.fromtimestamp(
|
||||
modified_at, timezone.utc
|
||||
).isoformat(),
|
||||
"generatedAt": datetime.now(timezone.utc).isoformat(),
|
||||
"incompleteWaysSkipped": sum(
|
||||
source["metadata"]["incompleteWaysSkipped"] for source in sources
|
||||
),
|
||||
"closedFairwaysSkipped": sum(
|
||||
source["metadata"]["closedFairwaysSkipped"] for source in sources
|
||||
),
|
||||
"duplicateWaysMerged": duplicate_ways_merged,
|
||||
"nodeCoordinateConflicts": coordinate_conflicts,
|
||||
"ways": exported,
|
||||
}
|
||||
|
||||
|
||||
def write_document(document: dict[str, Any], output: Path) -> None:
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary_path: Path | None = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(
|
||||
"w",
|
||||
encoding="utf-8",
|
||||
dir=output.parent,
|
||||
prefix=f".{output.name}.",
|
||||
suffix=".tmp",
|
||||
delete=False,
|
||||
) as temporary:
|
||||
json.dump(document, temporary, ensure_ascii=False, separators=(",", ":"))
|
||||
temporary.write("\n")
|
||||
temporary_path = Path(temporary.name)
|
||||
temporary_path.chmod(0o644)
|
||||
os.replace(temporary_path, output)
|
||||
except BaseException:
|
||||
if temporary_path is not None:
|
||||
temporary_path.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("pbf", type=Path, nargs="+")
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
default=Path("data/local/germany-netherlands-fairways.json"),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
pbf_paths: list[Path] = []
|
||||
seen_paths: set[Path] = set()
|
||||
for path in args.pbf:
|
||||
if not path.is_file():
|
||||
parser.error(f"PBF nicht gefunden: {path}")
|
||||
resolved = path.resolve()
|
||||
if resolved in seen_paths:
|
||||
parser.error(f"PBF doppelt angegeben: {path}")
|
||||
seen_paths.add(resolved)
|
||||
pbf_paths.append(path)
|
||||
|
||||
document = build_document(pbf_paths)
|
||||
write_document(document, args.output)
|
||||
print(
|
||||
f"{len(document['ways'])} Wege nach {args.output} geschrieben "
|
||||
f"({document['duplicateWaysMerged']} Duplikate zusammengeführt; "
|
||||
f"{document['incompleteWaysSkipped']} unvollständige Wege verworfen; "
|
||||
f"{args.output.stat().st_size / 1024 / 1024:.1f} MiB)",
|
||||
flush=True,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -29,23 +29,119 @@ for region in "${regions[@]}"; do
|
||||
esac
|
||||
target="${OUT_DIR}/${file_name}"
|
||||
checksum_target="${target}.md5"
|
||||
checksum_download="${checksum_target}.part"
|
||||
download_part="${target}.part"
|
||||
download_part_checksum="${download_part}.expected-md5"
|
||||
|
||||
echo "Downloading $url.md5"
|
||||
curl --fail --location --retry 5 --retry-delay 5 --output "$checksum_target" "${url}.md5"
|
||||
# Resolve the `latest` PBF redirect first and fetch the checksum belonging to
|
||||
# that exact dated snapshot. This prevents transparent download proxies from
|
||||
# combining a fresh PBF redirect with a stale `latest` checksum.
|
||||
resolved_url="$(
|
||||
curl \
|
||||
--fail \
|
||||
--head \
|
||||
--location \
|
||||
--silent \
|
||||
--show-error \
|
||||
--retry 5 \
|
||||
--retry-all-errors \
|
||||
--retry-delay 5 \
|
||||
--output /dev/null \
|
||||
--write-out '%{url_effective}' \
|
||||
"$url"
|
||||
)"
|
||||
resolved_url="${resolved_url%%\?*}"
|
||||
resolved_file_name="${resolved_url##*/}"
|
||||
if [[ "$resolved_file_name" =~ ^[a-z0-9-]+-[0-9]{6}\.osm\.pbf$ ]]; then
|
||||
download_url="$resolved_url"
|
||||
else
|
||||
download_url="$url"
|
||||
fi
|
||||
checksum_url="${download_url}.md5"
|
||||
|
||||
echo "Downloading $checksum_url"
|
||||
curl \
|
||||
--fail \
|
||||
--location \
|
||||
--silent \
|
||||
--show-error \
|
||||
--retry 5 \
|
||||
--retry-all-errors \
|
||||
--retry-delay 5 \
|
||||
--output "$checksum_download" \
|
||||
"$checksum_url"
|
||||
|
||||
expected_checksum="$(awk 'NR == 1 { print tolower($1) }' "$checksum_download")"
|
||||
if [[ ! "$expected_checksum" =~ ^[0-9a-f]{32}$ ]]; then
|
||||
echo "Invalid checksum response for $url" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
checksum_file_name="$(awk 'NR == 1 { name = $2; sub(/^\*/, "", name); print name }' "$checksum_download")"
|
||||
if [[ "$checksum_file_name" =~ ^[a-z0-9-]+-[0-9]{6}\.osm\.pbf$ ]] &&
|
||||
[[ "$checksum_file_name" != "${download_url##*/}" ]]; then
|
||||
echo "Checksum file does not match resolved snapshot: $checksum_file_name" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
expected_checksum="$(awk 'NR == 1 { print $1 }' "$checksum_target")"
|
||||
if [[ -f "$target" ]] && [[ "$(md5sum "$target" | awk '{ print $1 }')" == "$expected_checksum" ]]; then
|
||||
mv -f "$checksum_download" "$checksum_target"
|
||||
rm -f "$download_part" "$download_part_checksum"
|
||||
echo "$(basename "$target"): already current"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Downloading $url"
|
||||
curl --fail --location --continue-at - --retry 5 --retry-delay 5 --output "$target" "$url"
|
||||
previous_part_checksum=""
|
||||
if [[ -f "$download_part_checksum" ]]; then
|
||||
previous_part_checksum="$(awk 'NR == 1 { print tolower($1) }' "$download_part_checksum")"
|
||||
fi
|
||||
if [[ "$previous_part_checksum" != "$expected_checksum" ]]; then
|
||||
rm -f "$download_part"
|
||||
fi
|
||||
printf '%s\n' "$expected_checksum" >"$download_part_checksum"
|
||||
|
||||
actual_checksum="$(md5sum "$target" | awk '{ print $1 }')"
|
||||
if [[ -z "$expected_checksum" || "$actual_checksum" != "$expected_checksum" ]]; then
|
||||
echo "Checksum verification failed for $target" >&2
|
||||
part_is_complete=false
|
||||
if [[ -f "$download_part" ]] && [[ "$(md5sum "$download_part" | awk '{ print $1 }')" == "$expected_checksum" ]]; then
|
||||
part_is_complete=true
|
||||
fi
|
||||
|
||||
if [[ "$part_is_complete" == false ]]; then
|
||||
echo "Downloading $download_url"
|
||||
if ! curl \
|
||||
--fail \
|
||||
--location \
|
||||
--silent \
|
||||
--show-error \
|
||||
--continue-at - \
|
||||
--retry 5 \
|
||||
--retry-all-errors \
|
||||
--retry-delay 5 \
|
||||
--output "$download_part" \
|
||||
"$download_url"; then
|
||||
echo "Resuming failed; retrying $download_url from the beginning" >&2
|
||||
rm -f "$download_part"
|
||||
curl \
|
||||
--fail \
|
||||
--location \
|
||||
--silent \
|
||||
--show-error \
|
||||
--retry 5 \
|
||||
--retry-all-errors \
|
||||
--retry-delay 5 \
|
||||
--output "$download_part" \
|
||||
"$download_url"
|
||||
fi
|
||||
fi
|
||||
|
||||
actual_checksum="$(md5sum "$download_part" | awk '{ print $1 }')"
|
||||
if [[ "$actual_checksum" != "$expected_checksum" ]]; then
|
||||
rm -f "$download_part" "$download_part_checksum"
|
||||
echo "Checksum verification failed for $url; previous snapshot retained" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mv -f "$download_part" "$target"
|
||||
mv -f "$checksum_download" "$checksum_target"
|
||||
rm -f "$download_part_checksum"
|
||||
echo "$(basename "$target"): OK"
|
||||
done
|
||||
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
echo "Hinweis: setup-local-germany.sh ist veraltet; richte Deutschland und Niederlande gemeinsam ein." >&2
|
||||
exec "$ROOT_DIR/scripts/setup-local-routing.sh" "$@"
|
||||
Executable
+95
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
GEOFABRIK_DIR="${WATERMAPS_GEOFABRIK_DIR:-${SEA_COMPASS_GEOFABRIK_DIR:-data/geofabrik}}"
|
||||
PYTHON_TARGET="$ROOT_DIR/.tools/python"
|
||||
OUTPUT_PATH="${WATERMAPS_LOCAL_FAIRWAYS_PATH:-$ROOT_DIR/data/local/germany-netherlands-fairways.json}"
|
||||
|
||||
if [[ "$GEOFABRIK_DIR" != /* ]]; then
|
||||
GEOFABRIK_DIR="$ROOT_DIR/$GEOFABRIK_DIR"
|
||||
fi
|
||||
if [[ "$OUTPUT_PATH" != /* ]]; then
|
||||
OUTPUT_PATH="$ROOT_DIR/$OUTPUT_PATH"
|
||||
fi
|
||||
|
||||
PBF_PATHS=(
|
||||
"$GEOFABRIK_DIR/germany-latest.osm.pbf"
|
||||
"$GEOFABRIK_DIR/netherlands-latest.osm.pbf"
|
||||
)
|
||||
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
# This validates existing snapshots and only replaces them after a complete,
|
||||
# checksum-verified download.
|
||||
WATERMAPS_GEOFABRIK_DIR="$GEOFABRIK_DIR" \
|
||||
./scripts/download-geofabrik.sh germany netherlands
|
||||
|
||||
# A successful previous build can be reused when both downloaded snapshots
|
||||
# still match the checksums recorded in its per-source metadata.
|
||||
if python3 - "$OUTPUT_PATH" "${PBF_PATHS[@]}" <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
|
||||
output = Path(sys.argv[1])
|
||||
pbf_paths = [Path(value) for value in sys.argv[2:]]
|
||||
try:
|
||||
document = json.loads(output.read_text(encoding="utf-8"))
|
||||
sources = {
|
||||
source["file"]: source
|
||||
for source in document["sources"]
|
||||
if isinstance(source, dict) and isinstance(source.get("file"), str)
|
||||
}
|
||||
if (
|
||||
document.get("version") != 1
|
||||
or document.get("generatorVersion") != 2
|
||||
or not isinstance(document.get("ways"), list)
|
||||
):
|
||||
raise ValueError("unsupported index format")
|
||||
for pbf_path in pbf_paths:
|
||||
checksum_text = Path(f"{pbf_path}.md5").read_text(encoding="utf-8")
|
||||
checksum = checksum_text.split(maxsplit=1)[0].lower()
|
||||
metadata = sources[pbf_path.name]
|
||||
if not re.fullmatch(r"[0-9a-f]{32}", checksum):
|
||||
raise ValueError("invalid checksum")
|
||||
if metadata.get("checksumMd5") != checksum:
|
||||
raise ValueError("snapshot changed")
|
||||
if metadata.get("sizeBytes") != pbf_path.stat().st_size:
|
||||
raise ValueError("snapshot size changed")
|
||||
except (
|
||||
FileNotFoundError,
|
||||
IndexError,
|
||||
KeyError,
|
||||
TypeError,
|
||||
ValueError,
|
||||
json.JSONDecodeError,
|
||||
):
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
then
|
||||
echo "Lokaler Deutschland-/Niederlande-Routingindex ist bereits aktuell: $OUTPUT_PATH"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
mkdir -p "$PYTHON_TARGET"
|
||||
if ! PYTHONPATH="$PYTHON_TARGET" python3 - <<'PY'
|
||||
from importlib.metadata import version
|
||||
import osmium
|
||||
|
||||
raise SystemExit(0 if version("osmium") == "4.3.1" else 1)
|
||||
PY
|
||||
then
|
||||
python3 -m pip install \
|
||||
--disable-pip-version-check \
|
||||
--target "$PYTHON_TARGET" \
|
||||
--upgrade \
|
||||
"osmium==4.3.1"
|
||||
fi
|
||||
|
||||
PYTHONPATH="$PYTHON_TARGET" python3 scripts/build-local-fairways.py \
|
||||
"${PBF_PATHS[@]}" \
|
||||
--output "$OUTPUT_PATH"
|
||||
|
||||
echo "Lokale Deutschland-/Niederlande-Fahrrouten sind bereit: $OUTPUT_PATH"
|
||||
@@ -0,0 +1,194 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
try:
|
||||
import osmium
|
||||
except ImportError:
|
||||
osmium = None
|
||||
|
||||
|
||||
ROOT_DIR = Path(__file__).resolve().parents[2]
|
||||
BUILDER = ROOT_DIR / "scripts" / "build-local-fairways.py"
|
||||
|
||||
|
||||
def write_pbf(
|
||||
path: Path,
|
||||
nodes: list[tuple[int, float, float]],
|
||||
ways: list[tuple[int, int, list[int], dict[str, str]]],
|
||||
) -> None:
|
||||
assert osmium is not None
|
||||
with osmium.SimpleWriter(str(path)) as writer:
|
||||
for node_id, lon, lat in nodes:
|
||||
writer.add_node(
|
||||
osmium.osm.mutable.Node(
|
||||
id=node_id,
|
||||
version=1,
|
||||
location=(lon, lat),
|
||||
)
|
||||
)
|
||||
for way_id, version, node_ids, tags in ways:
|
||||
writer.add_way(
|
||||
osmium.osm.mutable.Way(
|
||||
id=way_id,
|
||||
version=version,
|
||||
nodes=node_ids,
|
||||
tags=tags,
|
||||
)
|
||||
)
|
||||
checksum = hashlib.md5(path.read_bytes()).hexdigest()
|
||||
Path(f"{path}.md5").write_text(
|
||||
f"{checksum} {path.name}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
@unittest.skipIf(osmium is None, "pyosmium/osmium is not installed")
|
||||
class BuildLocalFairwaysTest(unittest.TestCase):
|
||||
def test_merges_two_extracts_and_deduplicates_shared_osm_ways(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
work_dir = Path(temporary_directory)
|
||||
germany = work_dir / "germany-latest.osm.pbf"
|
||||
netherlands = work_dir / "netherlands-latest.osm.pbf"
|
||||
output = work_dir / "germany-netherlands-fairways.json"
|
||||
|
||||
write_pbf(
|
||||
germany,
|
||||
[
|
||||
(1, 7.0, 53.0),
|
||||
(2, 7.1, 53.1),
|
||||
(3, 7.2, 53.2),
|
||||
],
|
||||
[
|
||||
(100, 1, [1, 2], {"waterway": "canal", "name": "DE Kanal"}),
|
||||
(
|
||||
200,
|
||||
1,
|
||||
[2, 3],
|
||||
{"seamark:type": "navigation_line", "name": "Alte Linie"},
|
||||
),
|
||||
(300, 1, [1, 2, 1], {"seamark:type": "fairway"}),
|
||||
(500, 1, [1, 9], {"waterway": "canal", "name": "Grenzkanal"}),
|
||||
(600, 1, [1, 9999], {"waterway": "canal"}),
|
||||
],
|
||||
)
|
||||
write_pbf(
|
||||
netherlands,
|
||||
[
|
||||
(2, 7.1, 53.1),
|
||||
(3, 7.2, 53.2),
|
||||
(4, 7.3, 53.3),
|
||||
(9, 7.05, 53.05),
|
||||
],
|
||||
[
|
||||
(
|
||||
200,
|
||||
2,
|
||||
[2, 3, 4],
|
||||
{"seamark:type": "navigation_line", "name": "Nieuwe lijn"},
|
||||
),
|
||||
(400, 1, [3, 4], {"route": "ferry", "name": "Veerboot"}),
|
||||
],
|
||||
)
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(BUILDER),
|
||||
str(germany),
|
||||
str(netherlands),
|
||||
"--output",
|
||||
str(output),
|
||||
],
|
||||
cwd=ROOT_DIR,
|
||||
env={
|
||||
**os.environ,
|
||||
"PYTHONPATH": str(ROOT_DIR / ".tools" / "python"),
|
||||
},
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
document = json.loads(output.read_text(encoding="utf-8"))
|
||||
self.assertEqual(
|
||||
stat.S_IMODE(output.stat().st_mode),
|
||||
0o644,
|
||||
"the unprivileged application container must be able to read the index",
|
||||
)
|
||||
self.assertEqual(document["version"], 1)
|
||||
self.assertEqual(document["generatorVersion"], 2)
|
||||
self.assertEqual(document["source"], "germany+netherlands")
|
||||
self.assertEqual(document["duplicateWaysMerged"], 1)
|
||||
self.assertEqual(document["incompleteWaysSkipped"], 1)
|
||||
self.assertEqual(document["closedFairwaysSkipped"], 1)
|
||||
self.assertEqual([way["id"] for way in document["ways"]], ["100", "200", "400", "500"])
|
||||
|
||||
shared_way = next(way for way in document["ways"] if way["id"] == "200")
|
||||
self.assertEqual(shared_way["osmVersion"], 2)
|
||||
self.assertEqual(shared_way["regions"], ["germany", "netherlands"])
|
||||
self.assertEqual(
|
||||
shared_way["sourceFiles"],
|
||||
["germany-latest.osm.pbf", "netherlands-latest.osm.pbf"],
|
||||
)
|
||||
self.assertEqual(shared_way["tags"]["name"], "Nieuwe lijn")
|
||||
self.assertEqual(len(shared_way["coordinates"]), 3)
|
||||
|
||||
cross_border_way = next(
|
||||
way for way in document["ways"] if way["id"] == "500"
|
||||
)
|
||||
self.assertEqual(
|
||||
cross_border_way["coordinates"],
|
||||
[[53.0, 7.0], [53.05, 7.05]],
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
[source["region"] for source in document["sources"]],
|
||||
["germany", "netherlands"],
|
||||
)
|
||||
self.assertTrue(
|
||||
all(source["checksumMd5"] for source in document["sources"])
|
||||
)
|
||||
|
||||
def test_failed_source_read_keeps_existing_output(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
work_dir = Path(temporary_directory)
|
||||
broken_pbf = work_dir / "germany-latest.osm.pbf"
|
||||
output = work_dir / "germany-netherlands-fairways.json"
|
||||
broken_pbf.write_bytes(b"not an OSM PBF")
|
||||
previous_content = '{"version":1,"source":"previous","ways":[]}\n'
|
||||
output.write_text(previous_content, encoding="utf-8")
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(BUILDER),
|
||||
str(broken_pbf),
|
||||
"--output",
|
||||
str(output),
|
||||
],
|
||||
cwd=ROOT_DIR,
|
||||
env={
|
||||
**os.environ,
|
||||
"PYTHONPATH": str(ROOT_DIR / ".tools" / "python"),
|
||||
},
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual(output.read_text(encoding="utf-8"), previous_content)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user