Initial Watermaps import
This commit is contained in:
BuTzZ
Executable
+383
@@ -0,0 +1,383 @@
|
||||
#!/usr/bin/env node
|
||||
import { createReadStream } from "node:fs";
|
||||
import { createInterface } from "node:readline";
|
||||
import pg from "pg";
|
||||
import {
|
||||
featureName,
|
||||
isLockFeature,
|
||||
normalizedValue,
|
||||
sourceId
|
||||
} from "./osm-marine-classification.mjs";
|
||||
|
||||
const inputPath = process.argv[2];
|
||||
const databaseUrl = process.env.DATABASE_URL ?? "postgres://seacompass:seacompass@localhost:55432/seacompass";
|
||||
const importSource =
|
||||
process.env.WATERMAPS_IMPORT_SOURCE ??
|
||||
process.env.SEA_COMPASS_IMPORT_SOURCE ??
|
||||
"unknown-pbf";
|
||||
|
||||
if (!inputPath) {
|
||||
console.error("Usage: DATABASE_URL=postgres://... node scripts/load-osm-fairways.mjs /path/to/marine.geojsonseq");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const { Client } = pg;
|
||||
const client = new Client({ connectionString: databaseUrl });
|
||||
|
||||
const FEATURE_BATCH_SIZE = 500;
|
||||
const EDGE_BATCH_SIZE = 500;
|
||||
const featureBatch = [];
|
||||
const edgeBatch = [];
|
||||
let seenFeatures = 0;
|
||||
let importedFeatures = 0;
|
||||
let importedEdges = 0;
|
||||
|
||||
const truthy = new Set(["yes", "true", "1", "designated", "permissive"]);
|
||||
const falsy = new Set(["no", "false", "0", "private", "prohibited", "restricted"]);
|
||||
const harbourSeamarkTypes = new Set(["harbour", "harbour_basin", "marina"]);
|
||||
const inactiveWaterwayValues = new Set(["abandoned", "construction", "disused", "proposed"]);
|
||||
|
||||
function getString(value) {
|
||||
return typeof value === "string" ? value.trim() : "";
|
||||
}
|
||||
|
||||
function hasKey(properties, key) {
|
||||
return Object.prototype.hasOwnProperty.call(properties, key) && properties[key] != null;
|
||||
}
|
||||
|
||||
function hasActiveValue(value) {
|
||||
const normalized = normalizedValue(value);
|
||||
return Boolean(normalized) && !["no", "false", "0"].includes(normalized);
|
||||
}
|
||||
|
||||
function isHarbourFeature(properties) {
|
||||
const seamarkType = normalizedValue(properties["seamark:type"]);
|
||||
const waterway = normalizedValue(properties.waterway);
|
||||
|
||||
return (
|
||||
harbourSeamarkTypes.has(seamarkType) ||
|
||||
normalizedValue(properties.leisure) === "marina" ||
|
||||
hasActiveValue(properties.harbour) ||
|
||||
normalizedValue(properties.industrial) === "port" ||
|
||||
["harbour", "port"].includes(normalizedValue(properties.landuse)) ||
|
||||
waterway === "dock"
|
||||
);
|
||||
}
|
||||
|
||||
function layerFor(properties) {
|
||||
const seamarkType = normalizedValue(properties["seamark:type"]);
|
||||
const waterway = normalizedValue(properties.waterway);
|
||||
const route = normalizedValue(properties.route);
|
||||
|
||||
if (isLockFeature(properties)) {
|
||||
return "locks";
|
||||
}
|
||||
|
||||
if (isHarbourFeature(properties)) {
|
||||
return "harbours";
|
||||
}
|
||||
|
||||
if (hasKey(properties, "bridge")) {
|
||||
return "bridges";
|
||||
}
|
||||
|
||||
if (["fairway", "navigation_line", "recommended_track"].includes(seamarkType)) {
|
||||
return "fairways";
|
||||
}
|
||||
|
||||
if (["fairway", "canal", "river"].includes(waterway) || route === "ferry") {
|
||||
return "fairways";
|
||||
}
|
||||
|
||||
if (seamarkType) {
|
||||
return "seamarks";
|
||||
}
|
||||
|
||||
return "waterways";
|
||||
}
|
||||
|
||||
function isRelevantFeature(properties) {
|
||||
const waterway = normalizedValue(properties.waterway);
|
||||
|
||||
return (
|
||||
isLockFeature(properties) ||
|
||||
isHarbourFeature(properties) ||
|
||||
hasKey(properties, "seamark:type") ||
|
||||
["fairway", "canal", "river", "dock"].includes(waterway) ||
|
||||
normalizedValue(properties.route) === "ferry" ||
|
||||
normalizedValue(properties.natural) === "water" ||
|
||||
normalizedValue(properties.water) === "lake" ||
|
||||
isTruthy(properties.boat) ||
|
||||
isTruthy(properties.ship) ||
|
||||
hasKey(properties, "bridge")
|
||||
);
|
||||
}
|
||||
|
||||
function parseDepth(properties) {
|
||||
const raw = [
|
||||
properties["seamark:fairway:minimum_depth"],
|
||||
properties["seamark:recommended_track:minimum_depth"],
|
||||
properties["seamark:navigation_line:minimum_depth"],
|
||||
properties["depth"],
|
||||
properties["min_depth"]
|
||||
]
|
||||
.map(getString)
|
||||
.find(Boolean);
|
||||
|
||||
if (!raw) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const normalized = raw.replace(",", ".");
|
||||
const match = normalized.match(/-?\d+(?:\.\d+)?/);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const value = Number(match[0]);
|
||||
return Number.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
function isTruthy(value) {
|
||||
return truthy.has(getString(value).toLowerCase());
|
||||
}
|
||||
|
||||
function isFalsy(value) {
|
||||
return falsy.has(normalizedValue(value));
|
||||
}
|
||||
|
||||
function isInactiveOrUnderConstruction(properties) {
|
||||
const waterway = normalizedValue(properties.waterway);
|
||||
if (inactiveWaterwayValues.has(waterway)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const lifecycleKeys = [
|
||||
"abandoned",
|
||||
"construction",
|
||||
"disused",
|
||||
"proposed",
|
||||
"abandoned:waterway",
|
||||
"construction:waterway",
|
||||
"disused:waterway",
|
||||
"proposed:waterway",
|
||||
"disused:route"
|
||||
];
|
||||
return lifecycleKeys.some((key) => hasActiveValue(properties[key]));
|
||||
}
|
||||
|
||||
function isRoutable(properties, geometry) {
|
||||
if (!geometry || !["LineString", "MultiLineString"].includes(geometry.type)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const seamarkType = normalizedValue(properties["seamark:type"]);
|
||||
const waterway = normalizedValue(properties.waterway);
|
||||
const route = normalizedValue(properties.route);
|
||||
|
||||
if (
|
||||
["access", "boat", "ship", "motorboat"].some((key) => isFalsy(properties[key])) ||
|
||||
isInactiveOrUnderConstruction(properties) ||
|
||||
normalizedValue(properties.tunnel) === "culvert"
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (
|
||||
["navigation_line", "recommended_track", "fairway"].includes(seamarkType) ||
|
||||
["fairway", "canal", "river"].includes(waterway) ||
|
||||
route === "ferry" ||
|
||||
isTruthy(properties.boat) ||
|
||||
isTruthy(properties.ship)
|
||||
);
|
||||
}
|
||||
|
||||
function isClosedLine(coords) {
|
||||
if (!Array.isArray(coords) || coords.length < 2) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const first = coords[0];
|
||||
const last = coords[coords.length - 1];
|
||||
return Array.isArray(first) && Array.isArray(last) && first[0] === last[0] && first[1] === last[1];
|
||||
}
|
||||
|
||||
function edgeGeometries(geometry) {
|
||||
if (!geometry) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (geometry.type === "LineString") {
|
||||
return isClosedLine(geometry.coordinates)
|
||||
? []
|
||||
: [{ type: "LineString", coordinates: geometry.coordinates }];
|
||||
}
|
||||
|
||||
if (geometry.type === "MultiLineString") {
|
||||
return geometry.coordinates
|
||||
.filter((line) => Array.isArray(line) && line.length >= 2 && !isClosedLine(line))
|
||||
.map((line) => ({ type: "LineString", coordinates: line }));
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
function pushFeature(record) {
|
||||
featureBatch.push(record);
|
||||
}
|
||||
|
||||
function pushEdge(record) {
|
||||
edgeBatch.push(record);
|
||||
}
|
||||
|
||||
async function flushFeatures() {
|
||||
if (!featureBatch.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
const uniqueFeatures = [
|
||||
...new Map(featureBatch.map((feature) => [`${feature.layer}:${feature.sourceId}`, feature])).values()
|
||||
];
|
||||
const params = [];
|
||||
const values = uniqueFeatures.map((feature, index) => {
|
||||
const offset = index * 5;
|
||||
params.push(
|
||||
feature.layer,
|
||||
feature.sourceId,
|
||||
feature.name,
|
||||
JSON.stringify(feature.properties),
|
||||
JSON.stringify(feature.geometry)
|
||||
);
|
||||
return `($${offset + 1}, 'osm', $${offset + 2}, $${offset + 3}, $${offset + 4}::jsonb, ST_SetSRID(ST_GeomFromGeoJSON($${offset + 5}), 4326))`;
|
||||
});
|
||||
|
||||
await client.query(`
|
||||
WITH incoming (layer, source, source_id, name, properties, geom) AS (
|
||||
VALUES ${values.join(",")}
|
||||
),
|
||||
removed_stale_classifications AS (
|
||||
DELETE FROM marine_features existing
|
||||
USING incoming
|
||||
WHERE existing.source = incoming.source
|
||||
AND existing.source_id = incoming.source_id
|
||||
AND existing.layer <> incoming.layer
|
||||
)
|
||||
INSERT INTO marine_features (layer, source, source_id, name, properties, geom)
|
||||
SELECT layer, source, source_id, name, properties, geom
|
||||
FROM incoming
|
||||
ON CONFLICT (source, source_id, layer) WHERE source_id IS NOT NULL
|
||||
DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
properties = EXCLUDED.properties,
|
||||
geom = EXCLUDED.geom,
|
||||
updated_at = now()
|
||||
`, params);
|
||||
|
||||
importedFeatures += uniqueFeatures.length;
|
||||
featureBatch.length = 0;
|
||||
}
|
||||
|
||||
async function flushEdges() {
|
||||
if (!edgeBatch.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
const uniqueEdges = [...new Map(edgeBatch.map((edge) => [edge.sourceId, edge])).values()];
|
||||
const params = [];
|
||||
const values = uniqueEdges.map((edge, index) => {
|
||||
const offset = index * 5;
|
||||
params.push(
|
||||
edge.sourceId,
|
||||
edge.name,
|
||||
edge.depth,
|
||||
JSON.stringify(edge.properties),
|
||||
JSON.stringify(edge.geometry)
|
||||
);
|
||||
return `('osm', $${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}::jsonb, ST_SetSRID(ST_GeomFromGeoJSON($${offset + 5}), 4326)::geometry(LineString, 4326))`;
|
||||
});
|
||||
|
||||
await client.query(`
|
||||
INSERT INTO marine_fairway_edges (source, source_id, name, min_depth_m, properties, geom)
|
||||
VALUES ${values.join(",")}
|
||||
ON CONFLICT (source, source_id) WHERE source_id IS NOT NULL
|
||||
DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
min_depth_m = EXCLUDED.min_depth_m,
|
||||
properties = EXCLUDED.properties,
|
||||
geom = EXCLUDED.geom,
|
||||
updated_at = now()
|
||||
`, params);
|
||||
|
||||
importedEdges += uniqueEdges.length;
|
||||
edgeBatch.length = 0;
|
||||
}
|
||||
|
||||
async function flushAll() {
|
||||
await flushFeatures();
|
||||
await flushEdges();
|
||||
}
|
||||
|
||||
await client.connect();
|
||||
|
||||
try {
|
||||
const lines = createInterface({
|
||||
input: createReadStream(inputPath, { encoding: "utf8" }),
|
||||
crlfDelay: Infinity
|
||||
});
|
||||
|
||||
for await (const line of lines) {
|
||||
let trimmed = line.trim();
|
||||
if (trimmed.charCodeAt(0) === 0x1e) {
|
||||
trimmed = trimmed.slice(1).trimStart();
|
||||
}
|
||||
|
||||
if (!trimmed) {
|
||||
continue;
|
||||
}
|
||||
|
||||
seenFeatures += 1;
|
||||
const feature = JSON.parse(trimmed);
|
||||
const properties = feature.properties ?? {};
|
||||
const geometry = feature.geometry;
|
||||
if (!isRelevantFeature(properties)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const id = sourceId(feature, seenFeatures, importSource);
|
||||
const name = featureName(properties);
|
||||
|
||||
if (geometry) {
|
||||
pushFeature({
|
||||
layer: layerFor(properties),
|
||||
sourceId: id,
|
||||
name,
|
||||
properties,
|
||||
geometry
|
||||
});
|
||||
}
|
||||
|
||||
if (isRoutable(properties, geometry)) {
|
||||
const linestrings = edgeGeometries(geometry);
|
||||
const depth = parseDepth(properties);
|
||||
linestrings.forEach((edgeGeometry, index) => {
|
||||
pushEdge({
|
||||
sourceId: `${id}/line/${index}`,
|
||||
name,
|
||||
depth,
|
||||
properties,
|
||||
geometry: edgeGeometry
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
if (featureBatch.length >= FEATURE_BATCH_SIZE || edgeBatch.length >= EDGE_BATCH_SIZE) {
|
||||
await flushAll();
|
||||
}
|
||||
}
|
||||
|
||||
await flushAll();
|
||||
console.log(`Read ${seenFeatures} OSM marine features.`);
|
||||
console.log(`Queued ${importedFeatures} marine feature rows and ${importedEdges} routable fairway edge rows.`);
|
||||
} finally {
|
||||
await client.end();
|
||||
}
|
||||
Reference in New Issue
Block a user