#!/usr/bin/env node import { createReadStream } from "node:fs"; import { createInterface } from "node:readline"; import pg from "pg"; import { featureName, isLockFeature, normalizedValue, sourceId } from "./osm-marine-classification.mjs"; const inputPath = process.argv[2]; const databaseUrl = process.env.DATABASE_URL ?? "postgres://seacompass:seacompass@localhost:55432/seacompass"; const importSource = process.env.WATERMAPS_IMPORT_SOURCE ?? process.env.SEA_COMPASS_IMPORT_SOURCE ?? "unknown-pbf"; if (!inputPath) { console.error("Usage: DATABASE_URL=postgres://... node scripts/load-osm-fairways.mjs /path/to/marine.geojsonseq"); process.exit(1); } const { Client } = pg; const client = new Client({ connectionString: databaseUrl }); const FEATURE_BATCH_SIZE = 500; const EDGE_BATCH_SIZE = 500; const featureBatch = []; const edgeBatch = []; let seenFeatures = 0; let importedFeatures = 0; let importedEdges = 0; const truthy = new Set(["yes", "true", "1", "designated", "permissive"]); const falsy = new Set(["no", "false", "0", "private", "prohibited", "restricted"]); const harbourSeamarkTypes = new Set(["harbour", "harbour_basin", "marina"]); const inactiveWaterwayValues = new Set(["abandoned", "construction", "disused", "proposed"]); function getString(value) { return typeof value === "string" ? value.trim() : ""; } function hasKey(properties, key) { return Object.prototype.hasOwnProperty.call(properties, key) && properties[key] != null; } function hasActiveValue(value) { const normalized = normalizedValue(value); return Boolean(normalized) && !["no", "false", "0"].includes(normalized); } function isHarbourFeature(properties) { const seamarkType = normalizedValue(properties["seamark:type"]); const waterway = normalizedValue(properties.waterway); return ( harbourSeamarkTypes.has(seamarkType) || normalizedValue(properties.leisure) === "marina" || hasActiveValue(properties.harbour) || normalizedValue(properties.industrial) === "port" || ["harbour", "port"].includes(normalizedValue(properties.landuse)) || waterway === "dock" ); } function layerFor(properties) { const seamarkType = normalizedValue(properties["seamark:type"]); const waterway = normalizedValue(properties.waterway); const route = normalizedValue(properties.route); if (isLockFeature(properties)) { return "locks"; } if (isHarbourFeature(properties)) { return "harbours"; } if (hasKey(properties, "bridge")) { return "bridges"; } if (["fairway", "navigation_line", "recommended_track"].includes(seamarkType)) { return "fairways"; } if (["fairway", "canal", "river"].includes(waterway) || route === "ferry") { return "fairways"; } if (seamarkType) { return "seamarks"; } return "waterways"; } function isRelevantFeature(properties) { const waterway = normalizedValue(properties.waterway); return ( isLockFeature(properties) || isHarbourFeature(properties) || hasKey(properties, "seamark:type") || ["fairway", "canal", "river", "dock"].includes(waterway) || normalizedValue(properties.route) === "ferry" || normalizedValue(properties.natural) === "water" || normalizedValue(properties.water) === "lake" || isTruthy(properties.boat) || isTruthy(properties.ship) || hasKey(properties, "bridge") ); } function parseDepth(properties) { const raw = [ properties["seamark:fairway:minimum_depth"], properties["seamark:recommended_track:minimum_depth"], properties["seamark:navigation_line:minimum_depth"], properties["depth"], properties["min_depth"] ] .map(getString) .find(Boolean); if (!raw) { return null; } const normalized = raw.replace(",", "."); const match = normalized.match(/-?\d+(?:\.\d+)?/); if (!match) { return null; } const value = Number(match[0]); return Number.isFinite(value) ? value : null; } function isTruthy(value) { return truthy.has(getString(value).toLowerCase()); } function isFalsy(value) { return falsy.has(normalizedValue(value)); } function isInactiveOrUnderConstruction(properties) { const waterway = normalizedValue(properties.waterway); if (inactiveWaterwayValues.has(waterway)) { return true; } const lifecycleKeys = [ "abandoned", "construction", "disused", "proposed", "abandoned:waterway", "construction:waterway", "disused:waterway", "proposed:waterway", "disused:route" ]; return lifecycleKeys.some((key) => hasActiveValue(properties[key])); } function isRoutable(properties, geometry) { if (!geometry || !["LineString", "MultiLineString"].includes(geometry.type)) { return false; } const seamarkType = normalizedValue(properties["seamark:type"]); const waterway = normalizedValue(properties.waterway); const route = normalizedValue(properties.route); if ( ["access", "boat", "ship", "motorboat"].some((key) => isFalsy(properties[key])) || isInactiveOrUnderConstruction(properties) || normalizedValue(properties.tunnel) === "culvert" ) { return false; } return ( ["navigation_line", "recommended_track", "fairway"].includes(seamarkType) || ["fairway", "canal", "river"].includes(waterway) || route === "ferry" || isTruthy(properties.boat) || isTruthy(properties.ship) ); } function isClosedLine(coords) { if (!Array.isArray(coords) || coords.length < 2) { return false; } const first = coords[0]; const last = coords[coords.length - 1]; return Array.isArray(first) && Array.isArray(last) && first[0] === last[0] && first[1] === last[1]; } function edgeGeometries(geometry) { if (!geometry) { return []; } if (geometry.type === "LineString") { return isClosedLine(geometry.coordinates) ? [] : [{ type: "LineString", coordinates: geometry.coordinates }]; } if (geometry.type === "MultiLineString") { return geometry.coordinates .filter((line) => Array.isArray(line) && line.length >= 2 && !isClosedLine(line)) .map((line) => ({ type: "LineString", coordinates: line })); } return []; } function pushFeature(record) { featureBatch.push(record); } function pushEdge(record) { edgeBatch.push(record); } async function flushFeatures() { if (!featureBatch.length) { return; } const uniqueFeatures = [ ...new Map(featureBatch.map((feature) => [`${feature.layer}:${feature.sourceId}`, feature])).values() ]; const params = []; const values = uniqueFeatures.map((feature, index) => { const offset = index * 5; params.push( feature.layer, feature.sourceId, feature.name, JSON.stringify(feature.properties), JSON.stringify(feature.geometry) ); return `($${offset + 1}, 'osm', $${offset + 2}, $${offset + 3}, $${offset + 4}::jsonb, ST_SetSRID(ST_GeomFromGeoJSON($${offset + 5}), 4326))`; }); await client.query(` WITH incoming (layer, source, source_id, name, properties, geom) AS ( VALUES ${values.join(",")} ), removed_stale_classifications AS ( DELETE FROM marine_features existing USING incoming WHERE existing.source = incoming.source AND existing.source_id = incoming.source_id AND existing.layer <> incoming.layer ) INSERT INTO marine_features (layer, source, source_id, name, properties, geom) SELECT layer, source, source_id, name, properties, geom FROM incoming ON CONFLICT (source, source_id, layer) WHERE source_id IS NOT NULL DO UPDATE SET name = EXCLUDED.name, properties = EXCLUDED.properties, geom = EXCLUDED.geom, updated_at = now() `, params); importedFeatures += uniqueFeatures.length; featureBatch.length = 0; } async function flushEdges() { if (!edgeBatch.length) { return; } const uniqueEdges = [...new Map(edgeBatch.map((edge) => [edge.sourceId, edge])).values()]; const params = []; const values = uniqueEdges.map((edge, index) => { const offset = index * 5; params.push( edge.sourceId, edge.name, edge.depth, JSON.stringify(edge.properties), JSON.stringify(edge.geometry) ); return `('osm', $${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}::jsonb, ST_SetSRID(ST_GeomFromGeoJSON($${offset + 5}), 4326)::geometry(LineString, 4326))`; }); await client.query(` INSERT INTO marine_fairway_edges (source, source_id, name, min_depth_m, properties, geom) VALUES ${values.join(",")} ON CONFLICT (source, source_id) WHERE source_id IS NOT NULL DO UPDATE SET name = EXCLUDED.name, min_depth_m = EXCLUDED.min_depth_m, properties = EXCLUDED.properties, geom = EXCLUDED.geom, updated_at = now() `, params); importedEdges += uniqueEdges.length; edgeBatch.length = 0; } async function flushAll() { await flushFeatures(); await flushEdges(); } await client.connect(); try { const lines = createInterface({ input: createReadStream(inputPath, { encoding: "utf8" }), crlfDelay: Infinity }); for await (const line of lines) { let trimmed = line.trim(); if (trimmed.charCodeAt(0) === 0x1e) { trimmed = trimmed.slice(1).trimStart(); } if (!trimmed) { continue; } seenFeatures += 1; const feature = JSON.parse(trimmed); const properties = feature.properties ?? {}; const geometry = feature.geometry; if (!isRelevantFeature(properties)) { continue; } const id = sourceId(feature, seenFeatures, importSource); const name = featureName(properties); if (geometry) { pushFeature({ layer: layerFor(properties), sourceId: id, name, properties, geometry }); } if (isRoutable(properties, geometry)) { const linestrings = edgeGeometries(geometry); const depth = parseDepth(properties); linestrings.forEach((edgeGeometry, index) => { pushEdge({ sourceId: `${id}/line/${index}`, name, depth, properties, geometry: edgeGeometry }); }); } if (featureBatch.length >= FEATURE_BATCH_SIZE || edgeBatch.length >= EDGE_BATCH_SIZE) { await flushAll(); } } await flushAll(); console.log(`Read ${seenFeatures} OSM marine features.`); console.log(`Queued ${importedFeatures} marine feature rows and ${importedEdges} routable fairway edge rows.`); } finally { await client.end(); }