watermaps/scripts/load-osm-fairways.mjs

#!/usr/bin/env node
import { createReadStream } from "node:fs";
import { createInterface } from "node:readline";
import pg from "pg";
import {
  featureName,
  isLockFeature,
  normalizedValue,
  sourceId
} from "./osm-marine-classification.mjs";

const inputPath = process.argv[2];
const databaseUrl = process.env.DATABASE_URL ?? "postgres://seacompass:seacompass@localhost:55432/seacompass";
const importSource =
  process.env.WATERMAPS_IMPORT_SOURCE ??
  process.env.SEA_COMPASS_IMPORT_SOURCE ??
  "unknown-pbf";

if (!inputPath) {
  console.error("Usage: DATABASE_URL=postgres://... node scripts/load-osm-fairways.mjs /path/to/marine.geojsonseq");
  process.exit(1);
}

const { Client } = pg;
const client = new Client({ connectionString: databaseUrl });

const FEATURE_BATCH_SIZE = 500;
const EDGE_BATCH_SIZE = 500;
const featureBatch = [];
const edgeBatch = [];
let seenFeatures = 0;
let importedFeatures = 0;
let importedEdges = 0;

const truthy = new Set(["yes", "true", "1", "designated", "permissive"]);
const falsy = new Set(["no", "false", "0", "private", "prohibited", "restricted"]);
const harbourSeamarkTypes = new Set(["harbour", "harbour_basin", "marina"]);
const inactiveWaterwayValues = new Set(["abandoned", "construction", "disused", "proposed"]);

function getString(value) {
  return typeof value === "string" ? value.trim() : "";
}

function hasKey(properties, key) {
  return Object.prototype.hasOwnProperty.call(properties, key) && properties[key] != null;
}

function hasActiveValue(value) {
  const normalized = normalizedValue(value);
  return Boolean(normalized) && !["no", "false", "0"].includes(normalized);
}

function isHarbourFeature(properties) {
  const seamarkType = normalizedValue(properties["seamark:type"]);
  const waterway = normalizedValue(properties.waterway);

  return (
    harbourSeamarkTypes.has(seamarkType) ||
    normalizedValue(properties.leisure) === "marina" ||
    hasActiveValue(properties.harbour) ||
    normalizedValue(properties.industrial) === "port" ||
    ["harbour", "port"].includes(normalizedValue(properties.landuse)) ||
    waterway === "dock"
  );
}

function layerFor(properties) {
  const seamarkType = normalizedValue(properties["seamark:type"]);
  const waterway = normalizedValue(properties.waterway);
  const route = normalizedValue(properties.route);

  if (isLockFeature(properties)) {
    return "locks";
  }

  if (isHarbourFeature(properties)) {
    return "harbours";
  }

  if (hasKey(properties, "bridge")) {
    return "bridges";
  }

  if (["fairway", "navigation_line", "recommended_track"].includes(seamarkType)) {
    return "fairways";
  }

  if (["fairway", "canal", "river"].includes(waterway) || route === "ferry") {
    return "fairways";
  }

  if (seamarkType) {
    return "seamarks";
  }

  return "waterways";
}

function isRelevantFeature(properties) {
  const waterway = normalizedValue(properties.waterway);

  return (
    isLockFeature(properties) ||
    isHarbourFeature(properties) ||
    hasKey(properties, "seamark:type") ||
    ["fairway", "canal", "river", "dock"].includes(waterway) ||
    normalizedValue(properties.route) === "ferry" ||
    normalizedValue(properties.natural) === "water" ||
    normalizedValue(properties.water) === "lake" ||
    isTruthy(properties.boat) ||
    isTruthy(properties.ship) ||
    hasKey(properties, "bridge")
  );
}

function parseDepth(properties) {
  const raw = [
    properties["seamark:fairway:minimum_depth"],
    properties["seamark:recommended_track:minimum_depth"],
    properties["seamark:navigation_line:minimum_depth"],
    properties["depth"],
    properties["min_depth"]
  ]
    .map(getString)
    .find(Boolean);

  if (!raw) {
    return null;
  }

  const normalized = raw.replace(",", ".");
  const match = normalized.match(/-?\d+(?:\.\d+)?/);
  if (!match) {
    return null;
  }

  const value = Number(match[0]);
  return Number.isFinite(value) ? value : null;
}

function isTruthy(value) {
  return truthy.has(getString(value).toLowerCase());
}

function isFalsy(value) {
  return falsy.has(normalizedValue(value));
}

function isInactiveOrUnderConstruction(properties) {
  const waterway = normalizedValue(properties.waterway);
  if (inactiveWaterwayValues.has(waterway)) {
    return true;
  }

  const lifecycleKeys = [
    "abandoned",
    "construction",
    "disused",
    "proposed",
    "abandoned:waterway",
    "construction:waterway",
    "disused:waterway",
    "proposed:waterway",
    "disused:route"
  ];
  return lifecycleKeys.some((key) => hasActiveValue(properties[key]));
}

function isRoutable(properties, geometry) {
  if (!geometry || !["LineString", "MultiLineString"].includes(geometry.type)) {
    return false;
  }

  const seamarkType = normalizedValue(properties["seamark:type"]);
  const waterway = normalizedValue(properties.waterway);
  const route = normalizedValue(properties.route);

  if (
    ["access", "boat", "ship", "motorboat"].some((key) => isFalsy(properties[key])) ||
    isInactiveOrUnderConstruction(properties) ||
    normalizedValue(properties.tunnel) === "culvert"
  ) {
    return false;
  }

  return (
    ["navigation_line", "recommended_track", "fairway"].includes(seamarkType) ||
    ["fairway", "canal", "river"].includes(waterway) ||
    route === "ferry" ||
    isTruthy(properties.boat) ||
    isTruthy(properties.ship)
  );
}

function isClosedLine(coords) {
  if (!Array.isArray(coords) || coords.length < 2) {
    return false;
  }

  const first = coords[0];
  const last = coords[coords.length - 1];
  return Array.isArray(first) && Array.isArray(last) && first[0] === last[0] && first[1] === last[1];
}

function edgeGeometries(geometry) {
  if (!geometry) {
    return [];
  }

  if (geometry.type === "LineString") {
    return isClosedLine(geometry.coordinates)
      ? []
      : [{ type: "LineString", coordinates: geometry.coordinates }];
  }

  if (geometry.type === "MultiLineString") {
    return geometry.coordinates
      .filter((line) => Array.isArray(line) && line.length >= 2 && !isClosedLine(line))
      .map((line) => ({ type: "LineString", coordinates: line }));
  }

  return [];
}

function pushFeature(record) {
  featureBatch.push(record);
}

function pushEdge(record) {
  edgeBatch.push(record);
}

async function flushFeatures() {
  if (!featureBatch.length) {
    return;
  }

  const uniqueFeatures = [
    ...new Map(featureBatch.map((feature) => [`${feature.layer}:${feature.sourceId}`, feature])).values()
  ];
  const params = [];
  const values = uniqueFeatures.map((feature, index) => {
    const offset = index * 5;
    params.push(
      feature.layer,
      feature.sourceId,
      feature.name,
      JSON.stringify(feature.properties),
      JSON.stringify(feature.geometry)
    );
    return `($${offset + 1}, 'osm', $${offset + 2}, $${offset + 3}, $${offset + 4}::jsonb, ST_SetSRID(ST_GeomFromGeoJSON($${offset + 5}), 4326))`;
  });

  await client.query(`
    WITH incoming (layer, source, source_id, name, properties, geom) AS (
      VALUES ${values.join(",")}
    ),
    removed_stale_classifications AS (
      DELETE FROM marine_features existing
      USING incoming
      WHERE existing.source = incoming.source
        AND existing.source_id = incoming.source_id
        AND existing.layer <> incoming.layer
    )
    INSERT INTO marine_features (layer, source, source_id, name, properties, geom)
    SELECT layer, source, source_id, name, properties, geom
    FROM incoming
    ON CONFLICT (source, source_id, layer) WHERE source_id IS NOT NULL
    DO UPDATE SET
      name = EXCLUDED.name,
      properties = EXCLUDED.properties,
      geom = EXCLUDED.geom,
      updated_at = now()
  `, params);

  importedFeatures += uniqueFeatures.length;
  featureBatch.length = 0;
}

async function flushEdges() {
  if (!edgeBatch.length) {
    return;
  }

  const uniqueEdges = [...new Map(edgeBatch.map((edge) => [edge.sourceId, edge])).values()];
  const params = [];
  const values = uniqueEdges.map((edge, index) => {
    const offset = index * 5;
    params.push(
      edge.sourceId,
      edge.name,
      edge.depth,
      JSON.stringify(edge.properties),
      JSON.stringify(edge.geometry)
    );
    return `('osm', $${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}::jsonb, ST_SetSRID(ST_GeomFromGeoJSON($${offset + 5}), 4326)::geometry(LineString, 4326))`;
  });

  await client.query(`
    INSERT INTO marine_fairway_edges (source, source_id, name, min_depth_m, properties, geom)
    VALUES ${values.join(",")}
    ON CONFLICT (source, source_id) WHERE source_id IS NOT NULL
    DO UPDATE SET
      name = EXCLUDED.name,
      min_depth_m = EXCLUDED.min_depth_m,
      properties = EXCLUDED.properties,
      geom = EXCLUDED.geom,
      updated_at = now()
  `, params);

  importedEdges += uniqueEdges.length;
  edgeBatch.length = 0;
}

async function flushAll() {
  await flushFeatures();
  await flushEdges();
}

await client.connect();

try {
  const lines = createInterface({
    input: createReadStream(inputPath, { encoding: "utf8" }),
    crlfDelay: Infinity
  });

  for await (const line of lines) {
    let trimmed = line.trim();
    if (trimmed.charCodeAt(0) === 0x1e) {
      trimmed = trimmed.slice(1).trimStart();
    }

    if (!trimmed) {
      continue;
    }

    seenFeatures += 1;
    const feature = JSON.parse(trimmed);
    const properties = feature.properties ?? {};
    const geometry = feature.geometry;
    if (!isRelevantFeature(properties)) {
      continue;
    }

    const id = sourceId(feature, seenFeatures, importSource);
    const name = featureName(properties);

    if (geometry) {
      pushFeature({
        layer: layerFor(properties),
        sourceId: id,
        name,
        properties,
        geometry
      });
    }

    if (isRoutable(properties, geometry)) {
      const linestrings = edgeGeometries(geometry);
      const depth = parseDepth(properties);
      linestrings.forEach((edgeGeometry, index) => {
        pushEdge({
          sourceId: `${id}/line/${index}`,
          name,
          depth,
          properties,
          geometry: edgeGeometry
        });
      });
    }

    if (featureBatch.length >= FEATURE_BATCH_SIZE || edgeBatch.length >= EDGE_BATCH_SIZE) {
      await flushAll();
    }
  }

  await flushAll();
  console.log(`Read ${seenFeatures} OSM marine features.`);
  console.log(`Queued ${importedFeatures} marine feature rows and ${importedEdges} routable fairway edge rows.`);
} finally {
  await client.end();
}