Initial Watermaps import

This commit is contained in:
BuTzZ
2026-07-24 11:29:24 +02:00
commit 57f7b4dedb
129 changed files with 43136 additions and 0 deletions
+547
View File
@@ -0,0 +1,547 @@
#!/usr/bin/env node
import { pathToFileURL } from "node:url";
export const EURIS_BASE_URL = "https://www.eurisportal.eu";
export const EURIS_COMPACT_LOCKS_URL = `${EURIS_BASE_URL}/visuris/api/Locks_v2/GetCompactLocks`;
export const EURIS_RIS_INDEX_URL = `${EURIS_BASE_URL}/visuris/api/RisIndices_v2/GetRISIndexObjects`;
export const EURIS_LOCK_DETAIL_URL = `${EURIS_BASE_URL}/visuris/api/Locks_v2/GetLock`;
export const MAX_PAGE_SIZE = 100;
export const MAX_DETAIL_LIMIT = 20;
export const DETAIL_REQUEST_DELAY_MS = 1_000;
const DEFAULT_TIMEOUT_MS = 15_000;
const DEFAULT_MAX_RETRIES = 4;
const MAX_PAGES = 100;
const sleep = (milliseconds) =>
new Promise((resolve) => setTimeout(resolve, milliseconds));
export class EurisRequestError extends Error {
constructor(message, { status, url, cause } = {}) {
super(message, { cause });
this.name = "EurisRequestError";
this.status = status;
this.url = url;
}
}
function nonEmptyString(value) {
if (value === null || value === undefined) return null;
const normalized = String(value).trim();
return normalized || null;
}
function firstNonEmpty(...values) {
for (const value of values) {
const normalized = nonEmptyString(value);
if (normalized) return normalized;
}
return null;
}
function finiteCoordinate(value, minimum, maximum) {
const coordinate = Number(value);
return Number.isFinite(coordinate) && coordinate >= minimum && coordinate <= maximum
? coordinate
: null;
}
function unique(values) {
return [...new Set(values.filter(Boolean))];
}
export function parseCountries(value = "DE") {
const countries = unique(
String(value || "DE")
.split(/[;,\s]+/u)
.map((country) => country.trim().toUpperCase())
.filter(Boolean),
);
if (countries.length === 0) return ["DE"];
const invalid = countries.find((country) => !/^[A-Z]{2}$/u.test(country));
if (invalid) {
throw new Error(`Ungültiger ISO-Ländercode in EURIS_COUNTRIES: ${invalid}`);
}
return countries;
}
export function parseDetailLimit(value) {
if (value === undefined || value === null || value === "") return 0;
const parsed = Number.parseInt(String(value), 10);
if (!Number.isFinite(parsed) || parsed < 0) {
throw new Error("EURIS_DETAIL_LIMIT muss eine nichtnegative Ganzzahl sein.");
}
return Math.min(parsed, MAX_DETAIL_LIMIT);
}
export function parseBoolean(value) {
return /^(1|true|yes|on)$/iu.test(String(value || "").trim());
}
export function normalizePhones(value) {
if (!value) return [];
const values = Array.isArray(value) ? value : [value];
return unique(
values
.flatMap((entry) => String(entry).split(/\s*[;|]\s*/u))
.map((phone) => phone.trim().replace(/\s+/gu, " "))
.map((phone) => phone.replace(/^00(?=\d)/u, "+"))
.filter(Boolean),
);
}
export function normalizeVhf(value) {
return nonEmptyString(value)?.replace(/\s+/gu, " ") ?? null;
}
export function retryAfterMilliseconds(value, now = Date.now()) {
const normalized = nonEmptyString(value);
if (!normalized) return null;
const seconds = Number(normalized);
if (Number.isFinite(seconds) && seconds >= 0) return Math.ceil(seconds * 1_000);
const date = Date.parse(normalized);
if (!Number.isFinite(date)) return null;
return Math.max(0, date - now);
}
function backoffMilliseconds(attempt) {
return Math.min(30_000, 1_000 * 2 ** attempt);
}
async function responseBodySnippet(response) {
try {
return (await response.text()).replace(/\s+/gu, " ").slice(0, 300);
} catch {
return "";
}
}
export async function requestJson(
url,
{
fetchImpl = globalThis.fetch,
token,
timeoutMs = DEFAULT_TIMEOUT_MS,
maxRetries = DEFAULT_MAX_RETRIES,
sleepImpl = sleep,
now = Date.now,
} = {},
) {
if (typeof fetchImpl !== "function") throw new Error("Keine Fetch-Implementierung verfügbar.");
for (let attempt = 0; attempt <= maxRetries; attempt += 1) {
const controller = new AbortController();
const timeout = setTimeout(
() => controller.abort(new Error(`EuRIS-Anfrage nach ${timeoutMs} ms abgebrochen.`)),
timeoutMs,
);
try {
const headers = new Headers({ Accept: "application/json" });
if (token) headers.set("Authorization", `Bearer ${token}`);
const response = await fetchImpl(url, {
method: "GET",
headers,
signal: controller.signal,
});
if (response.ok) {
const body = await response.text();
return body ? JSON.parse(body) : null;
}
const retryable = [429, 502, 503].includes(response.status);
if (!retryable || attempt === maxRetries) {
const snippet = await responseBodySnippet(response);
throw new EurisRequestError(
`EuRIS antwortete mit HTTP ${response.status}${snippet ? `: ${snippet}` : ""}`,
{ status: response.status, url },
);
}
const retryAfter = retryAfterMilliseconds(response.headers.get("retry-after"), now());
await sleepImpl(retryAfter ?? backoffMilliseconds(attempt));
} catch (error) {
if (error instanceof EurisRequestError) throw error;
if (attempt === maxRetries) {
throw new EurisRequestError(`EuRIS-Anfrage fehlgeschlagen: ${error.message}`, {
url,
cause: error,
});
}
await sleepImpl(backoffMilliseconds(attempt));
} finally {
clearTimeout(timeout);
}
}
throw new EurisRequestError("EuRIS-Anfrage ohne Ergebnis beendet.", { url });
}
export function compactLocksFilter(countries) {
return countries.map((country) => `startswith(locode,'${country}')`).join(" or ");
}
export function risIndexFilter(countries) {
const countryFilter = countries.map((country) => `countryCode eq '${country}'`).join(" or ");
return `(${countryFilter}) and function eq 'lokare'`;
}
export async function fetchODataPages({
endpoint,
filter,
orderBy,
idForItem,
pageSize = MAX_PAGE_SIZE,
requestOptions,
}) {
const top = Math.max(1, Math.min(MAX_PAGE_SIZE, Number(pageSize) || MAX_PAGE_SIZE));
const records = new Map();
let skip = 0;
let expectedCount = null;
for (let page = 0; page < MAX_PAGES; page += 1) {
const url = new URL(endpoint);
url.searchParams.set("$filter", filter);
url.searchParams.set("$orderby", orderBy);
url.searchParams.set("$top", String(top));
url.searchParams.set("$skip", String(skip));
url.searchParams.set("$count", "true");
const payload = await requestJson(url, requestOptions);
const items = Array.isArray(payload?.items) ? payload.items : [];
if (expectedCount === null && Number.isFinite(Number(payload?.count))) {
expectedCount = Number(payload.count);
}
for (const item of items) {
const identifier = nonEmptyString(idForItem(item))?.toUpperCase();
if (identifier) records.set(identifier, item);
}
skip += items.length;
if (items.length < top || items.length === 0 || (expectedCount !== null && skip >= expectedCount)) {
return { items: [...records.values()], expectedCount, requestedCount: skip };
}
}
throw new Error(`EuRIS-Paginierung überschritt ${MAX_PAGES} Seiten für ${endpoint}.`);
}
function splitContactValues(values) {
if (!values) return [];
const entries = Array.isArray(values) ? values : [values];
return unique(
entries
.flatMap((entry) => String(entry).split(/\s*[;,|]\s*/u))
.map((entry) => entry.trim())
.filter(Boolean),
);
}
export function extractDetailFields(detail) {
if (!detail || typeof detail !== "object") return {};
const facility = detail.facility && typeof detail.facility === "object" ? detail.facility : {};
const contacts = Array.isArray(facility.contacts) ? facility.contacts : [];
const phones = normalizePhones(contacts.flatMap((contact) => contact?.phones || []));
const emails = splitContactValues(contacts.flatMap((contact) => contact?.emails || []));
const websites = splitContactValues(contacts.flatMap((contact) => contact?.urls || []));
const addressParts = [facility.street, facility.postCode, facility.city, facility.country]
.map(nonEmptyString)
.filter(Boolean);
return {
operator: firstNonEmpty(
facility.operator,
facility.owner,
...contacts.map((contact) => contact?.company),
detail.waterwayAuthority,
),
phones,
email: emails[0] ?? null,
website: websites[0] ?? null,
address: addressParts.length > 0 ? addressParts.join(", ") : null,
};
}
export function buildLockRecord({ compact, ris, detail, fetchedAt }) {
const sourceId = firstNonEmpty(ris?.isrs, compact?.locode)?.toUpperCase();
if (!sourceId) return null;
const longitude = finiteCoordinate(ris?.lon ?? ris?.longitude, -180, 180);
const latitude = finiteCoordinate(ris?.lat ?? ris?.latitude, -90, 90);
if (longitude === null || latitude === null) return null;
const detailFields = extractDetailFields(detail);
const compactPhones = normalizePhones(compact?.contactPhone);
const phones = unique([...compactPhones, ...(detailFields.phones || [])]);
const phone = phones[0] ?? null;
const name = firstNonEmpty(
compact?.nationalObjectName,
compact?.objectName,
ris?.nationalObjectName,
ris?.objectName,
sourceId,
);
const waterway = firstNonEmpty(compact?.waterwayName, ris?.fairwaySectionName, ris?.fairwayName);
const detailUrl = new URL(EURIS_LOCK_DETAIL_URL);
detailUrl.searchParams.set("isrs", sourceId);
const properties = {
isrs: sourceId,
"ref:EU:RIS": sourceId,
name,
lock_name: name,
country: firstNonEmpty(ris?.countryCode, sourceId.slice(0, 2)),
waterway_name: waterway,
waterwayName: waterway,
hectom: compact?.hectom ?? ris?.hectom ?? null,
phone,
phones,
phone_raw: nonEmptyString(compact?.contactPhone),
"contact:phone": phone,
vhf: normalizeVhf(compact?.comcha),
operator: detailFields.operator ?? null,
email: detailFields.email ?? null,
website: detailFields.website ?? null,
address: detailFields.address ?? null,
upstream_source: nonEmptyString(ris?.source),
upstreamSource: nonEmptyString(ris?.source),
data_source: nonEmptyString(ris?.source),
source_url: detailUrl.toString(),
compact_source_url: EURIS_COMPACT_LOCKS_URL,
ris_source_url: EURIS_RIS_INDEX_URL,
fetched_at: fetchedAt,
fetchedAt,
};
return { sourceId, name, longitude, latitude, properties };
}
async function fetchLockDetail(sourceId, requestOptions) {
const url = new URL(EURIS_LOCK_DETAIL_URL);
url.searchParams.set("isrs", sourceId);
return requestJson(url, requestOptions);
}
export async function collectEurisLocks({
countries = ["DE"],
detailLimit = 0,
pageSize = MAX_PAGE_SIZE,
fetchImpl = globalThis.fetch,
token,
timeoutMs = DEFAULT_TIMEOUT_MS,
maxRetries = DEFAULT_MAX_RETRIES,
sleepImpl = sleep,
fetchedAt = new Date().toISOString(),
logger = console,
} = {}) {
const normalizedCountries = parseCountries(
Array.isArray(countries) ? countries.join(",") : countries,
);
const requestOptions = { fetchImpl, token, timeoutMs, maxRetries, sleepImpl };
const compactPage = await fetchODataPages({
endpoint: EURIS_COMPACT_LOCKS_URL,
filter: compactLocksFilter(normalizedCountries),
orderBy: "objectName,waterwayName,hectom",
idForItem: (item) => item.locode,
pageSize,
requestOptions,
});
const risPage = await fetchODataPages({
endpoint: EURIS_RIS_INDEX_URL,
filter: risIndexFilter(normalizedCountries),
orderBy: "objectName",
idForItem: (item) => item.isrs,
pageSize,
requestOptions,
});
const compactByIsrs = new Map(
compactPage.items
.map((item) => [nonEmptyString(item.locode)?.toUpperCase(), item])
.filter(([identifier]) => identifier),
);
const risByIsrs = new Map(
risPage.items
.map((item) => [nonEmptyString(item.isrs)?.toUpperCase(), item])
.filter(([identifier]) => identifier),
);
const joinedIds = [...risByIsrs.keys()].filter((sourceId) => compactByIsrs.has(sourceId)).sort();
const details = new Map();
let detailFailures = 0;
const effectiveDetailLimit = Math.min(parseDetailLimit(detailLimit), joinedIds.length);
for (let index = 0; index < effectiveDetailLimit; index += 1) {
const sourceId = joinedIds[index];
try {
details.set(sourceId, await fetchLockDetail(sourceId, requestOptions));
} catch (error) {
detailFailures += 1;
logger.warn?.(`EuRIS-Details für ${sourceId} konnten nicht geladen werden: ${error.message}`);
}
if (index + 1 < effectiveDetailLimit) await sleepImpl(DETAIL_REQUEST_DELAY_MS);
}
const records = [];
let skippedWithoutCoordinates = 0;
for (const [sourceId, ris] of [...risByIsrs.entries()].sort(([left], [right]) => left.localeCompare(right))) {
const record = buildLockRecord({
compact: compactByIsrs.get(sourceId),
ris,
detail: details.get(sourceId),
fetchedAt,
});
if (record) records.push(record);
else skippedWithoutCoordinates += 1;
}
return {
records,
stats: {
countries: normalizedCountries,
compactLocks: compactByIsrs.size,
risLocks: risByIsrs.size,
joinedLocks: joinedIds.length,
risOnlyLocks: [...risByIsrs.keys()].filter((sourceId) => !compactByIsrs.has(sourceId)).length,
compactOnlyLocks: [...compactByIsrs.keys()].filter((sourceId) => !risByIsrs.has(sourceId)).length,
storedLocks: records.length,
skippedWithoutCoordinates,
detailsRequested: effectiveDetailLimit,
detailFailures,
fetchedAt,
},
};
}
export async function writeEurisLocks(records, { databaseUrl = process.env.DATABASE_URL } = {}) {
if (!databaseUrl) throw new Error("DATABASE_URL ist für den EuRIS-Import erforderlich.");
if (records.length === 0) return 0;
const { default: pg } = await import("pg");
const client = new pg.Client({ connectionString: databaseUrl });
await client.connect();
try {
await client.query("BEGIN");
const batchSize = 100;
for (let offset = 0; offset < records.length; offset += batchSize) {
const batch = records.slice(offset, offset + batchSize);
const values = [];
const rows = batch.map((record, index) => {
const position = index * 6;
values.push(
record.sourceId,
record.name,
JSON.stringify(record.properties),
record.longitude,
record.latitude,
record.properties.fetched_at,
);
return `(\$${position + 1}, \$${position + 2}, \$${position + 3}::jsonb, \$${position + 4}, \$${position + 5}, \$${position + 6}::timestamptz)`;
});
await client.query(
`
INSERT INTO marine_features
(source_id, name, properties, geom, updated_at, layer, source)
SELECT
incoming.source_id,
incoming.name,
incoming.properties,
ST_SetSRID(
ST_MakePoint(incoming.longitude::double precision, incoming.latitude::double precision),
4326
),
incoming.updated_at,
'locks',
'euris'
FROM (VALUES ${rows.join(",")}) AS incoming
(source_id, name, properties, longitude, latitude, updated_at)
ON CONFLICT (source, source_id, layer) WHERE source_id IS NOT NULL
DO UPDATE SET
name = EXCLUDED.name,
properties = EXCLUDED.properties,
geom = EXCLUDED.geom,
updated_at = EXCLUDED.updated_at
`,
values,
);
}
await client.query("COMMIT");
return records.length;
} catch (error) {
await client.query("ROLLBACK");
throw error;
} finally {
await client.end();
}
}
export async function runEurisSync({
dryRun = false,
collectorOptions,
writer = writeEurisLocks,
databaseUrl,
} = {}) {
const result = await collectEurisLocks(collectorOptions);
if (!dryRun) await writer(result.records, { databaseUrl });
return { ...result, dryRun };
}
function boundedInteger(value, fallback, minimum, maximum) {
if (value === undefined || value === null || value === "") return fallback;
const parsed = Number.parseInt(String(value), 10);
if (!Number.isFinite(parsed) || parsed < minimum || parsed > maximum) {
throw new Error(`Wert muss zwischen ${minimum} und ${maximum} liegen.`);
}
return parsed;
}
async function main() {
const countries = parseCountries(process.env.EURIS_COUNTRIES || "DE");
const configuredDetailLimit = Number.parseInt(process.env.EURIS_DETAIL_LIMIT || "0", 10);
const detailLimit = parseDetailLimit(process.env.EURIS_DETAIL_LIMIT);
const dryRun = parseBoolean(process.env.EURIS_DRY_RUN);
const timeoutMs = boundedInteger(process.env.EURIS_REQUEST_TIMEOUT_MS, DEFAULT_TIMEOUT_MS, 1_000, 60_000);
if (Number.isFinite(configuredDetailLimit) && configuredDetailLimit > MAX_DETAIL_LIMIT) {
console.warn(`EURIS_DETAIL_LIMIT wurde aus Rücksicht auf das EuRIS-Limit auf ${MAX_DETAIL_LIMIT} begrenzt.`);
}
console.log(
`EuRIS-Schleusenabgleich für ${countries.join(", ")} gestartet${dryRun ? " (Trockenlauf)" : ""}.`,
);
const result = await runEurisSync({
dryRun,
databaseUrl: process.env.DATABASE_URL,
collectorOptions: {
countries,
detailLimit,
token: nonEmptyString(process.env.EURIS_API_TOKEN),
timeoutMs,
},
});
console.log(JSON.stringify(result.stats, null, 2));
console.log(
dryRun
? `${result.records.length} Schleusen validiert; die Datenbank wurde nicht verändert.`
: `${result.records.length} EuRIS-Schleusen in marine_features aktualisiert.`,
);
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
main().catch((error) => {
console.error(error);
process.exitCode = 1;
});
}