1124 lines
36 KiB
JavaScript
1124 lines
36 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
import { lookup as dnsLookup } from "node:dns/promises";
|
|
import { request as httpRequest } from "node:http";
|
|
import { request as httpsRequest } from "node:https";
|
|
import { BlockList, isIP } from "node:net";
|
|
import { Readable } from "node:stream";
|
|
import { pathToFileURL } from "node:url";
|
|
|
|
export const DEFAULT_LIMIT = 25;
|
|
export const DEFAULT_CONCURRENCY = 2;
|
|
export const DEFAULT_HOST_DELAY_MS = 1_000;
|
|
export const DEFAULT_TIMEOUT_MS = 10_000;
|
|
export const MAX_HTML_BYTES = 512 * 1024;
|
|
export const MAX_REDIRECTS = 3;
|
|
export const USER_AGENT = "Watermaps/0.1 marine-facility-contact-enricher";
|
|
|
|
const MAX_LIMIT = 100;
|
|
const MAX_CONCURRENCY = 4;
|
|
const WEBSITE_KEYS = ["website", "contact:website", "url"];
|
|
const PHONE_KEYS = ["phone", "contact:phone"];
|
|
const EMAIL_KEYS = ["email", "contact:email"];
|
|
const OPERATOR_KEYS = ["operator", "operator:name", "owner"];
|
|
const ADDRESS_KEYS = ["address", "contact:address", "addr:full"];
|
|
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
|
|
|
|
const blockedAddresses = new BlockList();
|
|
for (const [network, prefix] of [
|
|
["0.0.0.0", 8],
|
|
["10.0.0.0", 8],
|
|
["100.64.0.0", 10],
|
|
["127.0.0.0", 8],
|
|
["169.254.0.0", 16],
|
|
["172.16.0.0", 12],
|
|
["192.0.0.0", 24],
|
|
["192.0.2.0", 24],
|
|
["192.168.0.0", 16],
|
|
["198.18.0.0", 15],
|
|
["198.51.100.0", 24],
|
|
["203.0.113.0", 24],
|
|
["224.0.0.0", 4],
|
|
["240.0.0.0", 4],
|
|
]) {
|
|
blockedAddresses.addSubnet(network, prefix, "ipv4");
|
|
}
|
|
for (const [network, prefix] of [
|
|
["::", 128],
|
|
["::1", 128],
|
|
["100::", 64],
|
|
["2001:db8::", 32],
|
|
["fc00::", 7],
|
|
["fe80::", 10],
|
|
["fec0::", 10],
|
|
["ff00::", 8],
|
|
]) {
|
|
blockedAddresses.addSubnet(network, prefix, "ipv6");
|
|
}
|
|
|
|
const sleep = (milliseconds) =>
|
|
new Promise((resolve) => setTimeout(resolve, milliseconds));
|
|
|
|
export class WebsiteEnrichmentError extends Error {
|
|
constructor(message, { code, url, status, cause } = {}) {
|
|
super(message, { cause });
|
|
this.name = "WebsiteEnrichmentError";
|
|
this.code = code;
|
|
this.url = url;
|
|
this.status = status;
|
|
}
|
|
}
|
|
|
|
function nonEmptyString(value) {
|
|
if (typeof value !== "string" && typeof value !== "number") return null;
|
|
const normalized = String(value).trim();
|
|
return normalized || null;
|
|
}
|
|
|
|
function firstString(properties, keys) {
|
|
for (const key of keys) {
|
|
const value = nonEmptyString(properties?.[key]);
|
|
if (value) return value;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function boundedInteger(value, fallback, minimum, maximum, name) {
|
|
if (value === undefined || value === null || value === "") return fallback;
|
|
if (!/^\d+$/u.test(String(value).trim())) {
|
|
throw new Error(`${name} muss eine Ganzzahl sein.`);
|
|
}
|
|
const parsed = Number(value);
|
|
if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) {
|
|
throw new Error(`${name} muss zwischen ${minimum} und ${maximum} liegen.`);
|
|
}
|
|
return parsed;
|
|
}
|
|
|
|
export function parseBooleanDefault(value, fallback = true) {
|
|
if (value === undefined || value === null || value === "") return fallback;
|
|
const normalized = String(value).trim().toLowerCase();
|
|
if (["1", "true", "yes", "on"].includes(normalized)) return true;
|
|
if (["0", "false", "no", "off"].includes(normalized)) return false;
|
|
throw new Error("Boolean-Wert muss true oder false sein.");
|
|
}
|
|
|
|
export function parseHttpUrl(value, base) {
|
|
let parsed;
|
|
try {
|
|
parsed = base ? new URL(String(value), base) : new URL(String(value));
|
|
} catch (error) {
|
|
throw new WebsiteEnrichmentError("Ungültige Facility-URL.", {
|
|
code: "INVALID_URL",
|
|
url: String(value),
|
|
cause: error,
|
|
});
|
|
}
|
|
|
|
if (!(["http:", "https:"].includes(parsed.protocol))) {
|
|
throw new WebsiteEnrichmentError("Nur HTTP- und HTTPS-Facility-URLs sind erlaubt.", {
|
|
code: "UNSAFE_SCHEME",
|
|
url: parsed.href,
|
|
});
|
|
}
|
|
if (parsed.username || parsed.password) {
|
|
throw new WebsiteEnrichmentError("Facility-URLs mit Zugangsdaten werden nicht abgerufen.", {
|
|
code: "URL_CREDENTIALS",
|
|
url: parsed.href,
|
|
});
|
|
}
|
|
parsed.hash = "";
|
|
return parsed;
|
|
}
|
|
|
|
function normalizedHostname(url) {
|
|
return url.hostname.replace(/^\[|\]$/gu, "").replace(/\.$/u, "").toLowerCase();
|
|
}
|
|
|
|
export function isBlockedIpAddress(address) {
|
|
const family = isIP(address);
|
|
if (family === 4) return blockedAddresses.check(address, "ipv4");
|
|
if (family === 6) return blockedAddresses.check(address, "ipv6");
|
|
return true;
|
|
}
|
|
|
|
export async function resolvePublicHttpUrl(
|
|
value,
|
|
{ lookupImpl = dnsLookup } = {},
|
|
) {
|
|
const url = value instanceof URL ? parseHttpUrl(value.href) : parseHttpUrl(value);
|
|
const hostname = normalizedHostname(url);
|
|
|
|
if (
|
|
hostname === "localhost" ||
|
|
hostname.endsWith(".localhost") ||
|
|
hostname.endsWith(".local") ||
|
|
hostname === "localhost.localdomain"
|
|
) {
|
|
throw new WebsiteEnrichmentError("Lokale Facility-Hosts sind nicht erlaubt.", {
|
|
code: "SSRF_BLOCKED_HOST",
|
|
url: url.href,
|
|
});
|
|
}
|
|
|
|
const literalFamily = isIP(hostname);
|
|
if (literalFamily) {
|
|
if (isBlockedIpAddress(hostname)) {
|
|
throw new WebsiteEnrichmentError("Private oder lokale Facility-IP ist nicht erlaubt.", {
|
|
code: "SSRF_BLOCKED_IP",
|
|
url: url.href,
|
|
});
|
|
}
|
|
return { url, addresses: [{ address: hostname, family: literalFamily }] };
|
|
}
|
|
|
|
let resolved;
|
|
try {
|
|
resolved = await lookupImpl(hostname, { all: true, verbatim: true });
|
|
} catch (error) {
|
|
throw new WebsiteEnrichmentError("DNS-Auflösung der Facility-Website ist fehlgeschlagen.", {
|
|
code: "DNS_FAILED",
|
|
url: url.href,
|
|
cause: error,
|
|
});
|
|
}
|
|
const entries = Array.isArray(resolved) ? resolved : resolved ? [resolved] : [];
|
|
if (entries.length === 0) {
|
|
throw new WebsiteEnrichmentError("Facility-Website hat keine DNS-Adresse.", {
|
|
code: "DNS_EMPTY",
|
|
url: url.href,
|
|
});
|
|
}
|
|
const addresses = [];
|
|
for (const entry of entries) {
|
|
const address = nonEmptyString(typeof entry === "string" ? entry : entry?.address);
|
|
if (!address || isBlockedIpAddress(address)) {
|
|
throw new WebsiteEnrichmentError("DNS verweist auf eine private oder lokale Adresse.", {
|
|
code: "SSRF_BLOCKED_DNS",
|
|
url: url.href,
|
|
});
|
|
}
|
|
addresses.push({ address, family: isIP(address) });
|
|
}
|
|
return { url, addresses };
|
|
}
|
|
|
|
export async function assertPublicHttpUrl(value, options) {
|
|
return (await resolvePublicHttpUrl(value, options)).url;
|
|
}
|
|
|
|
export function createPinnedLookup(addresses) {
|
|
const validated = addresses.map((entry) => {
|
|
const address = nonEmptyString(entry?.address);
|
|
const family = Number(entry?.family) || isIP(address || "");
|
|
if (!address || ![4, 6].includes(family) || isBlockedIpAddress(address)) {
|
|
throw new WebsiteEnrichmentError("Ungültige Adresse für gepinnte Facility-Verbindung.", {
|
|
code: "INVALID_PINNED_ADDRESS",
|
|
});
|
|
}
|
|
return { address, family };
|
|
});
|
|
if (validated.length === 0) {
|
|
throw new WebsiteEnrichmentError("Keine Adresse für gepinnte Facility-Verbindung.", {
|
|
code: "EMPTY_PINNED_ADDRESSES",
|
|
});
|
|
}
|
|
|
|
return (_hostname, options, callback) => {
|
|
const requestedFamily = typeof options === "number" ? options : Number(options?.family) || 0;
|
|
const compatible = requestedFamily
|
|
? validated.filter((entry) => entry.family === requestedFamily)
|
|
: validated;
|
|
if (compatible.length === 0) {
|
|
const error = new Error("Keine validierte Adresse der angeforderten IP-Familie.");
|
|
error.code = "ENETUNREACH";
|
|
callback(error);
|
|
return;
|
|
}
|
|
if (typeof options === "object" && options?.all) {
|
|
callback(null, compatible.map((entry) => ({ ...entry })));
|
|
return;
|
|
}
|
|
callback(null, compatible[0].address, compatible[0].family);
|
|
};
|
|
}
|
|
|
|
function responseHeaders(incoming) {
|
|
const headers = new Headers();
|
|
for (let index = 0; index < incoming.rawHeaders.length; index += 2) {
|
|
headers.append(incoming.rawHeaders[index], incoming.rawHeaders[index + 1]);
|
|
}
|
|
return headers;
|
|
}
|
|
|
|
export async function pinnedFetch(url, init = {}) {
|
|
const parsedUrl = parseHttpUrl(url instanceof URL ? url.href : url);
|
|
const lookup = createPinnedLookup(init.validatedAddresses || []);
|
|
const requestImpl = parsedUrl.protocol === "https:" ? httpsRequest : httpRequest;
|
|
|
|
return new Promise((resolve, reject) => {
|
|
const request = requestImpl(
|
|
parsedUrl,
|
|
{
|
|
method: init.method || "GET",
|
|
headers: init.headers,
|
|
signal: init.signal,
|
|
lookup,
|
|
agent: false,
|
|
},
|
|
(incoming) => {
|
|
const status = incoming.statusCode || 500;
|
|
const bodyForbidden = [101, 204, 205, 304].includes(status);
|
|
const body = bodyForbidden ? null : Readable.toWeb(incoming);
|
|
try {
|
|
resolve(
|
|
new Response(body, {
|
|
status,
|
|
statusText: incoming.statusMessage,
|
|
headers: responseHeaders(incoming),
|
|
}),
|
|
);
|
|
} catch (error) {
|
|
incoming.destroy(error);
|
|
reject(error);
|
|
}
|
|
},
|
|
);
|
|
request.once("error", reject);
|
|
request.end();
|
|
});
|
|
}
|
|
|
|
async function cancelResponseBody(response) {
|
|
try {
|
|
await response.body?.cancel();
|
|
} catch {
|
|
// The response may already be closed. There is nothing else to clean up.
|
|
}
|
|
}
|
|
|
|
export async function readBodyLimited(response, maximumBytes = MAX_HTML_BYTES) {
|
|
const declaredLength = Number(response.headers.get("content-length"));
|
|
if (Number.isFinite(declaredLength) && declaredLength > maximumBytes) {
|
|
await cancelResponseBody(response);
|
|
throw new WebsiteEnrichmentError(`HTML überschreitet das Limit von ${maximumBytes} Bytes.`, {
|
|
code: "BODY_TOO_LARGE",
|
|
});
|
|
}
|
|
|
|
if (!response.body) return "";
|
|
const reader = response.body.getReader();
|
|
const decoder = new TextDecoder("utf-8", { fatal: false });
|
|
let bytesRead = 0;
|
|
let html = "";
|
|
|
|
try {
|
|
while (true) {
|
|
const { done, value } = await reader.read();
|
|
if (done) break;
|
|
bytesRead += value.byteLength;
|
|
if (bytesRead > maximumBytes) {
|
|
await reader.cancel();
|
|
throw new WebsiteEnrichmentError(`HTML überschreitet das Limit von ${maximumBytes} Bytes.`, {
|
|
code: "BODY_TOO_LARGE",
|
|
});
|
|
}
|
|
html += decoder.decode(value, { stream: true });
|
|
}
|
|
html += decoder.decode();
|
|
return html;
|
|
} finally {
|
|
reader.releaseLock();
|
|
}
|
|
}
|
|
|
|
export function createHostLimiter({
|
|
delayMs = DEFAULT_HOST_DELAY_MS,
|
|
sleepImpl = sleep,
|
|
now = Date.now,
|
|
} = {}) {
|
|
const tails = new Map();
|
|
const lastStartedAt = new Map();
|
|
|
|
return async (url) => {
|
|
const hostname = normalizedHostname(url instanceof URL ? url : new URL(url));
|
|
const previous = tails.get(hostname) ?? Promise.resolve();
|
|
let release;
|
|
const gate = new Promise((resolve) => {
|
|
release = resolve;
|
|
});
|
|
tails.set(hostname, previous.catch(() => undefined).then(() => gate));
|
|
|
|
await previous.catch(() => undefined);
|
|
try {
|
|
const lastStart = lastStartedAt.get(hostname);
|
|
const waitFor = lastStart === undefined ? 0 : Math.max(0, lastStart + delayMs - now());
|
|
if (waitFor > 0) await sleepImpl(waitFor);
|
|
lastStartedAt.set(hostname, now());
|
|
} finally {
|
|
release();
|
|
}
|
|
};
|
|
}
|
|
|
|
async function withTimeout(promise, timeoutMs, errorFactory) {
|
|
let timeout;
|
|
try {
|
|
return await Promise.race([
|
|
promise,
|
|
new Promise((_resolve, reject) => {
|
|
timeout = setTimeout(() => reject(errorFactory()), timeoutMs);
|
|
}),
|
|
]);
|
|
} finally {
|
|
clearTimeout(timeout);
|
|
}
|
|
}
|
|
|
|
export async function fetchHtmlPage(
|
|
initialUrl,
|
|
{
|
|
fetchImpl = pinnedFetch,
|
|
lookupImpl = dnsLookup,
|
|
beforeRequest = async () => {},
|
|
timeoutMs = DEFAULT_TIMEOUT_MS,
|
|
maximumBytes = MAX_HTML_BYTES,
|
|
maxRedirects = MAX_REDIRECTS,
|
|
userAgent = USER_AGENT,
|
|
} = {},
|
|
) {
|
|
if (typeof fetchImpl !== "function") throw new Error("Keine Fetch-Implementierung verfügbar.");
|
|
let currentUrl = parseHttpUrl(initialUrl);
|
|
const bodyLimit = Math.max(1, Math.min(MAX_HTML_BYTES, Number(maximumBytes) || MAX_HTML_BYTES));
|
|
|
|
for (let redirects = 0; redirects <= maxRedirects; redirects += 1) {
|
|
const resolved = await withTimeout(
|
|
resolvePublicHttpUrl(currentUrl, { lookupImpl }),
|
|
timeoutMs,
|
|
() =>
|
|
new WebsiteEnrichmentError(`DNS-Auflösung nach ${timeoutMs} ms abgebrochen.`, {
|
|
code: "DNS_TIMEOUT",
|
|
url: currentUrl.href,
|
|
}),
|
|
);
|
|
currentUrl = resolved.url;
|
|
await beforeRequest(currentUrl);
|
|
|
|
const controller = new AbortController();
|
|
const timeout = setTimeout(
|
|
() => controller.abort(new Error(`Facility-Website nach ${timeoutMs} ms abgebrochen.`)),
|
|
timeoutMs,
|
|
);
|
|
|
|
try {
|
|
const response = await fetchImpl(currentUrl, {
|
|
method: "GET",
|
|
redirect: "manual",
|
|
headers: {
|
|
Accept: "text/html,application/xhtml+xml;q=0.9",
|
|
"Accept-Encoding": "identity",
|
|
"User-Agent": userAgent,
|
|
},
|
|
signal: controller.signal,
|
|
validatedAddresses: resolved.addresses,
|
|
});
|
|
|
|
if (REDIRECT_STATUSES.has(response.status)) {
|
|
const location = response.headers.get("location");
|
|
await cancelResponseBody(response);
|
|
if (!location) {
|
|
throw new WebsiteEnrichmentError("Facility-Redirect enthält kein Location-Ziel.", {
|
|
code: "REDIRECT_WITHOUT_LOCATION",
|
|
url: currentUrl.href,
|
|
});
|
|
}
|
|
if (redirects === maxRedirects) {
|
|
throw new WebsiteEnrichmentError("Zu viele Facility-Redirects.", {
|
|
code: "TOO_MANY_REDIRECTS",
|
|
url: currentUrl.href,
|
|
});
|
|
}
|
|
currentUrl = parseHttpUrl(location, currentUrl);
|
|
continue;
|
|
}
|
|
|
|
if (!response.ok) {
|
|
await cancelResponseBody(response);
|
|
throw new WebsiteEnrichmentError(`Facility-Website antwortete mit HTTP ${response.status}.`, {
|
|
code: "HTTP_ERROR",
|
|
url: currentUrl.href,
|
|
status: response.status,
|
|
});
|
|
}
|
|
|
|
const contentType = response.headers.get("content-type")?.split(";", 1)[0].trim().toLowerCase();
|
|
if (!(["text/html", "application/xhtml+xml"].includes(contentType))) {
|
|
await cancelResponseBody(response);
|
|
throw new WebsiteEnrichmentError("Facility-Antwort ist kein HTML-Dokument.", {
|
|
code: "UNSUPPORTED_CONTENT_TYPE",
|
|
url: currentUrl.href,
|
|
});
|
|
}
|
|
|
|
const contentEncoding = response.headers.get("content-encoding")?.trim().toLowerCase();
|
|
if (contentEncoding && contentEncoding !== "identity") {
|
|
await cancelResponseBody(response);
|
|
throw new WebsiteEnrichmentError("Komprimierte Facility-Antwort trotz identity-Anforderung abgelehnt.", {
|
|
code: "UNSUPPORTED_CONTENT_ENCODING",
|
|
url: currentUrl.href,
|
|
});
|
|
}
|
|
|
|
const html = await readBodyLimited(response, bodyLimit);
|
|
return { html, finalUrl: currentUrl.href, redirects, status: response.status };
|
|
} catch (error) {
|
|
if (error instanceof WebsiteEnrichmentError) throw error;
|
|
throw new WebsiteEnrichmentError(`Facility-Website konnte nicht geladen werden: ${error.message}`, {
|
|
code: "FETCH_FAILED",
|
|
url: currentUrl.href,
|
|
cause: error,
|
|
});
|
|
} finally {
|
|
clearTimeout(timeout);
|
|
}
|
|
}
|
|
|
|
throw new WebsiteEnrichmentError("Zu viele Facility-Redirects.", {
|
|
code: "TOO_MANY_REDIRECTS",
|
|
url: currentUrl.href,
|
|
});
|
|
}
|
|
|
|
function decodeHtmlEntities(value) {
|
|
const named = {
|
|
amp: "&",
|
|
apos: "'",
|
|
colon: ":",
|
|
commat: "@",
|
|
gt: ">",
|
|
lt: "<",
|
|
nbsp: " ",
|
|
quot: '"',
|
|
};
|
|
return String(value)
|
|
.replace(/&#(x[0-9a-f]+|\d+);?/giu, (_match, encoded) => {
|
|
const radix = encoded[0].toLowerCase() === "x" ? 16 : 10;
|
|
const number = Number.parseInt(radix === 16 ? encoded.slice(1) : encoded, radix);
|
|
const validCodePoint =
|
|
Number.isFinite(number) &&
|
|
number >= 0 &&
|
|
number <= 0x10ffff &&
|
|
!(number >= 0xd800 && number <= 0xdfff);
|
|
return validCodePoint ? String.fromCodePoint(number) : "\ufffd";
|
|
})
|
|
.replace(/&([a-z]+);/giu, (match, name) => named[name.toLowerCase()] ?? match);
|
|
}
|
|
|
|
function decodedUriValue(value) {
|
|
const decoded = decodeHtmlEntities(value).trim();
|
|
try {
|
|
return decodeURIComponent(decoded);
|
|
} catch {
|
|
return decoded;
|
|
}
|
|
}
|
|
|
|
function normalizedPhone(value) {
|
|
let phone = nonEmptyString(value);
|
|
if (!phone) return null;
|
|
phone = phone.replace(/^tel:/iu, "").split(/[;?]/u, 1)[0].trim();
|
|
if (phone.length < 3 || phone.length > 64 || !/\d/u.test(phone)) return null;
|
|
if (!/^[+\d().\-/\s]+$/u.test(phone)) return null;
|
|
return phone.replace(/^00(?=\d)/u, "+").replace(/\s+/gu, " ");
|
|
}
|
|
|
|
function normalizedEmail(value) {
|
|
let email = nonEmptyString(value);
|
|
if (!email) return null;
|
|
email = email.replace(/^mailto:/iu, "").split(/[?,]/u, 1)[0].trim();
|
|
if (email.length > 254 || /[\u0000-\u001f\u007f]/u.test(email)) return null;
|
|
return /^[^\s@]+@[^\s@]+\.[^\s@]+$/u.test(email) ? email : null;
|
|
}
|
|
|
|
function normalizedLabel(value) {
|
|
const raw = nonEmptyString(value);
|
|
if (!raw || /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/u.test(raw)) return null;
|
|
const label = raw.replace(/\s+/gu, " ");
|
|
if (!label || label.length > 240 || /[<>]/u.test(label)) return null;
|
|
return label;
|
|
}
|
|
|
|
function valuesOf(value) {
|
|
return Array.isArray(value) ? value : value === undefined || value === null ? [] : [value];
|
|
}
|
|
|
|
function contactPoints(node) {
|
|
return valuesOf(node?.contactPoint).filter((value) => value && typeof value === "object");
|
|
}
|
|
|
|
function structuredPhone(node) {
|
|
for (const value of [
|
|
...valuesOf(node?.telephone),
|
|
...contactPoints(node).flatMap((point) => valuesOf(point.telephone)),
|
|
]) {
|
|
const phone = normalizedPhone(value);
|
|
if (phone) return phone;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function structuredEmail(node) {
|
|
for (const value of [
|
|
...valuesOf(node?.email),
|
|
...contactPoints(node).flatMap((point) => valuesOf(point.email)),
|
|
]) {
|
|
const email = normalizedEmail(value);
|
|
if (email) return email;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function structuredAddress(node) {
|
|
for (const value of valuesOf(node?.address)) {
|
|
if (typeof value === "string") {
|
|
const address = normalizedLabel(value);
|
|
if (address) return address;
|
|
continue;
|
|
}
|
|
if (!value || typeof value !== "object") continue;
|
|
const locality = [value.postalCode, value.addressLocality].map(normalizedLabel).filter(Boolean).join(" ");
|
|
const address = [
|
|
normalizedLabel(value.streetAddress),
|
|
locality || null,
|
|
normalizedLabel(value.addressRegion),
|
|
normalizedLabel(
|
|
typeof value.addressCountry === "object" ? value.addressCountry?.name : value.addressCountry,
|
|
),
|
|
]
|
|
.filter(Boolean)
|
|
.join(", ");
|
|
if (address) return address;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function structuredTypePriority(node) {
|
|
const types = valuesOf(node?.["@type"]).map((value) => String(value).toLowerCase());
|
|
if (types.some((type) => /marina|localbusiness|governmentorganization|civicstructure/u.test(type))) return 3;
|
|
if (types.some((type) => /organization|place|corporation|ngo/u.test(type))) return 2;
|
|
return 1;
|
|
}
|
|
|
|
function jsonLdNodes(value) {
|
|
if (Array.isArray(value)) return value.flatMap(jsonLdNodes);
|
|
if (!value || typeof value !== "object") return [];
|
|
const graph = Array.isArray(value["@graph"]) ? value["@graph"].flatMap(jsonLdNodes) : [];
|
|
return [value, ...graph];
|
|
}
|
|
|
|
function extractStructuredContacts(html) {
|
|
const nodes = [];
|
|
const scriptPattern = /<script\b([^>]*)>([\s\S]*?)<\/script\s*>/giu;
|
|
for (const match of html.matchAll(scriptPattern)) {
|
|
const scriptType = (htmlAttribute(match[1], "type") ?? "")
|
|
.split(";", 1)[0]
|
|
.trim()
|
|
.toLowerCase();
|
|
if (scriptType !== "application/ld+json") {
|
|
continue;
|
|
}
|
|
const raw = match[2].trim();
|
|
if (!raw) continue;
|
|
let parsed;
|
|
try {
|
|
parsed = JSON.parse(raw);
|
|
} catch {
|
|
try {
|
|
parsed = JSON.parse(decodeHtmlEntities(raw));
|
|
} catch {
|
|
continue;
|
|
}
|
|
}
|
|
nodes.push(...jsonLdNodes(parsed));
|
|
}
|
|
|
|
const candidates = nodes
|
|
.map((node) => ({
|
|
node,
|
|
phone: structuredPhone(node),
|
|
email: structuredEmail(node),
|
|
address: structuredAddress(node),
|
|
}))
|
|
.filter((candidate) => candidate.phone || candidate.email || candidate.address)
|
|
.sort((left, right) => structuredTypePriority(right.node) - structuredTypePriority(left.node));
|
|
|
|
const result = {};
|
|
for (const candidate of candidates) {
|
|
result.phone ??= candidate.phone;
|
|
result.email ??= candidate.email;
|
|
result.address ??= candidate.address;
|
|
if (!result.operator) {
|
|
result.operator = normalizedLabel(candidate.node.legalName) ?? normalizedLabel(candidate.node.name);
|
|
}
|
|
}
|
|
if (!result.operator) {
|
|
const namedFacility = nodes
|
|
.filter((node) => structuredTypePriority(node) >= 2)
|
|
.sort((left, right) => structuredTypePriority(right) - structuredTypePriority(left))
|
|
.find((node) => normalizedLabel(node.legalName) ?? normalizedLabel(node.name));
|
|
result.operator = namedFacility
|
|
? normalizedLabel(namedFacility.legalName) ?? normalizedLabel(namedFacility.name)
|
|
: null;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function htmlAttribute(attributes, requestedName) {
|
|
let index = 0;
|
|
while (index < attributes.length) {
|
|
while (/\s/u.test(attributes[index] || "")) index += 1;
|
|
const nameStart = index;
|
|
while (index < attributes.length && !/[\s=/>]/u.test(attributes[index])) index += 1;
|
|
const name = attributes.slice(nameStart, index).toLowerCase();
|
|
if (!name) {
|
|
index += 1;
|
|
continue;
|
|
}
|
|
while (/\s/u.test(attributes[index] || "")) index += 1;
|
|
|
|
let value = "";
|
|
if (attributes[index] === "=") {
|
|
index += 1;
|
|
while (/\s/u.test(attributes[index] || "")) index += 1;
|
|
const quote = attributes[index] === '"' || attributes[index] === "'" ? attributes[index] : null;
|
|
if (quote) {
|
|
index += 1;
|
|
const valueStart = index;
|
|
while (index < attributes.length && attributes[index] !== quote) index += 1;
|
|
value = attributes.slice(valueStart, index);
|
|
if (attributes[index] === quote) index += 1;
|
|
} else {
|
|
const valueStart = index;
|
|
while (index < attributes.length && !/[\s>]/u.test(attributes[index])) index += 1;
|
|
value = attributes.slice(valueStart, index);
|
|
}
|
|
}
|
|
if (name === requestedName.toLowerCase()) return value;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function extractLinkedContacts(html) {
|
|
const result = {};
|
|
const anchorPattern = /<a\b([^>]*)>/giu;
|
|
for (const anchor of html.matchAll(anchorPattern)) {
|
|
const href = decodedUriValue(htmlAttribute(anchor[1], "href") ?? "");
|
|
if (!result.phone && /^tel:/iu.test(href)) result.phone = normalizedPhone(href);
|
|
if (!result.email && /^mailto:/iu.test(href)) result.email = normalizedEmail(href);
|
|
if (result.phone && result.email) break;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
export function extractContactsFromHtml(html) {
|
|
const structured = extractStructuredContacts(String(html));
|
|
const linked = extractLinkedContacts(String(html));
|
|
return {
|
|
phone: structured.phone ?? linked.phone ?? null,
|
|
email: structured.email ?? linked.email ?? null,
|
|
operator: structured.operator ?? null,
|
|
address: structured.address ?? null,
|
|
};
|
|
}
|
|
|
|
function normalizedAddress(properties) {
|
|
const explicit = firstString(properties, ADDRESS_KEYS);
|
|
if (explicit) return explicit;
|
|
const locality = [properties?.["addr:postcode"], properties?.["addr:city"]]
|
|
.map(nonEmptyString)
|
|
.filter(Boolean)
|
|
.join(" ");
|
|
const street = [properties?.["addr:street"], properties?.["addr:housenumber"]]
|
|
.map(nonEmptyString)
|
|
.filter(Boolean)
|
|
.join(" ");
|
|
return [street, locality, nonEmptyString(properties?.["addr:country"])]
|
|
.filter(Boolean)
|
|
.join(", ") || null;
|
|
}
|
|
|
|
export function existingContacts(properties = {}) {
|
|
return {
|
|
phone: firstString(properties, PHONE_KEYS),
|
|
email: firstString(properties, EMAIL_KEYS),
|
|
operator: firstString(properties, OPERATOR_KEYS),
|
|
address: normalizedAddress(properties),
|
|
};
|
|
}
|
|
|
|
export function candidateContacts(candidate) {
|
|
const original = existingContacts(candidate?.properties);
|
|
const previousEnrichment = existingContacts(candidate?.enrichmentProperties);
|
|
return Object.fromEntries(
|
|
Object.keys(original).map((key) => [key, original[key] ?? previousEnrichment[key]]),
|
|
);
|
|
}
|
|
|
|
export function linkedWebsite(properties = {}) {
|
|
for (const key of WEBSITE_KEYS) {
|
|
const raw = nonEmptyString(properties[key]);
|
|
if (!raw) continue;
|
|
const candidates = raw.split(/\s*;\s*(?=https?:\/\/)/iu);
|
|
for (const candidate of candidates) {
|
|
try {
|
|
return { key, original: raw, url: parseHttpUrl(candidate).href };
|
|
} catch {
|
|
// Try another explicitly linked URL field; never invent or search for one.
|
|
}
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
export function stableWebsiteSourceId(candidate) {
|
|
const source = nonEmptyString(candidate?.source);
|
|
const sourceId = nonEmptyString(candidate?.sourceId);
|
|
if (!source || !sourceId) return null;
|
|
return `${source}:${sourceId}`;
|
|
}
|
|
|
|
export function buildEnrichmentRecord({ candidate, website, page, extracted, fetchedAt }) {
|
|
const sourceId = stableWebsiteSourceId(candidate);
|
|
if (!sourceId) return null;
|
|
const current = candidateContacts(candidate);
|
|
const addedFields = [];
|
|
const contacts = { ...current };
|
|
|
|
for (const key of ["phone", "email", "operator", "address"]) {
|
|
if (!contacts[key] && extracted[key]) {
|
|
contacts[key] = extracted[key];
|
|
addedFields.push(key);
|
|
}
|
|
}
|
|
if (addedFields.length === 0) return null;
|
|
|
|
return {
|
|
originalId: candidate.id,
|
|
sourceId,
|
|
name: nonEmptyString(candidate.name) ?? nonEmptyString(candidate.properties?.name),
|
|
properties: {
|
|
name: nonEmptyString(candidate.name) ?? nonEmptyString(candidate.properties?.name),
|
|
website: website.url,
|
|
phone: contacts.phone,
|
|
"contact:phone": contacts.phone,
|
|
email: contacts.email,
|
|
"contact:email": contacts.email,
|
|
operator: contacts.operator,
|
|
address: contacts.address,
|
|
original_source: candidate.source,
|
|
original_source_id: candidate.sourceId,
|
|
enrichmentSource: "facility-website",
|
|
enrichmentSourceUrl: page.finalUrl,
|
|
sourceUrl: page.finalUrl,
|
|
source_url: page.finalUrl,
|
|
fetchedAt,
|
|
fetched_at: fetchedAt,
|
|
enriched_fields: addedFields,
|
|
},
|
|
};
|
|
}
|
|
|
|
export async function mapWithConcurrency(items, concurrency, mapper) {
|
|
const results = new Array(items.length);
|
|
let nextIndex = 0;
|
|
const workerCount = Math.max(1, Math.min(items.length || 1, concurrency));
|
|
await Promise.all(
|
|
Array.from({ length: workerCount }, async () => {
|
|
while (true) {
|
|
const index = nextIndex;
|
|
nextIndex += 1;
|
|
if (index >= items.length) return;
|
|
results[index] = await mapper(items[index], index);
|
|
}
|
|
}),
|
|
);
|
|
return results;
|
|
}
|
|
|
|
export async function enrichCandidates(
|
|
candidates,
|
|
{
|
|
fetchImpl = pinnedFetch,
|
|
lookupImpl = dnsLookup,
|
|
concurrency = DEFAULT_CONCURRENCY,
|
|
hostDelayMs = DEFAULT_HOST_DELAY_MS,
|
|
timeoutMs = DEFAULT_TIMEOUT_MS,
|
|
maximumBytes = MAX_HTML_BYTES,
|
|
maxRedirects = MAX_REDIRECTS,
|
|
sleepImpl = sleep,
|
|
now = Date.now,
|
|
fetchedAt = new Date().toISOString(),
|
|
logger = console,
|
|
} = {},
|
|
) {
|
|
const waitForHost = createHostLimiter({ delayMs: hostDelayMs, sleepImpl, now });
|
|
const pageCache = new Map();
|
|
const stats = {
|
|
candidates: candidates.length,
|
|
uniquePages: 0,
|
|
enriched: 0,
|
|
alreadyComplete: 0,
|
|
invalidWebsite: 0,
|
|
noContactsFound: 0,
|
|
failed: 0,
|
|
fetchedAt,
|
|
};
|
|
|
|
const effectiveConcurrency = Math.max(1, Math.min(MAX_CONCURRENCY, Number(concurrency) || 1));
|
|
const records = await mapWithConcurrency(candidates, effectiveConcurrency, async (candidate) => {
|
|
const website = linkedWebsite(candidate.properties);
|
|
if (!website) {
|
|
stats.invalidWebsite += 1;
|
|
return null;
|
|
}
|
|
if (Object.values(candidateContacts(candidate)).every(Boolean)) {
|
|
stats.alreadyComplete += 1;
|
|
return null;
|
|
}
|
|
|
|
let pagePromise = pageCache.get(website.url);
|
|
if (!pagePromise) {
|
|
pagePromise = fetchHtmlPage(website.url, {
|
|
fetchImpl,
|
|
lookupImpl,
|
|
beforeRequest: waitForHost,
|
|
timeoutMs,
|
|
maximumBytes,
|
|
maxRedirects,
|
|
});
|
|
pageCache.set(website.url, pagePromise);
|
|
stats.uniquePages += 1;
|
|
}
|
|
|
|
try {
|
|
const page = await pagePromise;
|
|
const extracted = extractContactsFromHtml(page.html);
|
|
const record = buildEnrichmentRecord({ candidate, website, page, extracted, fetchedAt });
|
|
if (!record) {
|
|
stats.noContactsFound += 1;
|
|
return null;
|
|
}
|
|
stats.enriched += 1;
|
|
return record;
|
|
} catch (error) {
|
|
stats.failed += 1;
|
|
logger.warn?.(
|
|
`Website-Anreicherung für ${candidate.source}:${candidate.sourceId} fehlgeschlagen: ${error.message}`,
|
|
);
|
|
return null;
|
|
}
|
|
});
|
|
|
|
return { records: records.filter(Boolean), stats };
|
|
}
|
|
|
|
export async function loadWebsiteCandidates({
|
|
databaseUrl = process.env.DATABASE_URL,
|
|
limit = DEFAULT_LIMIT,
|
|
} = {}) {
|
|
if (!databaseUrl) throw new Error("DATABASE_URL ist für die Website-Anreicherung erforderlich.");
|
|
const { default: pg } = await import("pg");
|
|
const client = new pg.Client({ connectionString: databaseUrl });
|
|
await client.connect();
|
|
|
|
try {
|
|
const result = await client.query(
|
|
`
|
|
SELECT
|
|
original.id,
|
|
original.layer,
|
|
original.source,
|
|
original.source_id AS "sourceId",
|
|
original.name,
|
|
original.properties,
|
|
enrichment.properties AS "enrichmentProperties"
|
|
FROM marine_features AS original
|
|
LEFT JOIN marine_features AS enrichment
|
|
ON enrichment.source = 'facility-website'
|
|
AND enrichment.layer = original.layer
|
|
AND enrichment.source_id = original.source || ':' || original.source_id
|
|
WHERE original.layer IN ('locks', 'harbours')
|
|
AND original.source IN ('osm', 'euris')
|
|
AND original.source_id IS NOT NULL
|
|
AND COALESCE(
|
|
NULLIF(BTRIM(original.properties->>'website'), ''),
|
|
NULLIF(BTRIM(original.properties->>'contact:website'), ''),
|
|
NULLIF(BTRIM(original.properties->>'url'), '')
|
|
) IS NOT NULL
|
|
AND (
|
|
COALESCE(
|
|
NULLIF(BTRIM(original.properties->>'phone'), ''),
|
|
NULLIF(BTRIM(original.properties->>'contact:phone'), ''),
|
|
NULLIF(BTRIM(enrichment.properties->>'phone'), ''),
|
|
NULLIF(BTRIM(enrichment.properties->>'contact:phone'), '')
|
|
) IS NULL
|
|
OR COALESCE(
|
|
NULLIF(BTRIM(original.properties->>'email'), ''),
|
|
NULLIF(BTRIM(original.properties->>'contact:email'), ''),
|
|
NULLIF(BTRIM(enrichment.properties->>'email'), ''),
|
|
NULLIF(BTRIM(enrichment.properties->>'contact:email'), '')
|
|
) IS NULL
|
|
OR COALESCE(
|
|
NULLIF(BTRIM(original.properties->>'operator'), ''),
|
|
NULLIF(BTRIM(original.properties->>'operator:name'), ''),
|
|
NULLIF(BTRIM(original.properties->>'owner'), ''),
|
|
NULLIF(BTRIM(enrichment.properties->>'operator'), '')
|
|
) IS NULL
|
|
OR COALESCE(
|
|
NULLIF(BTRIM(original.properties->>'address'), ''),
|
|
NULLIF(BTRIM(original.properties->>'contact:address'), ''),
|
|
NULLIF(BTRIM(original.properties->>'addr:full'), ''),
|
|
NULLIF(BTRIM(original.properties->>'addr:street'), ''),
|
|
NULLIF(BTRIM(original.properties->>'addr:city'), ''),
|
|
NULLIF(BTRIM(enrichment.properties->>'address'), ''),
|
|
NULLIF(BTRIM(enrichment.properties->>'contact:address'), ''),
|
|
NULLIF(BTRIM(enrichment.properties->>'addr:full'), '')
|
|
) IS NULL
|
|
)
|
|
ORDER BY
|
|
(enrichment.id IS NULL) DESC,
|
|
CASE WHEN enrichment.id IS NULL THEN random() ELSE 0 END,
|
|
enrichment.updated_at ASC NULLS FIRST,
|
|
original.id
|
|
LIMIT $1
|
|
`,
|
|
[limit],
|
|
);
|
|
return result.rows;
|
|
} finally {
|
|
await client.end();
|
|
}
|
|
}
|
|
|
|
export async function writeWebsiteEnrichments(
|
|
records,
|
|
{ databaseUrl = process.env.DATABASE_URL } = {},
|
|
) {
|
|
if (!databaseUrl) throw new Error("DATABASE_URL ist für die Website-Anreicherung erforderlich.");
|
|
if (records.length === 0) return 0;
|
|
const { default: pg } = await import("pg");
|
|
const client = new pg.Client({ connectionString: databaseUrl });
|
|
await client.connect();
|
|
|
|
try {
|
|
await client.query("BEGIN");
|
|
const values = [];
|
|
const rows = records.map((record, index) => {
|
|
const position = index * 5;
|
|
values.push(
|
|
record.originalId,
|
|
record.sourceId,
|
|
record.name,
|
|
JSON.stringify(record.properties),
|
|
record.properties.fetchedAt,
|
|
);
|
|
return `(\$${position + 1}::bigint, \$${position + 2}, \$${position + 3}, \$${position + 4}::jsonb, \$${position + 5}::timestamptz)`;
|
|
});
|
|
|
|
const result = await client.query(
|
|
`
|
|
INSERT INTO marine_features (layer, source, source_id, name, properties, geom, updated_at)
|
|
SELECT
|
|
original.layer,
|
|
'facility-website',
|
|
incoming.source_id,
|
|
incoming.name,
|
|
jsonb_strip_nulls(incoming.properties),
|
|
original.geom,
|
|
incoming.updated_at
|
|
FROM (VALUES ${rows.join(",")}) AS incoming
|
|
(original_id, source_id, name, properties, updated_at)
|
|
JOIN marine_features AS original ON original.id = incoming.original_id
|
|
ON CONFLICT (source, source_id, layer) WHERE source_id IS NOT NULL
|
|
DO UPDATE SET
|
|
name = EXCLUDED.name,
|
|
properties = marine_features.properties || jsonb_strip_nulls(EXCLUDED.properties),
|
|
geom = EXCLUDED.geom,
|
|
updated_at = EXCLUDED.updated_at
|
|
`,
|
|
values,
|
|
);
|
|
await client.query("COMMIT");
|
|
return result.rowCount;
|
|
} catch (error) {
|
|
await client.query("ROLLBACK");
|
|
throw error;
|
|
} finally {
|
|
await client.end();
|
|
}
|
|
}
|
|
|
|
export async function runWebsiteEnrichment({
|
|
dryRun = true,
|
|
limit = DEFAULT_LIMIT,
|
|
databaseUrl,
|
|
candidates,
|
|
loader = loadWebsiteCandidates,
|
|
writer = writeWebsiteEnrichments,
|
|
enrichmentOptions,
|
|
} = {}) {
|
|
const selectedCandidates = candidates ?? await loader({ databaseUrl, limit });
|
|
const result = await enrichCandidates(selectedCandidates.slice(0, limit), enrichmentOptions);
|
|
if (!dryRun) await writer(result.records, { databaseUrl });
|
|
return { ...result, dryRun };
|
|
}
|
|
|
|
async function main() {
|
|
const dryRun = parseBooleanDefault(process.env.MARINE_WEBSITE_DRY_RUN, true);
|
|
const limit = boundedInteger(process.env.MARINE_WEBSITE_LIMIT, DEFAULT_LIMIT, 1, MAX_LIMIT, "MARINE_WEBSITE_LIMIT");
|
|
const concurrency = boundedInteger(
|
|
process.env.MARINE_WEBSITE_CONCURRENCY,
|
|
DEFAULT_CONCURRENCY,
|
|
1,
|
|
MAX_CONCURRENCY,
|
|
"MARINE_WEBSITE_CONCURRENCY",
|
|
);
|
|
const hostDelayMs = boundedInteger(
|
|
process.env.MARINE_WEBSITE_HOST_DELAY_MS,
|
|
DEFAULT_HOST_DELAY_MS,
|
|
250,
|
|
60_000,
|
|
"MARINE_WEBSITE_HOST_DELAY_MS",
|
|
);
|
|
const timeoutMs = boundedInteger(
|
|
process.env.MARINE_WEBSITE_TIMEOUT_MS,
|
|
DEFAULT_TIMEOUT_MS,
|
|
1_000,
|
|
60_000,
|
|
"MARINE_WEBSITE_TIMEOUT_MS",
|
|
);
|
|
|
|
console.log(
|
|
`Website-Anreicherung gestartet: Limit ${limit}, Parallelität ${concurrency}${dryRun ? " (Trockenlauf)" : ""}.`,
|
|
);
|
|
const result = await runWebsiteEnrichment({
|
|
dryRun,
|
|
limit,
|
|
databaseUrl: process.env.DATABASE_URL,
|
|
enrichmentOptions: { concurrency, hostDelayMs, timeoutMs },
|
|
});
|
|
console.log(JSON.stringify(result.stats, null, 2));
|
|
console.log(
|
|
dryRun
|
|
? `${result.records.length} Ergänzungen validiert; die Datenbank wurde nicht verändert.`
|
|
: `${result.records.length} Facility-Website-Ergänzungen gespeichert.`,
|
|
);
|
|
}
|
|
|
|
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
|
main().catch((error) => {
|
|
console.error(error);
|
|
process.exitCode = 1;
|
|
});
|
|
}
|