1624 lines
51 KiB
JavaScript
1624 lines
51 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
import { createHash } from "node:crypto";
|
|
import { pathToFileURL } from "node:url";
|
|
|
|
import {
|
|
DEFAULT_TIMEOUT_MS,
|
|
MAX_HTML_BYTES,
|
|
candidateContacts,
|
|
createHostLimiter,
|
|
extractContactsFromHtml,
|
|
fetchHtmlPage,
|
|
linkedWebsite,
|
|
mapWithConcurrency,
|
|
parseBooleanDefault,
|
|
readBodyLimited,
|
|
stableWebsiteSourceId,
|
|
} from "./enrich-marine-websites.mjs";
|
|
|
|
export const DEFAULT_SEARCH_LIMIT = 10;
|
|
export const DEFAULT_SEARCH_CONCURRENCY = 1;
|
|
export const DEFAULT_SEARCH_DELAY_MS = 5_000;
|
|
export const DEFAULT_SITE_DELAY_MS = 1_000;
|
|
export const DEFAULT_SEARCH_RESULTS = 5;
|
|
export const DEFAULT_SEARCH_PAGES = 3;
|
|
export const DEFAULT_SEARCH_SCORE = 70;
|
|
export const DEFAULT_PAGE_SCORE = 65;
|
|
export const DEFAULT_SCORE_MARGIN = 15;
|
|
export const DUCKDUCKGO_HTML_ENDPOINT = "https://html.duckduckgo.com/html/";
|
|
export const BRAVE_SEARCH_ENDPOINT = "https://api.search.brave.com/res/v1/web/search";
|
|
export const SEARCH_USER_AGENT = "Watermaps/0.1 marine-facility-search-enricher";
|
|
|
|
const MAX_SEARCH_LIMIT = 100;
|
|
const MAX_SEARCH_CONCURRENCY = 2;
|
|
const MAX_SEARCH_RESULTS = 10;
|
|
const MAX_SEARCH_PAGES = 5;
|
|
const MAX_SEARCH_RESPONSE_BYTES = 1024 * 1024;
|
|
const ATTEMPT_STATUSES = new Set([
|
|
"success",
|
|
"no_match",
|
|
"ambiguous",
|
|
"no_contacts",
|
|
"fetch_failed",
|
|
"provider_blocked",
|
|
"invalid_candidate",
|
|
]);
|
|
|
|
const GENERIC_WORDS = new Set([
|
|
"am",
|
|
"an",
|
|
"anlage",
|
|
"binnenhafen",
|
|
"boat",
|
|
"boot",
|
|
"brug",
|
|
"club",
|
|
"de",
|
|
"der",
|
|
"die",
|
|
"dock",
|
|
"e",
|
|
"ev",
|
|
"gate",
|
|
"haven",
|
|
"hafen",
|
|
"harbour",
|
|
"im",
|
|
"jachthaven",
|
|
"kanaal",
|
|
"kanal",
|
|
"lock",
|
|
"lockgate",
|
|
"marina",
|
|
"noord",
|
|
"ost",
|
|
"port",
|
|
"schleuse",
|
|
"seaport",
|
|
"sluis",
|
|
"sportboothaven",
|
|
"steg",
|
|
"sud",
|
|
"sued",
|
|
"tor",
|
|
"und",
|
|
"van",
|
|
"verein",
|
|
"voor",
|
|
"water",
|
|
"west",
|
|
"yacht",
|
|
"yachthafen",
|
|
]);
|
|
|
|
const BLOCKED_RESULT_HOSTS = [
|
|
/(^|\.)duckduckgo\.com$/u,
|
|
/(^|\.)facebook\.com$/u,
|
|
/(^|\.)instagram\.com$/u,
|
|
/(^|\.)linkedin\.com$/u,
|
|
/(^|\.)marinas\.com$/u,
|
|
/(^|\.)tripadvisor\./u,
|
|
/(^|\.)wikipedia\.org$/u,
|
|
/(^|\.)youtube\.com$/u,
|
|
];
|
|
|
|
function nonEmptyString(value) {
|
|
if (typeof value !== "string" && typeof value !== "number") return null;
|
|
const normalized = String(value).trim();
|
|
return normalized || null;
|
|
}
|
|
|
|
function boundedInteger(value, fallback, minimum, maximum, name) {
|
|
if (value === undefined || value === null || value === "") return fallback;
|
|
if (!/^\d+$/u.test(String(value).trim())) {
|
|
throw new Error(`${name} muss eine Ganzzahl sein.`);
|
|
}
|
|
const parsed = Number(value);
|
|
if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) {
|
|
throw new Error(`${name} muss zwischen ${minimum} und ${maximum} liegen.`);
|
|
}
|
|
return parsed;
|
|
}
|
|
|
|
function normalizedMatchText(value) {
|
|
return String(value ?? "")
|
|
.normalize("NFKD")
|
|
.replace(/\p{M}+/gu, "")
|
|
.replace(/ß/gu, "ss")
|
|
.toLowerCase()
|
|
.replace(/[^a-z0-9]+/gu, " ")
|
|
.trim()
|
|
.replace(/\s+/gu, " ");
|
|
}
|
|
|
|
function words(value) {
|
|
const normalized = normalizedMatchText(value);
|
|
return normalized ? normalized.split(" ") : [];
|
|
}
|
|
|
|
function unique(values) {
|
|
return [...new Set(values.filter(Boolean))];
|
|
}
|
|
|
|
function decodeHtmlEntities(value) {
|
|
const named = {
|
|
aacute: "á",
|
|
amp: "&",
|
|
apos: "'",
|
|
auml: "ä",
|
|
ccedil: "ç",
|
|
colon: ":",
|
|
commat: "@",
|
|
eacute: "é",
|
|
egrave: "è",
|
|
gt: ">",
|
|
iacute: "í",
|
|
lt: "<",
|
|
nbsp: " ",
|
|
oacute: "ó",
|
|
ouml: "ö",
|
|
quot: '"',
|
|
szlig: "ß",
|
|
uacute: "ú",
|
|
uuml: "ü",
|
|
};
|
|
return String(value)
|
|
.replace(/&#(x[0-9a-f]+|\d+);?/giu, (_match, encoded) => {
|
|
const radix = encoded[0].toLowerCase() === "x" ? 16 : 10;
|
|
const codePoint = Number.parseInt(radix === 16 ? encoded.slice(1) : encoded, radix);
|
|
return Number.isInteger(codePoint) && codePoint >= 0 && codePoint <= 0x10ffff
|
|
? String.fromCodePoint(codePoint)
|
|
: "\ufffd";
|
|
})
|
|
.replace(/&([a-z]+);/giu, (match, name) => named[name.toLowerCase()] ?? match);
|
|
}
|
|
|
|
function htmlAttribute(attributes, requestedName) {
|
|
let index = 0;
|
|
while (index < attributes.length) {
|
|
while (/\s/u.test(attributes[index] || "")) index += 1;
|
|
const nameStart = index;
|
|
while (index < attributes.length && !/[\s=/>]/u.test(attributes[index])) index += 1;
|
|
const name = attributes.slice(nameStart, index).toLowerCase();
|
|
if (!name) {
|
|
index += 1;
|
|
continue;
|
|
}
|
|
while (/\s/u.test(attributes[index] || "")) index += 1;
|
|
|
|
let value = "";
|
|
if (attributes[index] === "=") {
|
|
index += 1;
|
|
while (/\s/u.test(attributes[index] || "")) index += 1;
|
|
const quote = attributes[index] === '"' || attributes[index] === "'" ? attributes[index] : null;
|
|
if (quote) {
|
|
index += 1;
|
|
const valueStart = index;
|
|
while (index < attributes.length && attributes[index] !== quote) index += 1;
|
|
value = attributes.slice(valueStart, index);
|
|
if (attributes[index] === quote) index += 1;
|
|
} else {
|
|
const valueStart = index;
|
|
while (index < attributes.length && !/[\s>]/u.test(attributes[index])) index += 1;
|
|
value = attributes.slice(valueStart, index);
|
|
}
|
|
}
|
|
if (name === requestedName.toLowerCase()) return value;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function plainHtmlText(value) {
|
|
return decodeHtmlEntities(
|
|
String(value ?? "")
|
|
.replace(/<script\b[\s\S]*?<\/script\s*>/giu, " ")
|
|
.replace(/<style\b[\s\S]*?<\/style\s*>/giu, " ")
|
|
.replace(/<[^>]+>/gu, " "),
|
|
)
|
|
.replace(/\s+/gu, " ")
|
|
.trim();
|
|
}
|
|
|
|
function candidateName(candidate) {
|
|
return (
|
|
nonEmptyString(candidate?.name) ??
|
|
nonEmptyString(candidate?.properties?.official_name) ??
|
|
nonEmptyString(candidate?.properties?.lock_name) ??
|
|
nonEmptyString(candidate?.properties?.["seamark:name"]) ??
|
|
nonEmptyString(candidate?.properties?.name)
|
|
);
|
|
}
|
|
|
|
function propertyString(candidate, keys) {
|
|
for (const key of keys) {
|
|
const value =
|
|
nonEmptyString(candidate?.properties?.[key]) ??
|
|
nonEmptyString(candidate?.enrichmentProperties?.[key]);
|
|
if (value) return value;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function candidateLocality(candidate) {
|
|
return propertyString(candidate, [
|
|
"addr:city",
|
|
"addressLocality",
|
|
"city",
|
|
"locality",
|
|
"place",
|
|
"is_in:city",
|
|
"is_in",
|
|
]);
|
|
}
|
|
|
|
function candidatePostcode(candidate) {
|
|
return propertyString(candidate, ["addr:postcode", "postalCode", "postcode"]);
|
|
}
|
|
|
|
function candidateCountry(candidate) {
|
|
const country = propertyString(candidate, [
|
|
"addr:country",
|
|
"country",
|
|
"country_code",
|
|
"is_in:country_code",
|
|
]);
|
|
return country?.slice(0, 2).toUpperCase() ?? null;
|
|
}
|
|
|
|
function candidateOperator(candidate) {
|
|
return propertyString(candidate, ["operator", "operator:name", "owner"]);
|
|
}
|
|
|
|
function candidateWaterway(candidate) {
|
|
return (
|
|
propertyString(candidate, [
|
|
"waterway_name",
|
|
"waterwayName",
|
|
"waterway:name",
|
|
"canal",
|
|
"river",
|
|
"seamark:lock:waterway",
|
|
]) ??
|
|
nonEmptyString(candidate?.searchWaterway)
|
|
);
|
|
}
|
|
|
|
function distinctiveTokens(candidate) {
|
|
const identity = [candidateName(candidate), candidateOperator(candidate)].filter(Boolean).join(" ");
|
|
return unique(
|
|
words(identity).filter(
|
|
(token) =>
|
|
token.length >= 3 &&
|
|
!GENERIC_WORDS.has(token) &&
|
|
!/^\d+$/u.test(token),
|
|
),
|
|
);
|
|
}
|
|
|
|
function facilityTypeWords(candidate) {
|
|
return candidate?.layer === "locks"
|
|
? ["schleuse", "sluis", "lock"]
|
|
: ["hafen", "haven", "harbour", "marina", "jachthaven", "yachthafen"];
|
|
}
|
|
|
|
function quotedSearchPart(value) {
|
|
const cleaned = nonEmptyString(value)?.replace(/["\u0000-\u001f]/gu, " ").replace(/\s+/gu, " ");
|
|
return cleaned ? `"${cleaned.slice(0, 160)}"` : null;
|
|
}
|
|
|
|
export function buildFacilitySearchQuery(candidate) {
|
|
const name = candidateName(candidate);
|
|
if (!name || distinctiveTokens(candidate).length === 0) return null;
|
|
|
|
const country = candidateCountry(candidate);
|
|
const type =
|
|
candidate?.layer === "locks"
|
|
? country === "NL"
|
|
? "sluis contact telefoon"
|
|
: country === "DE"
|
|
? "Schleuse Kontakt Telefon"
|
|
: "lock contact phone"
|
|
: country === "NL"
|
|
? "jachthaven contact telefoon"
|
|
: country === "DE"
|
|
? "Hafen Kontakt Telefon"
|
|
: "marina contact phone";
|
|
|
|
const context = unique([
|
|
candidateLocality(candidate),
|
|
candidatePostcode(candidate),
|
|
candidateWaterway(candidate),
|
|
])
|
|
.map((value) => nonEmptyString(value)?.replace(/["\u0000-\u001f]/gu, " "))
|
|
.filter(Boolean);
|
|
|
|
return [quotedSearchPart(name), ...context, type].filter(Boolean).join(" ").slice(0, 380);
|
|
}
|
|
|
|
function safeHttpUrl(value, base) {
|
|
try {
|
|
const url = new URL(String(value), base);
|
|
if (!["http:", "https:"].includes(url.protocol) || url.username || url.password) return null;
|
|
url.hash = "";
|
|
return url;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
export function unwrapDuckDuckGoUrl(value) {
|
|
const decoded = decodeHtmlEntities(value).trim();
|
|
const outer = safeHttpUrl(decoded.startsWith("//") ? `https:${decoded}` : decoded, DUCKDUCKGO_HTML_ENDPOINT);
|
|
if (!outer) return null;
|
|
|
|
const host = outer.hostname.replace(/^www\./u, "").toLowerCase();
|
|
if (host === "duckduckgo.com" || host.endsWith(".duckduckgo.com")) {
|
|
const wrapped = outer.searchParams.get("uddg");
|
|
if (!wrapped) return null;
|
|
const target = safeHttpUrl(wrapped);
|
|
return target?.href ?? null;
|
|
}
|
|
return outer.href;
|
|
}
|
|
|
|
function resultSnippet(segment) {
|
|
const snippetPattern =
|
|
/<(?:a|div|span)\b([^>]*\bclass\s*=\s*(?:"[^"]*(?:result__snippet|result-snippet)[^"]*"|'[^']*(?:result__snippet|result-snippet)[^']*'))[^>]*>([\s\S]*?)<\/(?:a|div|span)\s*>/iu;
|
|
return plainHtmlText(segment.match(snippetPattern)?.[2] ?? "").slice(0, 600);
|
|
}
|
|
|
|
export function parseDuckDuckGoResults(html, { limit = DEFAULT_SEARCH_RESULTS } = {}) {
|
|
const source = String(html ?? "");
|
|
const resultLimit = Math.max(1, Math.min(MAX_SEARCH_RESULTS, Number(limit) || DEFAULT_SEARCH_RESULTS));
|
|
const anchors = [];
|
|
const anchorPattern = /<a\b([^>]*)>([\s\S]*?)<\/a\s*>/giu;
|
|
for (const match of source.matchAll(anchorPattern)) {
|
|
const className = htmlAttribute(match[1], "class") ?? "";
|
|
if (!/(?:^|\s)(?:result__a|result-link)(?:\s|$)/u.test(className)) continue;
|
|
anchors.push({ match, index: match.index ?? 0 });
|
|
}
|
|
|
|
const results = [];
|
|
const seen = new Set();
|
|
for (let index = 0; index < anchors.length && results.length < resultLimit; index += 1) {
|
|
const { match, index: start } = anchors[index];
|
|
const precedingHtml = source.slice(Math.max(0, start - 800), start);
|
|
const nearestResultStart = Math.max(
|
|
precedingHtml.lastIndexOf('<div class="result'),
|
|
precedingHtml.lastIndexOf("<div class='result"),
|
|
);
|
|
if (
|
|
nearestResultStart >= 0 &&
|
|
/\bresult--ad\b/u.test(precedingHtml.slice(nearestResultStart))
|
|
) {
|
|
continue;
|
|
}
|
|
const href = unwrapDuckDuckGoUrl(htmlAttribute(match[1], "href") ?? "");
|
|
const url = href ? safeHttpUrl(href) : null;
|
|
if (!url || BLOCKED_RESULT_HOSTS.some((pattern) => pattern.test(url.hostname.toLowerCase()))) continue;
|
|
const dedupeKey = `${url.hostname.toLowerCase()}${url.pathname.replace(/\/+$/u, "")}`;
|
|
if (seen.has(dedupeKey)) continue;
|
|
|
|
const title = plainHtmlText(match[2]).slice(0, 300);
|
|
if (!title) continue;
|
|
const end = anchors[index + 1]?.index ?? Math.min(source.length, start + 4_000);
|
|
const snippet = resultSnippet(source.slice(start + match[0].length, end));
|
|
seen.add(dedupeKey);
|
|
results.push({
|
|
title,
|
|
url: url.href,
|
|
snippet,
|
|
rank: results.length + 1,
|
|
});
|
|
}
|
|
return results;
|
|
}
|
|
|
|
function hostnameWithoutWww(value) {
|
|
return safeHttpUrl(value)?.hostname.replace(/^www\./u, "").toLowerCase() ?? "";
|
|
}
|
|
|
|
function domainTokenMatch(tokens, url) {
|
|
const hostname = hostnameWithoutWww(url).replace(/[^a-z0-9]+/gu, " ");
|
|
return tokens.some((token) => token.length >= 4 && hostname.includes(token));
|
|
}
|
|
|
|
function matchCoverage(tokens, text) {
|
|
if (tokens.length === 0) return 0;
|
|
const haystack = new Set(words(text));
|
|
return tokens.filter((token) => haystack.has(token)).length / tokens.length;
|
|
}
|
|
|
|
function contextEvidence(candidate, text) {
|
|
const normalized = normalizedMatchText(text);
|
|
const evidence = [];
|
|
let score = 0;
|
|
const locality = normalizedMatchText(candidateLocality(candidate));
|
|
const postcode = normalizedMatchText(candidatePostcode(candidate));
|
|
const operator = normalizedMatchText(candidateOperator(candidate));
|
|
const waterway = normalizedMatchText(candidateWaterway(candidate));
|
|
|
|
if (locality && normalized.includes(locality)) {
|
|
score += 15;
|
|
evidence.push("locality");
|
|
}
|
|
if (postcode && normalized.includes(postcode)) {
|
|
score += 10;
|
|
evidence.push("postcode");
|
|
}
|
|
if (operator && operator.length >= 5 && normalized.includes(operator)) {
|
|
score += 15;
|
|
evidence.push("operator");
|
|
}
|
|
if (waterway && waterway.length >= 4 && normalized.includes(waterway)) {
|
|
score += 10;
|
|
evidence.push("waterway");
|
|
}
|
|
return { score, evidence };
|
|
}
|
|
|
|
function hasTypeConflict(candidate, text) {
|
|
const normalized = normalizedMatchText(text);
|
|
const lock = /\b(?:schleuse|sluis|lock)\b/u.test(normalized);
|
|
const harbour = /\b(?:hafen|haven|harbour|marina|jachthaven|yachthafen)\b/u.test(normalized);
|
|
return candidate?.layer === "locks" ? harbour && !lock : lock && !harbour;
|
|
}
|
|
|
|
export function scoreSearchResult(candidate, result) {
|
|
const name = normalizedMatchText(candidateName(candidate));
|
|
const tokens = distinctiveTokens(candidate);
|
|
const url = safeHttpUrl(result?.url);
|
|
if (!name || tokens.length === 0 || !url) {
|
|
return { accepted: false, score: 0, evidence: ["invalid_identity"] };
|
|
}
|
|
if (BLOCKED_RESULT_HOSTS.some((pattern) => pattern.test(url.hostname.toLowerCase()))) {
|
|
return { accepted: false, score: 0, evidence: ["blocked_host"] };
|
|
}
|
|
|
|
const combined = `${result?.title ?? ""} ${result?.snippet ?? ""} ${url.hostname} ${url.pathname}`;
|
|
const normalized = normalizedMatchText(combined);
|
|
const evidence = [];
|
|
let score = 0;
|
|
const exactName = name.length >= 5 && normalized.includes(name);
|
|
if (exactName) {
|
|
score += 55;
|
|
evidence.push("exact_name");
|
|
}
|
|
|
|
const coverage = matchCoverage(tokens, normalized);
|
|
score += Math.round(coverage * 35);
|
|
if (coverage === 1) evidence.push("all_identity_tokens");
|
|
|
|
const context = contextEvidence(candidate, combined);
|
|
score += context.score;
|
|
evidence.push(...context.evidence);
|
|
|
|
const domainMatch = domainTokenMatch(tokens, url.href);
|
|
if (domainMatch) {
|
|
score += 15;
|
|
evidence.push("domain");
|
|
}
|
|
if (facilityTypeWords(candidate).some((word) => normalized.includes(word))) {
|
|
score += 5;
|
|
evidence.push("type");
|
|
}
|
|
score += Math.max(0, 6 - Math.max(1, Number(result?.rank) || 1));
|
|
|
|
if (hasTypeConflict(candidate, combined)) {
|
|
score -= 45;
|
|
evidence.push("type_conflict");
|
|
}
|
|
if (tokens.length === 1 && context.score === 0 && !domainMatch) {
|
|
score -= 50;
|
|
evidence.push("missing_context");
|
|
}
|
|
|
|
const strongName =
|
|
tokens.length === 1
|
|
? (exactName || coverage === 1) && (context.score > 0 || domainMatch)
|
|
: exactName || (tokens.length >= 2 && coverage === 1);
|
|
const accepted = strongName && score >= DEFAULT_SEARCH_SCORE && !evidence.includes("type_conflict");
|
|
return { accepted, score: Math.max(0, score), evidence };
|
|
}
|
|
|
|
function extractIdentityHtml(html) {
|
|
const source = String(html ?? "");
|
|
const parts = [];
|
|
for (const pattern of [
|
|
/<title\b[^>]*>([\s\S]*?)<\/title\s*>/giu,
|
|
/<h1\b[^>]*>([\s\S]*?)<\/h1\s*>/giu,
|
|
/<meta\b[^>]*(?:property|name)\s*=\s*["'](?:og:title|twitter:title)["'][^>]*>/giu,
|
|
]) {
|
|
for (const match of source.matchAll(pattern)) {
|
|
if (/^<meta/iu.test(match[0])) {
|
|
parts.push(htmlAttribute(match[0].replace(/^<meta\b|>$/giu, ""), "content") ?? "");
|
|
} else {
|
|
parts.push(match[1] ?? "");
|
|
}
|
|
}
|
|
}
|
|
|
|
const jsonLdPattern = /<script\b([^>]*)>([\s\S]*?)<\/script\s*>/giu;
|
|
for (const match of source.matchAll(jsonLdPattern)) {
|
|
const type = (htmlAttribute(match[1], "type") ?? "").split(";", 1)[0].toLowerCase();
|
|
if (type !== "application/ld+json") continue;
|
|
try {
|
|
const value = JSON.parse(match[2]);
|
|
const stack = Array.isArray(value) ? [...value] : [value];
|
|
while (stack.length > 0) {
|
|
const node = stack.pop();
|
|
if (!node || typeof node !== "object") continue;
|
|
if (Array.isArray(node)) {
|
|
stack.push(...node);
|
|
continue;
|
|
}
|
|
parts.push(node.name ?? "", node.legalName ?? "");
|
|
if (node.address && typeof node.address === "object") {
|
|
parts.push(
|
|
node.address.streetAddress ?? "",
|
|
node.address.postalCode ?? "",
|
|
node.address.addressLocality ?? "",
|
|
);
|
|
}
|
|
if (Array.isArray(node["@graph"])) stack.push(...node["@graph"]);
|
|
}
|
|
} catch {
|
|
// Broken JSON-LD is ignored; title and headings remain usable evidence.
|
|
}
|
|
}
|
|
return plainHtmlText(parts.join(" "));
|
|
}
|
|
|
|
export function evaluateFacilityPageMatch(candidate, { html, finalUrl }) {
|
|
const name = normalizedMatchText(candidateName(candidate));
|
|
const tokens = distinctiveTokens(candidate);
|
|
const url = safeHttpUrl(finalUrl);
|
|
if (!name || tokens.length === 0 || !url) {
|
|
return { accepted: false, score: 0, evidence: ["invalid_identity"] };
|
|
}
|
|
if (BLOCKED_RESULT_HOSTS.some((pattern) => pattern.test(url.hostname.toLowerCase()))) {
|
|
return { accepted: false, score: 0, evidence: ["blocked_host"] };
|
|
}
|
|
|
|
const identityText = extractIdentityHtml(html);
|
|
const pageText = plainHtmlText(String(html ?? "").slice(0, MAX_HTML_BYTES));
|
|
const normalizedIdentity = normalizedMatchText(identityText);
|
|
const evidence = [];
|
|
let score = 0;
|
|
const exactName = name.length >= 5 && normalizedIdentity.includes(name);
|
|
if (exactName) {
|
|
score += 60;
|
|
evidence.push("page_exact_name");
|
|
}
|
|
const coverage = matchCoverage(tokens, normalizedIdentity);
|
|
score += Math.round(coverage * 30);
|
|
if (coverage === 1) evidence.push("page_identity_tokens");
|
|
|
|
const context = contextEvidence(candidate, pageText);
|
|
score += context.score;
|
|
evidence.push(...context.evidence.map((item) => `page_${item}`));
|
|
|
|
const domainMatch = domainTokenMatch(tokens, url.href);
|
|
if (domainMatch) {
|
|
score += 15;
|
|
evidence.push("page_domain");
|
|
}
|
|
if (facilityTypeWords(candidate).some((word) => normalizedMatchText(pageText).includes(word))) {
|
|
score += 5;
|
|
evidence.push("page_type");
|
|
}
|
|
if (hasTypeConflict(candidate, identityText)) {
|
|
score -= 50;
|
|
evidence.push("page_type_conflict");
|
|
}
|
|
|
|
const strongName =
|
|
tokens.length === 1
|
|
? (exactName || coverage === 1) && (context.score > 0 || domainMatch)
|
|
: exactName || (tokens.length >= 2 && coverage === 1);
|
|
const accepted =
|
|
strongName && score >= DEFAULT_PAGE_SCORE && !evidence.includes("page_type_conflict");
|
|
return { accepted, score: Math.max(0, score), evidence };
|
|
}
|
|
|
|
export class SearchProviderError extends Error {
|
|
constructor(message, { code, status, cause } = {}) {
|
|
super(message, { cause });
|
|
this.name = "SearchProviderError";
|
|
this.code = code;
|
|
this.status = status;
|
|
}
|
|
}
|
|
|
|
function duckDuckGoChallenge(html) {
|
|
return /challenge-form|anomaly\.js|anomaly-modal|Unfortunately,\s+bots\s+use\s+DuckDuckGo/iu.test(
|
|
String(html ?? ""),
|
|
);
|
|
}
|
|
|
|
export async function searchDuckDuckGo(
|
|
query,
|
|
{
|
|
limit = DEFAULT_SEARCH_RESULTS,
|
|
fetchPageImpl = fetchHtmlPage,
|
|
lookupImpl,
|
|
beforeRequest = async () => {},
|
|
timeoutMs = DEFAULT_TIMEOUT_MS,
|
|
} = {},
|
|
) {
|
|
const url = new URL(DUCKDUCKGO_HTML_ENDPOINT);
|
|
url.searchParams.set("q", query);
|
|
|
|
let page;
|
|
try {
|
|
page = await fetchPageImpl(url.href, {
|
|
lookupImpl,
|
|
beforeRequest,
|
|
timeoutMs,
|
|
maximumBytes: MAX_SEARCH_RESPONSE_BYTES,
|
|
userAgent: SEARCH_USER_AGENT,
|
|
});
|
|
} catch (error) {
|
|
const blocked = error?.status === 202 || error?.status === 403 || error?.status === 429 ||
|
|
/\bHTTP (?:202|403|429)\b/u.test(error?.message ?? "");
|
|
throw new SearchProviderError(
|
|
blocked
|
|
? "DuckDuckGo hat den automatisierten Suchabruf blockiert."
|
|
: `DuckDuckGo-Suche fehlgeschlagen: ${error.message}`,
|
|
{
|
|
code: blocked ? "PROVIDER_BLOCKED" : "SEARCH_FAILED",
|
|
status: error?.status,
|
|
cause: error,
|
|
},
|
|
);
|
|
}
|
|
|
|
if ((page.status && page.status !== 200) || duckDuckGoChallenge(page.html)) {
|
|
throw new SearchProviderError(
|
|
"DuckDuckGo verlangt eine Browserprüfung; der Batch wird gestoppt.",
|
|
{
|
|
code: "PROVIDER_BLOCKED",
|
|
status: page.status,
|
|
},
|
|
);
|
|
}
|
|
return parseDuckDuckGoResults(page.html, { limit });
|
|
}
|
|
|
|
export function createDuckDuckGoProvider(options = {}) {
|
|
return {
|
|
id: "duckduckgo-html",
|
|
async search(query, { limit } = {}) {
|
|
return searchDuckDuckGo(query, { ...options, limit });
|
|
},
|
|
};
|
|
}
|
|
|
|
function parseBraveResults(data, limit) {
|
|
const results = [];
|
|
const seen = new Set();
|
|
for (const item of data?.web?.results ?? []) {
|
|
const url = safeHttpUrl(item?.url);
|
|
const title = nonEmptyString(item?.title);
|
|
if (!url || !title || BLOCKED_RESULT_HOSTS.some((pattern) => pattern.test(url.hostname))) continue;
|
|
const key = `${url.hostname.toLowerCase()}${url.pathname.replace(/\/+$/u, "")}`;
|
|
if (seen.has(key)) continue;
|
|
seen.add(key);
|
|
results.push({
|
|
title: plainHtmlText(title).slice(0, 300),
|
|
url: url.href,
|
|
snippet: plainHtmlText(item?.description ?? "").slice(0, 600),
|
|
rank: results.length + 1,
|
|
});
|
|
if (results.length >= limit) break;
|
|
}
|
|
return results;
|
|
}
|
|
|
|
export async function searchBrave(
|
|
query,
|
|
{
|
|
apiKey,
|
|
limit = DEFAULT_SEARCH_RESULTS,
|
|
country,
|
|
language,
|
|
fetchImpl = globalThis.fetch,
|
|
timeoutMs = DEFAULT_TIMEOUT_MS,
|
|
} = {},
|
|
) {
|
|
if (!nonEmptyString(apiKey)) {
|
|
throw new SearchProviderError("BRAVE_SEARCH_API_KEY fehlt.", { code: "PROVIDER_CONFIG" });
|
|
}
|
|
const url = new URL(BRAVE_SEARCH_ENDPOINT);
|
|
url.searchParams.set("q", query);
|
|
url.searchParams.set("count", String(Math.max(1, Math.min(MAX_SEARCH_RESULTS, Number(limit) || 5))));
|
|
url.searchParams.set("safesearch", "strict");
|
|
url.searchParams.set("text_decorations", "false");
|
|
if (country) url.searchParams.set("country", country);
|
|
if (language) url.searchParams.set("search_lang", language);
|
|
|
|
const controller = new AbortController();
|
|
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
try {
|
|
const response = await fetchImpl(url, {
|
|
headers: {
|
|
Accept: "application/json",
|
|
"Accept-Encoding": "identity",
|
|
"User-Agent": SEARCH_USER_AGENT,
|
|
"X-Subscription-Token": apiKey,
|
|
},
|
|
signal: controller.signal,
|
|
});
|
|
if (!response.ok) {
|
|
const blocked = response.status === 401 || response.status === 403 || response.status === 429;
|
|
throw new SearchProviderError(`Brave Search antwortete mit HTTP ${response.status}.`, {
|
|
code: blocked ? "PROVIDER_BLOCKED" : "SEARCH_FAILED",
|
|
status: response.status,
|
|
});
|
|
}
|
|
const body = await readBodyLimited(response, MAX_SEARCH_RESPONSE_BYTES);
|
|
let data;
|
|
try {
|
|
data = JSON.parse(body);
|
|
} catch (error) {
|
|
throw new SearchProviderError("Brave Search lieferte ungültiges JSON.", {
|
|
code: "INVALID_RESPONSE",
|
|
cause: error,
|
|
});
|
|
}
|
|
return parseBraveResults(data, Math.max(1, Math.min(MAX_SEARCH_RESULTS, Number(limit) || 5)));
|
|
} catch (error) {
|
|
if (error instanceof SearchProviderError) throw error;
|
|
throw new SearchProviderError(`Brave Search konnte nicht geladen werden: ${error.message}`, {
|
|
code: "SEARCH_FAILED",
|
|
cause: error,
|
|
});
|
|
} finally {
|
|
clearTimeout(timeout);
|
|
}
|
|
}
|
|
|
|
export function createBraveSearchProvider({ apiKey, ...options } = {}) {
|
|
return {
|
|
id: "brave-web",
|
|
async search(query, { limit, candidate } = {}) {
|
|
const country = candidateCountry(candidate);
|
|
const language = country === "DE" ? "de" : country === "NL" ? "nl" : undefined;
|
|
return searchBrave(query, { ...options, apiKey, limit, country, language });
|
|
},
|
|
};
|
|
}
|
|
|
|
function searchFingerprint(providerId, query) {
|
|
return createHash("sha256")
|
|
.update(`${providerId}\n${normalizedMatchText(query)}`, "utf8")
|
|
.digest("hex");
|
|
}
|
|
|
|
function retryAfter(status, attemptedAt) {
|
|
const days =
|
|
status === "success"
|
|
? 180
|
|
: status === "fetch_failed"
|
|
? 7
|
|
: status === "provider_blocked"
|
|
? 1
|
|
: 90;
|
|
return new Date(new Date(attemptedAt).getTime() + days * 86_400_000).toISOString();
|
|
}
|
|
|
|
function attemptRecord(candidate, providerId, query, status, attemptedAt, options = {}) {
|
|
if (!ATTEMPT_STATUSES.has(status)) throw new Error(`Unbekannter Suchstatus: ${status}`);
|
|
return {
|
|
layer: candidate.layer,
|
|
originalSource: candidate.source,
|
|
originalSourceId: candidate.sourceId,
|
|
provider: providerId,
|
|
queryFingerprint: searchFingerprint(providerId, query ?? candidateName(candidate) ?? ""),
|
|
query,
|
|
status,
|
|
resultUrl: options.resultUrl ?? null,
|
|
details: options.details ?? {},
|
|
attemptedAt,
|
|
retryAfter: retryAfter(status, attemptedAt),
|
|
};
|
|
}
|
|
|
|
function priorWebsite(candidate) {
|
|
return linkedWebsite(candidate?.properties ?? {}) ?? linkedWebsite(candidate?.enrichmentProperties ?? {});
|
|
}
|
|
|
|
function mergeProvenance(candidate, fields, data) {
|
|
const existing =
|
|
candidate?.enrichmentProperties?.enrichmentProvenance &&
|
|
typeof candidate.enrichmentProperties.enrichmentProvenance === "object"
|
|
? candidate.enrichmentProperties.enrichmentProvenance
|
|
: {};
|
|
return {
|
|
...existing,
|
|
...Object.fromEntries(fields.map((field) => [field, data])),
|
|
};
|
|
}
|
|
|
|
export function buildSearchEnrichmentRecord(options) {
|
|
const {
|
|
candidate,
|
|
page,
|
|
extracted,
|
|
query,
|
|
providerId,
|
|
fetchedAt,
|
|
} = options;
|
|
const result = options.result ?? options.searchResult;
|
|
const resultMatch = options.resultMatch ?? options.match;
|
|
const pageMatch = options.pageMatch ?? options.match;
|
|
const sourceId = stableWebsiteSourceId(candidate);
|
|
if (!sourceId || !result || !resultMatch || !pageMatch) return null;
|
|
const current = candidateContacts(candidate);
|
|
const existingWebsite = priorWebsite(candidate);
|
|
const contacts = { ...current };
|
|
const addedFields = [];
|
|
|
|
for (const key of ["phone", "email", "operator", "address"]) {
|
|
if (!contacts[key] && extracted?.[key]) {
|
|
contacts[key] = extracted[key];
|
|
addedFields.push(key);
|
|
}
|
|
}
|
|
if (!existingWebsite) addedFields.unshift("website");
|
|
if (addedFields.length === 0) return null;
|
|
|
|
const verifiedUrl = safeHttpUrl(page?.finalUrl)?.href;
|
|
if (!verifiedUrl) return null;
|
|
const website = existingWebsite?.url ?? verifiedUrl;
|
|
const name = candidateName(candidate);
|
|
const provenance = {
|
|
provider: providerId,
|
|
sourceUrl: verifiedUrl,
|
|
fetchedAt,
|
|
searchScore: resultMatch.score,
|
|
pageScore: pageMatch.score,
|
|
};
|
|
return {
|
|
originalId: candidate.id,
|
|
sourceId,
|
|
name,
|
|
properties: {
|
|
name,
|
|
website,
|
|
phone: contacts.phone,
|
|
"contact:phone": contacts.phone,
|
|
email: contacts.email,
|
|
"contact:email": contacts.email,
|
|
operator: contacts.operator,
|
|
address: contacts.address,
|
|
original_source: candidate.source,
|
|
original_source_id: candidate.sourceId,
|
|
enrichmentSource: "facility-search",
|
|
enrichmentProvider: providerId,
|
|
enrichmentSourceUrl: verifiedUrl,
|
|
sourceUrl: verifiedUrl,
|
|
source_url: verifiedUrl,
|
|
searchQuery: query,
|
|
searchResultUrl: result.url,
|
|
searchResultRank: result.rank,
|
|
searchResultTitle: nonEmptyString(result.title)?.slice(0, 300),
|
|
searchScore: resultMatch.score,
|
|
pageMatchScore: pageMatch.score,
|
|
identityEvidence: unique([...resultMatch.evidence, ...pageMatch.evidence]),
|
|
enrichmentProvenance: mergeProvenance(candidate, addedFields, provenance),
|
|
fetchedAt,
|
|
fetched_at: fetchedAt,
|
|
enriched_fields: addedFields,
|
|
},
|
|
};
|
|
}
|
|
|
|
function acceptedSearchResults(candidate, results, { minScore, minMargin }) {
|
|
const scored = results
|
|
.map((result) => ({ result, match: scoreSearchResult(candidate, result) }))
|
|
.filter(({ match }) => match.accepted && match.score >= minScore)
|
|
.sort((left, right) => right.match.score - left.match.score);
|
|
|
|
const bestByHost = new Map();
|
|
for (const item of scored) {
|
|
const host = hostnameWithoutWww(item.result.url);
|
|
if (!bestByHost.has(host)) bestByHost.set(host, item);
|
|
}
|
|
const distinctHosts = [...bestByHost.values()].sort(
|
|
(left, right) => right.match.score - left.match.score,
|
|
);
|
|
if (distinctHosts.length === 0) return { status: "no_match", matches: [] };
|
|
if (
|
|
distinctHosts.length > 1 &&
|
|
distinctHosts[0].match.score - distinctHosts[1].match.score < minMargin
|
|
) {
|
|
return { status: "ambiguous", matches: distinctHosts };
|
|
}
|
|
return { status: "matched", matches: distinctHosts };
|
|
}
|
|
|
|
function providerBlocked(error) {
|
|
return (
|
|
error?.code === "PROVIDER_BLOCKED" ||
|
|
[202, 403, 429].includes(Number(error?.status)) ||
|
|
/(?:captcha|browserprüfung|blockiert)/iu.test(error?.message ?? "")
|
|
);
|
|
}
|
|
|
|
export async function enrichSearchCandidates(
|
|
candidates,
|
|
{
|
|
searchProvider,
|
|
fetchPageImpl = fetchHtmlPage,
|
|
lookupImpl,
|
|
concurrency = DEFAULT_SEARCH_CONCURRENCY,
|
|
searchDelayMs = DEFAULT_SEARCH_DELAY_MS,
|
|
hostDelayMs = DEFAULT_SITE_DELAY_MS,
|
|
timeoutMs = DEFAULT_TIMEOUT_MS,
|
|
maximumBytes = MAX_HTML_BYTES,
|
|
maxResults = DEFAULT_SEARCH_RESULTS,
|
|
maxPages = DEFAULT_SEARCH_PAGES,
|
|
minScore = DEFAULT_SEARCH_SCORE,
|
|
minPageScore = DEFAULT_PAGE_SCORE,
|
|
minMargin = DEFAULT_SCORE_MARGIN,
|
|
sleepImpl,
|
|
now,
|
|
fetchedAt = new Date().toISOString(),
|
|
logger = console,
|
|
} = {},
|
|
) {
|
|
if (!searchProvider?.id || typeof searchProvider.search !== "function") {
|
|
throw new Error("Ein gültiger Suchprovider ist erforderlich.");
|
|
}
|
|
|
|
const siteLimiter = createHostLimiter({ delayMs: hostDelayMs, sleepImpl, now });
|
|
const searchLimiter = createHostLimiter({ delayMs: searchDelayMs, sleepImpl, now });
|
|
const pageCache = new Map();
|
|
const records = [];
|
|
const attempts = [];
|
|
let blockedError = null;
|
|
const stats = {
|
|
candidates: candidates.length,
|
|
searched: 0,
|
|
searchResults: 0,
|
|
pagesChecked: 0,
|
|
enriched: 0,
|
|
alreadyComplete: 0,
|
|
invalidCandidate: 0,
|
|
noMatch: 0,
|
|
ambiguous: 0,
|
|
noContacts: 0,
|
|
failed: 0,
|
|
providerBlocked: 0,
|
|
skippedAfterBlock: 0,
|
|
fetchedAt,
|
|
};
|
|
|
|
const effectiveConcurrency = Math.max(
|
|
1,
|
|
Math.min(MAX_SEARCH_CONCURRENCY, Number(concurrency) || 1),
|
|
);
|
|
await mapWithConcurrency(candidates, effectiveConcurrency, async (candidate) => {
|
|
if (blockedError) {
|
|
stats.skippedAfterBlock += 1;
|
|
return;
|
|
}
|
|
if (Object.values(candidateContacts(candidate)).every(Boolean) && priorWebsite(candidate)) {
|
|
stats.alreadyComplete += 1;
|
|
return;
|
|
}
|
|
|
|
const query = buildFacilitySearchQuery(candidate);
|
|
if (!query) {
|
|
stats.invalidCandidate += 1;
|
|
attempts.push(
|
|
attemptRecord(candidate, searchProvider.id, null, "invalid_candidate", fetchedAt, {
|
|
details: { reason: "generic_or_missing_name" },
|
|
}),
|
|
);
|
|
return;
|
|
}
|
|
|
|
let results;
|
|
try {
|
|
await searchLimiter(new URL(`https://${searchProvider.id}.search-provider.invalid/`));
|
|
stats.searched += 1;
|
|
results = await searchProvider.search(query, {
|
|
limit: Math.max(1, Math.min(MAX_SEARCH_RESULTS, Number(maxResults) || DEFAULT_SEARCH_RESULTS)),
|
|
candidate,
|
|
});
|
|
results = Array.isArray(results) ? results : results?.results ?? [];
|
|
stats.searchResults += results.length;
|
|
} catch (error) {
|
|
if (providerBlocked(error)) {
|
|
blockedError = error;
|
|
stats.providerBlocked += 1;
|
|
attempts.push(
|
|
attemptRecord(candidate, searchProvider.id, query, "provider_blocked", fetchedAt, {
|
|
details: { code: error.code, status: error.status },
|
|
}),
|
|
);
|
|
logger.warn?.(`Suchprovider ${searchProvider.id} blockiert den Batch: ${error.message}`);
|
|
return;
|
|
}
|
|
stats.failed += 1;
|
|
attempts.push(
|
|
attemptRecord(candidate, searchProvider.id, query, "fetch_failed", fetchedAt, {
|
|
details: { stage: "search", code: error.code, message: error.message?.slice(0, 300) },
|
|
}),
|
|
);
|
|
logger.warn?.(
|
|
`Suche für ${candidate.source}:${candidate.sourceId} fehlgeschlagen: ${error.message}`,
|
|
);
|
|
return;
|
|
}
|
|
|
|
const selection = acceptedSearchResults(candidate, results, { minScore, minMargin });
|
|
if (selection.status !== "matched") {
|
|
if (selection.status === "ambiguous") stats.ambiguous += 1;
|
|
else stats.noMatch += 1;
|
|
attempts.push(
|
|
attemptRecord(candidate, searchProvider.id, query, selection.status, fetchedAt, {
|
|
details: {
|
|
resultCount: results.length,
|
|
topScores: selection.matches.slice(0, 3).map(({ result, match }) => ({
|
|
host: hostnameWithoutWww(result.url),
|
|
score: match.score,
|
|
})),
|
|
},
|
|
}),
|
|
);
|
|
return;
|
|
}
|
|
|
|
let matchedPage = null;
|
|
let pageFailures = 0;
|
|
for (const { result, match: resultMatch } of selection.matches.slice(0, maxPages)) {
|
|
let pagePromise = pageCache.get(result.url);
|
|
if (!pagePromise) {
|
|
pagePromise = fetchPageImpl(result.url, {
|
|
lookupImpl,
|
|
beforeRequest: siteLimiter,
|
|
timeoutMs,
|
|
maximumBytes,
|
|
userAgent: SEARCH_USER_AGENT,
|
|
});
|
|
pageCache.set(result.url, pagePromise);
|
|
}
|
|
|
|
let page;
|
|
try {
|
|
page = await pagePromise;
|
|
stats.pagesChecked += 1;
|
|
} catch (error) {
|
|
pageFailures += 1;
|
|
logger.warn?.(
|
|
`Suchtreffer für ${candidate.source}:${candidate.sourceId} nicht ladbar: ${error.message}`,
|
|
);
|
|
continue;
|
|
}
|
|
const pageMatch = evaluateFacilityPageMatch(candidate, page);
|
|
if (!pageMatch.accepted || pageMatch.score < minPageScore) continue;
|
|
matchedPage = { result, resultMatch, page, pageMatch };
|
|
break;
|
|
}
|
|
|
|
if (!matchedPage) {
|
|
const status = pageFailures >= Math.min(maxPages, selection.matches.length)
|
|
? "fetch_failed"
|
|
: "no_match";
|
|
if (status === "fetch_failed") stats.failed += 1;
|
|
else stats.noMatch += 1;
|
|
attempts.push(
|
|
attemptRecord(candidate, searchProvider.id, query, status, fetchedAt, {
|
|
details: { stage: "target_page", pageFailures },
|
|
}),
|
|
);
|
|
return;
|
|
}
|
|
|
|
const extracted = extractContactsFromHtml(matchedPage.page.html);
|
|
const record = buildSearchEnrichmentRecord({
|
|
candidate,
|
|
...matchedPage,
|
|
extracted,
|
|
query,
|
|
providerId: searchProvider.id,
|
|
fetchedAt,
|
|
});
|
|
if (!record) {
|
|
stats.noContacts += 1;
|
|
attempts.push(
|
|
attemptRecord(candidate, searchProvider.id, query, "no_contacts", fetchedAt, {
|
|
resultUrl: matchedPage.page.finalUrl,
|
|
details: {
|
|
searchScore: matchedPage.resultMatch.score,
|
|
pageScore: matchedPage.pageMatch.score,
|
|
},
|
|
}),
|
|
);
|
|
return;
|
|
}
|
|
|
|
records.push(record);
|
|
stats.enriched += 1;
|
|
attempts.push(
|
|
attemptRecord(candidate, searchProvider.id, query, "success", fetchedAt, {
|
|
resultUrl: matchedPage.page.finalUrl,
|
|
details: {
|
|
enrichedFields: record.properties.enriched_fields,
|
|
searchScore: matchedPage.resultMatch.score,
|
|
pageScore: matchedPage.pageMatch.score,
|
|
},
|
|
}),
|
|
);
|
|
});
|
|
|
|
return { records, attempts, stats, providerError: blockedError };
|
|
}
|
|
|
|
export async function ensureSearchAttemptSchema({
|
|
databaseUrl = process.env.DATABASE_URL,
|
|
} = {}) {
|
|
if (!databaseUrl) throw new Error("DATABASE_URL ist für die Suchanreicherung erforderlich.");
|
|
const { default: pg } = await import("pg");
|
|
const client = new pg.Client({ connectionString: databaseUrl });
|
|
await client.connect();
|
|
try {
|
|
await client.query(`
|
|
CREATE TABLE IF NOT EXISTS marine_enrichment_attempts (
|
|
id bigserial PRIMARY KEY,
|
|
layer text NOT NULL CHECK (layer IN ('locks', 'harbours')),
|
|
original_source text NOT NULL,
|
|
original_source_id text NOT NULL,
|
|
provider text NOT NULL,
|
|
query_fingerprint text NOT NULL,
|
|
query text,
|
|
status text NOT NULL CHECK (
|
|
status IN (
|
|
'success',
|
|
'no_match',
|
|
'ambiguous',
|
|
'no_contacts',
|
|
'fetch_failed',
|
|
'provider_blocked',
|
|
'invalid_candidate'
|
|
)
|
|
),
|
|
result_url text,
|
|
details jsonb NOT NULL DEFAULT '{}'::jsonb,
|
|
attempted_at timestamptz NOT NULL DEFAULT now(),
|
|
retry_after timestamptz,
|
|
UNIQUE (layer, original_source, original_source_id, provider, query_fingerprint)
|
|
);
|
|
CREATE INDEX IF NOT EXISTS marine_enrichment_attempts_retry_idx
|
|
ON marine_enrichment_attempts (provider, retry_after);
|
|
`);
|
|
} finally {
|
|
await client.end();
|
|
}
|
|
}
|
|
|
|
export async function loadSearchCandidates({
|
|
databaseUrl = process.env.DATABASE_URL,
|
|
limit = DEFAULT_SEARCH_LIMIT,
|
|
providerId = "duckduckgo-html",
|
|
includeAttempts = false,
|
|
now = new Date(),
|
|
} = {}) {
|
|
if (!databaseUrl) throw new Error("DATABASE_URL ist für die Suchanreicherung erforderlich.");
|
|
const { default: pg } = await import("pg");
|
|
const client = new pg.Client({ connectionString: databaseUrl });
|
|
await client.connect();
|
|
|
|
const attemptJoin = includeAttempts
|
|
? `
|
|
LEFT JOIN LATERAL (
|
|
SELECT
|
|
attempt.query_fingerprint,
|
|
attempt.status,
|
|
attempt.retry_after
|
|
FROM marine_enrichment_attempts AS attempt
|
|
WHERE attempt.layer = original.layer
|
|
AND attempt.original_source = original.source
|
|
AND attempt.original_source_id = original.source_id
|
|
AND attempt.provider = $1
|
|
ORDER BY attempt.attempted_at DESC
|
|
LIMIT 1
|
|
) AS recent_attempt ON true
|
|
`
|
|
: "";
|
|
const attemptColumns = includeAttempts
|
|
? `
|
|
recent_attempt.query_fingerprint AS "attemptFingerprint",
|
|
recent_attempt.status AS "attemptStatus",
|
|
recent_attempt.retry_after AS "attemptRetryAfter"
|
|
`
|
|
: `
|
|
NULL::text AS "attemptFingerprint",
|
|
NULL::text AS "attemptStatus",
|
|
NULL::timestamptz AS "attemptRetryAfter"
|
|
`;
|
|
|
|
try {
|
|
const result = await client.query(
|
|
`
|
|
SELECT
|
|
original.id,
|
|
original.layer,
|
|
original.source,
|
|
original.source_id AS "sourceId",
|
|
original.name,
|
|
original.properties,
|
|
enrichment.properties AS "enrichmentProperties",
|
|
nearest_waterway.name AS "searchWaterway",
|
|
${attemptColumns}
|
|
FROM marine_features AS original
|
|
LEFT JOIN marine_features AS enrichment
|
|
ON enrichment.source = 'facility-website'
|
|
AND enrichment.layer = original.layer
|
|
AND enrichment.source_id = original.source || ':' || original.source_id
|
|
LEFT JOIN LATERAL (
|
|
SELECT waterway.name
|
|
FROM marine_features AS waterway
|
|
WHERE waterway.layer = 'waterways'
|
|
AND NULLIF(BTRIM(waterway.name), '') IS NOT NULL
|
|
AND waterway.geom && ST_Expand(original.geom, 0.02)
|
|
ORDER BY waterway.geom <-> original.geom
|
|
LIMIT 1
|
|
) AS nearest_waterway ON true
|
|
${attemptJoin}
|
|
WHERE original.layer IN ('locks', 'harbours')
|
|
AND original.source IN ('osm', 'euris')
|
|
AND original.source_id IS NOT NULL
|
|
AND COALESCE(
|
|
NULLIF(BTRIM(original.name), ''),
|
|
NULLIF(BTRIM(original.properties->>'official_name'), ''),
|
|
NULLIF(BTRIM(original.properties->>'lock_name'), ''),
|
|
NULLIF(BTRIM(original.properties->>'seamark:name'), ''),
|
|
NULLIF(BTRIM(original.properties->>'name'), '')
|
|
) IS NOT NULL
|
|
AND NOT (
|
|
original.source = 'osm'
|
|
AND (
|
|
(
|
|
original.layer = 'locks'
|
|
AND (
|
|
LOWER(COALESCE(original.properties->>'waterway', '')) = 'lock_gate'
|
|
OR LOWER(COALESCE(original.properties->>'seamark:type', '')) = 'lock_gate'
|
|
OR (
|
|
LOWER(COALESCE(original.properties->>'seamark:type', '')) = 'gate'
|
|
AND LOWER(COALESCE(original.properties->>'seamark:gate:category', '')) LIKE '%lock%'
|
|
)
|
|
)
|
|
)
|
|
OR (
|
|
original.layer = 'harbours'
|
|
AND LOWER(COALESCE(original.properties->>'waterway', '')) = 'dock'
|
|
AND LOWER(COALESCE(original.properties->>'seamark:type', ''))
|
|
NOT IN ('harbour', 'harbour_basin', 'marina')
|
|
AND LOWER(COALESCE(original.properties->>'leisure', '')) <> 'marina'
|
|
AND NULLIF(BTRIM(original.properties->>'harbour'), '') IS NULL
|
|
AND LOWER(COALESCE(original.properties->>'industrial', '')) <> 'port'
|
|
AND LOWER(COALESCE(original.properties->>'landuse', ''))
|
|
NOT IN ('harbour', 'port')
|
|
)
|
|
)
|
|
)
|
|
AND (
|
|
COALESCE(
|
|
NULLIF(BTRIM(original.properties->>'website'), ''),
|
|
NULLIF(BTRIM(original.properties->>'contact:website'), ''),
|
|
NULLIF(BTRIM(original.properties->>'url'), ''),
|
|
NULLIF(BTRIM(enrichment.properties->>'website'), '')
|
|
) IS NULL
|
|
OR COALESCE(
|
|
NULLIF(BTRIM(original.properties->>'phone'), ''),
|
|
NULLIF(BTRIM(original.properties->>'contact:phone'), ''),
|
|
NULLIF(BTRIM(enrichment.properties->>'phone'), ''),
|
|
NULLIF(BTRIM(enrichment.properties->>'contact:phone'), '')
|
|
) IS NULL
|
|
OR COALESCE(
|
|
NULLIF(BTRIM(original.properties->>'email'), ''),
|
|
NULLIF(BTRIM(original.properties->>'contact:email'), ''),
|
|
NULLIF(BTRIM(enrichment.properties->>'email'), ''),
|
|
NULLIF(BTRIM(enrichment.properties->>'contact:email'), '')
|
|
) IS NULL
|
|
OR COALESCE(
|
|
NULLIF(BTRIM(original.properties->>'operator'), ''),
|
|
NULLIF(BTRIM(original.properties->>'operator:name'), ''),
|
|
NULLIF(BTRIM(original.properties->>'owner'), ''),
|
|
NULLIF(BTRIM(enrichment.properties->>'operator'), '')
|
|
) IS NULL
|
|
OR COALESCE(
|
|
NULLIF(BTRIM(original.properties->>'address'), ''),
|
|
NULLIF(BTRIM(original.properties->>'contact:address'), ''),
|
|
NULLIF(BTRIM(original.properties->>'addr:full'), ''),
|
|
NULLIF(BTRIM(original.properties->>'addr:street'), ''),
|
|
NULLIF(BTRIM(original.properties->>'addr:city'), ''),
|
|
NULLIF(BTRIM(enrichment.properties->>'address'), '')
|
|
) IS NULL
|
|
)
|
|
ORDER BY
|
|
(
|
|
COALESCE(
|
|
NULLIF(BTRIM(original.properties->>'website'), ''),
|
|
NULLIF(BTRIM(original.properties->>'contact:website'), ''),
|
|
NULLIF(BTRIM(original.properties->>'url'), ''),
|
|
NULLIF(BTRIM(enrichment.properties->>'website'), '')
|
|
) IS NULL
|
|
) DESC,
|
|
CASE WHEN original.source = 'euris' THEN 0 ELSE 1 END,
|
|
original.id
|
|
`,
|
|
includeAttempts ? [providerId] : [],
|
|
);
|
|
|
|
const currentTime = new Date(now).getTime();
|
|
return result.rows
|
|
.filter((candidate) => {
|
|
if (!includeAttempts || !candidate.attemptFingerprint) return true;
|
|
const query = buildFacilitySearchQuery(candidate);
|
|
const fingerprint = searchFingerprint(providerId, query ?? candidateName(candidate) ?? "");
|
|
if (fingerprint !== candidate.attemptFingerprint) return true;
|
|
const retryAt = candidate.attemptRetryAfter
|
|
? new Date(candidate.attemptRetryAfter).getTime()
|
|
: Number.POSITIVE_INFINITY;
|
|
return retryAt <= currentTime;
|
|
})
|
|
.slice(0, Math.max(1, Math.min(MAX_SEARCH_LIMIT, Number(limit) || DEFAULT_SEARCH_LIMIT)));
|
|
} finally {
|
|
await client.end();
|
|
}
|
|
}
|
|
|
|
async function insertEnrichmentRecords(client, records) {
|
|
if (records.length === 0) return 0;
|
|
const values = [];
|
|
const rows = records.map((record, index) => {
|
|
const position = index * 5;
|
|
values.push(
|
|
record.originalId,
|
|
record.sourceId,
|
|
record.name,
|
|
JSON.stringify(record.properties),
|
|
record.properties.fetchedAt,
|
|
);
|
|
return `(\$${position + 1}::bigint, \$${position + 2}, \$${position + 3}, \$${position + 4}::jsonb, \$${position + 5}::timestamptz)`;
|
|
});
|
|
const result = await client.query(
|
|
`
|
|
INSERT INTO marine_features (layer, source, source_id, name, properties, geom, updated_at)
|
|
SELECT
|
|
original.layer,
|
|
'facility-website',
|
|
incoming.source_id,
|
|
incoming.name,
|
|
jsonb_strip_nulls(incoming.properties),
|
|
original.geom,
|
|
incoming.updated_at
|
|
FROM (VALUES ${rows.join(",")}) AS incoming
|
|
(original_id, source_id, name, properties, updated_at)
|
|
JOIN marine_features AS original ON original.id = incoming.original_id
|
|
ON CONFLICT (source, source_id, layer) WHERE source_id IS NOT NULL
|
|
DO UPDATE SET
|
|
name = EXCLUDED.name,
|
|
properties = marine_features.properties || jsonb_strip_nulls(EXCLUDED.properties),
|
|
geom = EXCLUDED.geom,
|
|
updated_at = EXCLUDED.updated_at
|
|
`,
|
|
values,
|
|
);
|
|
return result.rowCount;
|
|
}
|
|
|
|
async function insertAttemptRecords(client, attempts) {
|
|
if (attempts.length === 0) return 0;
|
|
const values = [];
|
|
const rows = attempts.map((attempt, index) => {
|
|
const position = index * 11;
|
|
values.push(
|
|
attempt.layer,
|
|
attempt.originalSource,
|
|
attempt.originalSourceId,
|
|
attempt.provider,
|
|
attempt.queryFingerprint,
|
|
attempt.query,
|
|
attempt.status,
|
|
attempt.resultUrl,
|
|
JSON.stringify(attempt.details ?? {}),
|
|
attempt.attemptedAt,
|
|
attempt.retryAfter,
|
|
);
|
|
return `(\$${position + 1}, \$${position + 2}, \$${position + 3}, \$${position + 4}, \$${position + 5}, \$${position + 6}, \$${position + 7}, \$${position + 8}, \$${position + 9}::jsonb, \$${position + 10}::timestamptz, \$${position + 11}::timestamptz)`;
|
|
});
|
|
const result = await client.query(
|
|
`
|
|
INSERT INTO marine_enrichment_attempts (
|
|
layer,
|
|
original_source,
|
|
original_source_id,
|
|
provider,
|
|
query_fingerprint,
|
|
query,
|
|
status,
|
|
result_url,
|
|
details,
|
|
attempted_at,
|
|
retry_after
|
|
)
|
|
VALUES ${rows.join(",")}
|
|
ON CONFLICT (layer, original_source, original_source_id, provider, query_fingerprint)
|
|
DO UPDATE SET
|
|
query = EXCLUDED.query,
|
|
status = EXCLUDED.status,
|
|
result_url = EXCLUDED.result_url,
|
|
details = EXCLUDED.details,
|
|
attempted_at = EXCLUDED.attempted_at,
|
|
retry_after = EXCLUDED.retry_after
|
|
`,
|
|
values,
|
|
);
|
|
return result.rowCount;
|
|
}
|
|
|
|
export async function writeSearchEnrichmentResults(
|
|
{ records, attempts },
|
|
{ databaseUrl = process.env.DATABASE_URL } = {},
|
|
) {
|
|
if (!databaseUrl) throw new Error("DATABASE_URL ist für die Suchanreicherung erforderlich.");
|
|
const { default: pg } = await import("pg");
|
|
const client = new pg.Client({ connectionString: databaseUrl });
|
|
await client.connect();
|
|
try {
|
|
await client.query("BEGIN");
|
|
const enrichments = await insertEnrichmentRecords(client, records);
|
|
const attemptCount = await insertAttemptRecords(client, attempts);
|
|
await client.query("COMMIT");
|
|
return { enrichments, attempts: attemptCount };
|
|
} catch (error) {
|
|
await client.query("ROLLBACK");
|
|
throw error;
|
|
} finally {
|
|
await client.end();
|
|
}
|
|
}
|
|
|
|
export async function runSearchEnrichment({
|
|
dryRun = true,
|
|
limit = DEFAULT_SEARCH_LIMIT,
|
|
databaseUrl,
|
|
provider,
|
|
candidates,
|
|
loader = loadSearchCandidates,
|
|
writer = writeSearchEnrichmentResults,
|
|
enrichmentOptions,
|
|
includeAttempts = false,
|
|
} = {}) {
|
|
const effectiveProvider = provider ?? enrichmentOptions?.searchProvider;
|
|
if (!effectiveProvider) throw new Error("Ein Suchprovider ist erforderlich.");
|
|
const selectedCandidates =
|
|
candidates ??
|
|
(await loader({
|
|
databaseUrl,
|
|
limit,
|
|
providerId: effectiveProvider.id,
|
|
includeAttempts,
|
|
}));
|
|
const result = await enrichSearchCandidates(selectedCandidates.slice(0, limit), {
|
|
...enrichmentOptions,
|
|
searchProvider: effectiveProvider,
|
|
});
|
|
if (!dryRun) {
|
|
await writer(
|
|
{ records: result.records, attempts: result.attempts },
|
|
{ databaseUrl },
|
|
);
|
|
}
|
|
return { ...result, dryRun };
|
|
}
|
|
|
|
async function main() {
|
|
const dryRun = parseBooleanDefault(process.env.MARINE_SEARCH_DRY_RUN, true);
|
|
const limit = boundedInteger(
|
|
process.env.MARINE_SEARCH_LIMIT,
|
|
DEFAULT_SEARCH_LIMIT,
|
|
1,
|
|
MAX_SEARCH_LIMIT,
|
|
"MARINE_SEARCH_LIMIT",
|
|
);
|
|
const concurrency = boundedInteger(
|
|
process.env.MARINE_SEARCH_CONCURRENCY,
|
|
DEFAULT_SEARCH_CONCURRENCY,
|
|
1,
|
|
MAX_SEARCH_CONCURRENCY,
|
|
"MARINE_SEARCH_CONCURRENCY",
|
|
);
|
|
const searchDelayMs = boundedInteger(
|
|
process.env.MARINE_SEARCH_DELAY_MS,
|
|
DEFAULT_SEARCH_DELAY_MS,
|
|
1_000,
|
|
60_000,
|
|
"MARINE_SEARCH_DELAY_MS",
|
|
);
|
|
const hostDelayMs = boundedInteger(
|
|
process.env.MARINE_SEARCH_HOST_DELAY_MS,
|
|
DEFAULT_SITE_DELAY_MS,
|
|
250,
|
|
60_000,
|
|
"MARINE_SEARCH_HOST_DELAY_MS",
|
|
);
|
|
const timeoutMs = boundedInteger(
|
|
process.env.MARINE_SEARCH_TIMEOUT_MS,
|
|
DEFAULT_TIMEOUT_MS,
|
|
1_000,
|
|
60_000,
|
|
"MARINE_SEARCH_TIMEOUT_MS",
|
|
);
|
|
const maxResults = boundedInteger(
|
|
process.env.MARINE_SEARCH_RESULTS,
|
|
DEFAULT_SEARCH_RESULTS,
|
|
1,
|
|
MAX_SEARCH_RESULTS,
|
|
"MARINE_SEARCH_RESULTS",
|
|
);
|
|
const maxPages = boundedInteger(
|
|
process.env.MARINE_SEARCH_PAGES,
|
|
DEFAULT_SEARCH_PAGES,
|
|
1,
|
|
MAX_SEARCH_PAGES,
|
|
"MARINE_SEARCH_PAGES",
|
|
);
|
|
const providerName = (process.env.MARINE_SEARCH_PROVIDER || "duckduckgo").trim().toLowerCase();
|
|
const provider =
|
|
providerName === "brave"
|
|
? createBraveSearchProvider({ apiKey: process.env.BRAVE_SEARCH_API_KEY, timeoutMs })
|
|
: providerName === "duckduckgo"
|
|
? createDuckDuckGoProvider({ timeoutMs })
|
|
: null;
|
|
if (!provider) {
|
|
throw new Error("MARINE_SEARCH_PROVIDER muss duckduckgo oder brave sein.");
|
|
}
|
|
|
|
if (!dryRun) await ensureSearchAttemptSchema({ databaseUrl: process.env.DATABASE_URL });
|
|
console.log(
|
|
`Suchanreicherung gestartet: Provider ${provider.id}, Limit ${limit}, Parallelität ${concurrency}${dryRun ? " (Trockenlauf)" : ""}.`,
|
|
);
|
|
const result = await runSearchEnrichment({
|
|
dryRun,
|
|
limit,
|
|
databaseUrl: process.env.DATABASE_URL,
|
|
provider,
|
|
includeAttempts: !dryRun,
|
|
enrichmentOptions: {
|
|
concurrency,
|
|
searchDelayMs,
|
|
hostDelayMs,
|
|
timeoutMs,
|
|
maxResults,
|
|
maxPages,
|
|
},
|
|
});
|
|
console.log(JSON.stringify(result.stats, null, 2));
|
|
console.log(
|
|
dryRun
|
|
? `${result.records.length} Suchanreicherungen validiert; die Datenbank wurde nicht verändert.`
|
|
: `${result.records.length} Suchanreicherungen und ${result.attempts.length} Versuchsstände gespeichert.`,
|
|
);
|
|
if (result.providerError) {
|
|
console.error(
|
|
`Der Provider hat den Batch vorzeitig gestoppt: ${result.providerError.message}`,
|
|
);
|
|
process.exitCode = 2;
|
|
}
|
|
}
|
|
|
|
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
|
main().catch((error) => {
|
|
console.error(error);
|
|
process.exitCode = 1;
|
|
});
|
|
}
|