468 lines
15 KiB
JavaScript
468 lines
15 KiB
JavaScript
import { describe, expect, it, vi } from "vitest";
|
|
import {
|
|
buildFacilitySearchQuery,
|
|
buildSearchEnrichmentRecord,
|
|
enrichSearchCandidates,
|
|
evaluateFacilityPageMatch,
|
|
parseDuckDuckGoResults,
|
|
runSearchEnrichment,
|
|
searchBrave,
|
|
searchDuckDuckGo,
|
|
scoreSearchResult,
|
|
unwrapDuckDuckGoUrl,
|
|
} from "../../../scripts/enrich-marine-search.mjs";
|
|
|
|
const MATCH_THRESHOLD = 70;
|
|
const FETCHED_AT = "2026-07-23T09:30:00.000Z";
|
|
|
|
function searchCandidate(overrides = {}) {
|
|
return {
|
|
id: "42",
|
|
layer: "locks",
|
|
source: "osm",
|
|
sourceId: "w123",
|
|
name: "Schleuse Werries",
|
|
properties: {
|
|
"addr:city": "Hamm",
|
|
"addr:country": "DE",
|
|
waterway_name: "Datteln-Hamm-Kanal",
|
|
},
|
|
enrichmentProperties: {},
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
const matchingResult = {
|
|
url: "https://www.wsa.example/schleuse-werries",
|
|
title: "Schleuse Werries | WSA Westdeutsche Kanäle",
|
|
snippet: "Offizielle Informationen und Kontakt zur Schleuse Werries in Hamm am Datteln-Hamm-Kanal.",
|
|
};
|
|
|
|
const matchingPageHtml = `
|
|
<!doctype html>
|
|
<html lang="de">
|
|
<head>
|
|
<title>Schleuse Werries | WSA Westdeutsche Kanäle</title>
|
|
<script type="application/ld+json">
|
|
{
|
|
"@context": "https://schema.org",
|
|
"@type": "GovernmentOrganization",
|
|
"name": "WSA Westdeutsche Kanäle",
|
|
"address": {
|
|
"@type": "PostalAddress",
|
|
"addressLocality": "Hamm",
|
|
"addressCountry": "DE"
|
|
}
|
|
}
|
|
</script>
|
|
</head>
|
|
<body>
|
|
<main>
|
|
<h1>Schleuse Werries</h1>
|
|
<p>Datteln-Hamm-Kanal in Hamm</p>
|
|
</main>
|
|
</body>
|
|
</html>
|
|
`;
|
|
|
|
const identityOnlyPageHtml = `
|
|
<!doctype html>
|
|
<html lang="de">
|
|
<head><title>Schleuse Werries | WSA Westdeutsche Kanäle</title></head>
|
|
<body>
|
|
<main>
|
|
<h1>Schleuse Werries</h1>
|
|
<p>Datteln-Hamm-Kanal in Hamm</p>
|
|
</main>
|
|
</body>
|
|
</html>
|
|
`;
|
|
|
|
describe("marine facility search discovery", () => {
|
|
it("builds a stable, specific query and refuses generic facility names", () => {
|
|
const query = buildFacilitySearchQuery(searchCandidate());
|
|
|
|
expect(query).toContain('"Schleuse Werries"');
|
|
expect(query).toContain("Hamm");
|
|
expect(query).toContain("Datteln-Hamm-Kanal");
|
|
expect(query).toMatch(/Kontakt/iu);
|
|
expect(query).not.toMatch(/\b(?:undefined|null)\b/iu);
|
|
|
|
expect(
|
|
buildFacilitySearchQuery(
|
|
searchCandidate({
|
|
name: "Hafen",
|
|
layer: "harbours",
|
|
properties: { "addr:city": "Hamm" },
|
|
}),
|
|
),
|
|
).toBeNull();
|
|
expect(buildFacilitySearchQuery(searchCandidate({ name: "Schleuse", properties: {} }))).toBeNull();
|
|
expect(buildFacilitySearchQuery(searchCandidate({ name: null, properties: {} }))).toBeNull();
|
|
});
|
|
|
|
it("unwraps DuckDuckGo targets but rejects internal and unsafe links", () => {
|
|
const target = "https://www.wsa.example/schleuse-werries?view=contact";
|
|
const wrapped =
|
|
`//duckduckgo.com/l/?uddg=${encodeURIComponent(target)}` +
|
|
"&rut=0123456789";
|
|
|
|
expect(unwrapDuckDuckGoUrl(wrapped)).toBe(target);
|
|
expect(unwrapDuckDuckGoUrl(target)).toBe(target);
|
|
expect(unwrapDuckDuckGoUrl("/html/?q=schleuse+werries")).toBeNull();
|
|
expect(unwrapDuckDuckGoUrl("javascript:alert(1)")).toBeNull();
|
|
expect(unwrapDuckDuckGoUrl("mailto:test@example.test")).toBeNull();
|
|
});
|
|
|
|
it("parses organic DuckDuckGo results, decodes text and removes ads and duplicates", () => {
|
|
const wrappedTarget =
|
|
"//duckduckgo.com/l/?uddg=https%3A%2F%2Fwww.wsa.example%2Fschleuse-werries%23kontakt" +
|
|
"&rut=abc";
|
|
const html = `
|
|
<div class="result results_links results_links_deep web-result">
|
|
<h2 class="result__title">
|
|
<a rel="nofollow" class="result__a" href="${wrappedTarget}">
|
|
Schleuse <b>Werries</b> & Kontakt
|
|
</a>
|
|
</h2>
|
|
<a class="result__snippet">
|
|
Offizielle Informationen für Hamm. Telefon & E-Mail.
|
|
</a>
|
|
</div>
|
|
<div class="result result--ad">
|
|
<h2>
|
|
<a class="result__a" href="https://advertising.example/werries">
|
|
Anzeige für Werries
|
|
</a>
|
|
</h2>
|
|
<span class="result__badge">Ad</span>
|
|
</div>
|
|
<div class="result">
|
|
<a class="result__a" href="https://www.wsa.example/schleuse-werries#anfahrt">
|
|
Derselbe Treffer ein zweites Mal
|
|
</a>
|
|
<a class="result__snippet">Duplikat</a>
|
|
</div>
|
|
<div class="result">
|
|
<a class="result__a" href="https://hafen.example/kontakt?lang=de">
|
|
Hafenservice Hamm
|
|
</a>
|
|
<a class="result__snippet">Ein zweiter organischer Treffer.</a>
|
|
</div>
|
|
<a href="javascript:alert(1)" class="result__a">Unsicher</a>
|
|
`;
|
|
|
|
expect(parseDuckDuckGoResults(html, { limit: 10 })).toEqual([
|
|
{
|
|
url: "https://www.wsa.example/schleuse-werries",
|
|
title: "Schleuse Werries & Kontakt",
|
|
snippet: "Offizielle Informationen für Hamm. Telefon & E-Mail.",
|
|
rank: 1,
|
|
},
|
|
{
|
|
url: "https://hafen.example/kontakt?lang=de",
|
|
title: "Hafenservice Hamm",
|
|
snippet: "Ein zweiter organischer Treffer.",
|
|
rank: 2,
|
|
},
|
|
]);
|
|
expect(parseDuckDuckGoResults(html, { limit: 1 })).toHaveLength(1);
|
|
});
|
|
|
|
it("treats a DuckDuckGo browser challenge as a provider block, not as no results", async () => {
|
|
await expect(
|
|
searchDuckDuckGo("Schleuse Werries Hamm", {
|
|
fetchPageImpl: async () => ({
|
|
status: 202,
|
|
finalUrl: "https://html.duckduckgo.com/html/",
|
|
html: '<form id="challenge-form"><script src="/anomaly.js"></script></form>',
|
|
}),
|
|
}),
|
|
).rejects.toMatchObject({ code: "PROVIDER_BLOCKED", status: 202 });
|
|
});
|
|
|
|
it("supports the authenticated Brave API through the same normalized result shape", async () => {
|
|
const fetchImpl = vi.fn(async (_url, init) => {
|
|
expect(init.headers["X-Subscription-Token"]).toBe("test-token");
|
|
return new Response(
|
|
JSON.stringify({
|
|
web: {
|
|
results: [
|
|
{
|
|
title: "Schleuse Werries",
|
|
url: "https://www.wsa.example/schleuse-werries",
|
|
description: "Kontakt in Hamm",
|
|
},
|
|
],
|
|
},
|
|
}),
|
|
{ status: 200, headers: { "content-type": "application/json" } },
|
|
);
|
|
});
|
|
|
|
await expect(
|
|
searchBrave("Schleuse Werries Hamm", {
|
|
apiKey: "test-token",
|
|
fetchImpl,
|
|
limit: 3,
|
|
}),
|
|
).resolves.toEqual([
|
|
{
|
|
title: "Schleuse Werries",
|
|
url: "https://www.wsa.example/schleuse-werries",
|
|
snippet: "Kontakt in Hamm",
|
|
rank: 1,
|
|
},
|
|
]);
|
|
});
|
|
|
|
it("requires a distinctive name plus matching place evidence before accepting a result", () => {
|
|
const accepted = scoreSearchResult(searchCandidate(), matchingResult);
|
|
const wrongPlace = scoreSearchResult(searchCandidate(), {
|
|
...matchingResult,
|
|
url: "https://tourismus.example/amsterdam/werries",
|
|
title: "Schleuse Werries in Amsterdam",
|
|
snippet: "Besuchen Sie die historische Schleuse in Amsterdam, Noord-Holland.",
|
|
});
|
|
const missingName = scoreSearchResult(searchCandidate(), {
|
|
url: "https://www.hamm.example/schleusen",
|
|
title: "Wasserstraßen und Schleusen in Hamm",
|
|
snippet: "Kontakt für den Datteln-Hamm-Kanal.",
|
|
});
|
|
const genericName = scoreSearchResult(
|
|
searchCandidate({ name: "Schleuse", properties: { "addr:city": "Hamm" } }),
|
|
matchingResult,
|
|
);
|
|
|
|
expect(accepted.accepted).toBe(true);
|
|
expect(accepted.score).toBeGreaterThanOrEqual(MATCH_THRESHOLD);
|
|
expect(accepted.evidence.length).toBeGreaterThan(0);
|
|
|
|
expect(wrongPlace.accepted).toBe(false);
|
|
expect(wrongPlace.score).toBeLessThan(MATCH_THRESHOLD);
|
|
expect(missingName.accepted).toBe(false);
|
|
expect(genericName).toMatchObject({ accepted: false, score: 0 });
|
|
});
|
|
|
|
it("validates the fetched page itself and rejects a misleading redirect or generic homepage", () => {
|
|
const accepted = evaluateFacilityPageMatch(searchCandidate(), {
|
|
html: matchingPageHtml,
|
|
finalUrl: "https://www.wsa.example/schleuse-werries",
|
|
});
|
|
const genericHomepage = evaluateFacilityPageMatch(searchCandidate(), {
|
|
html: `
|
|
<html>
|
|
<head><title>WSA Westdeutsche Kanäle</title></head>
|
|
<body><h1>Willkommen</h1><p>Allgemeine Informationen zur Wasserstraßenverwaltung.</p></body>
|
|
</html>
|
|
`,
|
|
finalUrl: "https://www.wsa.example/",
|
|
});
|
|
const wrongFacility = evaluateFacilityPageMatch(searchCandidate(), {
|
|
html: `
|
|
<html>
|
|
<head><title>Schleuse Werries Amsterdam</title></head>
|
|
<body><h1>Schleuse Werries</h1><address>Amsterdam, NL</address></body>
|
|
</html>
|
|
`,
|
|
finalUrl: "https://tourismus.example/amsterdam/werries",
|
|
});
|
|
|
|
expect(accepted.accepted).toBe(true);
|
|
expect(accepted.score).toBeGreaterThanOrEqual(MATCH_THRESHOLD);
|
|
expect(accepted.evidence.length).toBeGreaterThan(0);
|
|
expect(genericHomepage.accepted).toBe(false);
|
|
expect(wrongFacility.accepted).toBe(false);
|
|
});
|
|
|
|
it("stores the verified website with auditable provenance and never overwrites existing contacts", () => {
|
|
const candidate = searchCandidate({
|
|
properties: {
|
|
"addr:city": "Hamm",
|
|
phone: "+49 2381 100",
|
|
},
|
|
enrichmentProperties: {
|
|
operator: "Vorhandener Betreiber",
|
|
},
|
|
});
|
|
const query = buildFacilitySearchQuery(candidate);
|
|
const page = {
|
|
html: matchingPageHtml,
|
|
finalUrl: "https://www.wsa.example/anlagen/schleuse-werries",
|
|
};
|
|
const match = evaluateFacilityPageMatch(candidate, page);
|
|
const record = buildSearchEnrichmentRecord({
|
|
candidate,
|
|
query,
|
|
providerId: "duckduckgo",
|
|
searchResult: matchingResult,
|
|
page,
|
|
match,
|
|
extracted: {
|
|
phone: "+49 2381 999",
|
|
email: "schleuse-werries@example.test",
|
|
operator: "Anderer Betreiber",
|
|
address: null,
|
|
},
|
|
fetchedAt: FETCHED_AT,
|
|
});
|
|
|
|
expect(record).toMatchObject({
|
|
originalId: "42",
|
|
sourceId: "osm:w123",
|
|
properties: {
|
|
website: "https://www.wsa.example/anlagen/schleuse-werries",
|
|
phone: "+49 2381 100",
|
|
email: "schleuse-werries@example.test",
|
|
operator: "Vorhandener Betreiber",
|
|
original_source: "osm",
|
|
original_source_id: "w123",
|
|
enrichmentSource: "facility-search",
|
|
enrichmentProvider: "duckduckgo",
|
|
searchQuery: query,
|
|
searchResultUrl: matchingResult.url,
|
|
source_url: page.finalUrl,
|
|
fetched_at: FETCHED_AT,
|
|
},
|
|
});
|
|
expect(record.properties.searchScore).toBeGreaterThanOrEqual(MATCH_THRESHOLD);
|
|
expect(record.properties.enriched_fields).toEqual(
|
|
expect.arrayContaining(["website", "email"]),
|
|
);
|
|
expect(record.properties.enriched_fields).not.toContain("phone");
|
|
expect(record.properties.enriched_fields).not.toContain("operator");
|
|
});
|
|
|
|
it("persists a verified discovered website even when the page exposes no contact fields", async () => {
|
|
const search = vi.fn(async () => [matchingResult]);
|
|
const fetchPageImpl = vi.fn(async () => ({
|
|
html: identityOnlyPageHtml,
|
|
finalUrl: matchingResult.url,
|
|
redirects: 0,
|
|
}));
|
|
|
|
const result = await enrichSearchCandidates([searchCandidate()], {
|
|
searchProvider: { id: "duckduckgo", search },
|
|
fetchPageImpl,
|
|
concurrency: 1,
|
|
hostDelayMs: 0,
|
|
maxResults: 5,
|
|
maxPages: 2,
|
|
fetchedAt: FETCHED_AT,
|
|
logger: { warn: vi.fn() },
|
|
});
|
|
|
|
expect(search).toHaveBeenCalledTimes(1);
|
|
expect(search.mock.calls[0][0]).toContain("Schleuse Werries");
|
|
expect(fetchPageImpl).toHaveBeenCalledTimes(1);
|
|
expect(result.records).toHaveLength(1);
|
|
expect(result.records[0]).toMatchObject({
|
|
originalId: "42",
|
|
sourceId: "osm:w123",
|
|
properties: {
|
|
website: matchingResult.url,
|
|
enrichmentProvider: "duckduckgo",
|
|
fetched_at: FETCHED_AT,
|
|
},
|
|
});
|
|
expect(result.records[0].properties.enriched_fields).toContain("website");
|
|
});
|
|
|
|
it("opens the provider circuit after a block response and does not continue querying", async () => {
|
|
const blockedError = Object.assign(new Error("DuckDuckGo hat weitere Anfragen blockiert."), {
|
|
code: "SEARCH_PROVIDER_BLOCKED",
|
|
status: 429,
|
|
});
|
|
const search = vi.fn(async () => {
|
|
throw blockedError;
|
|
});
|
|
const fetchPageImpl = vi.fn();
|
|
const logger = { warn: vi.fn() };
|
|
|
|
const result = await enrichSearchCandidates(
|
|
[
|
|
searchCandidate(),
|
|
searchCandidate({
|
|
id: "43",
|
|
sourceId: "w124",
|
|
name: "Schleuse Uentrop",
|
|
properties: { "addr:city": "Hamm" },
|
|
}),
|
|
],
|
|
{
|
|
searchProvider: { id: "duckduckgo", search },
|
|
fetchPageImpl,
|
|
concurrency: 1,
|
|
hostDelayMs: 0,
|
|
fetchedAt: FETCHED_AT,
|
|
logger,
|
|
},
|
|
);
|
|
|
|
expect(result.records).toEqual([]);
|
|
expect(search).toHaveBeenCalledTimes(1);
|
|
expect(fetchPageImpl).not.toHaveBeenCalled();
|
|
expect(logger.warn).toHaveBeenCalled();
|
|
});
|
|
|
|
it("keeps the database untouched in the default dry-run", async () => {
|
|
const writer = vi.fn();
|
|
const search = vi.fn(async () => [matchingResult]);
|
|
const fetchPageImpl = vi.fn(async () => ({
|
|
html: identityOnlyPageHtml,
|
|
finalUrl: matchingResult.url,
|
|
redirects: 0,
|
|
}));
|
|
|
|
const result = await runSearchEnrichment({
|
|
candidates: [searchCandidate()],
|
|
writer,
|
|
enrichmentOptions: {
|
|
searchProvider: { id: "duckduckgo", search },
|
|
fetchPageImpl,
|
|
concurrency: 1,
|
|
hostDelayMs: 0,
|
|
fetchedAt: FETCHED_AT,
|
|
},
|
|
});
|
|
|
|
expect(result.dryRun).toBe(true);
|
|
expect(result.records).toHaveLength(1);
|
|
expect(writer).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it("passes validated records and checkpoint attempts to the writer only when enabled", async () => {
|
|
const writer = vi.fn(async () => ({ enrichments: 1, attempts: 1 }));
|
|
const result = await runSearchEnrichment({
|
|
dryRun: false,
|
|
candidates: [searchCandidate()],
|
|
writer,
|
|
enrichmentOptions: {
|
|
searchProvider: { id: "duckduckgo", search: async () => [matchingResult] },
|
|
fetchPageImpl: async () => ({
|
|
html: identityOnlyPageHtml,
|
|
finalUrl: matchingResult.url,
|
|
redirects: 0,
|
|
}),
|
|
concurrency: 1,
|
|
hostDelayMs: 0,
|
|
fetchedAt: FETCHED_AT,
|
|
},
|
|
});
|
|
|
|
expect(result.records).toHaveLength(1);
|
|
expect(result.attempts).toHaveLength(1);
|
|
expect(result.attempts[0]).toMatchObject({
|
|
status: "success",
|
|
provider: "duckduckgo",
|
|
originalSource: "osm",
|
|
originalSourceId: "w123",
|
|
});
|
|
expect(writer).toHaveBeenCalledTimes(1);
|
|
expect(writer.mock.calls[0][0]).toMatchObject({
|
|
records: [expect.objectContaining({ sourceId: "osm:w123" })],
|
|
attempts: [expect.objectContaining({ status: "success" })],
|
|
});
|
|
});
|
|
});
|