import { describe, expect, it, vi } from "vitest";
import {
buildFacilitySearchQuery,
buildSearchEnrichmentRecord,
enrichSearchCandidates,
evaluateFacilityPageMatch,
parseDuckDuckGoResults,
runSearchEnrichment,
searchBrave,
searchDuckDuckGo,
scoreSearchResult,
unwrapDuckDuckGoUrl,
} from "../../../scripts/enrich-marine-search.mjs";
const MATCH_THRESHOLD = 70;
const FETCHED_AT = "2026-07-23T09:30:00.000Z";
function searchCandidate(overrides = {}) {
return {
id: "42",
layer: "locks",
source: "osm",
sourceId: "w123",
name: "Schleuse Werries",
properties: {
"addr:city": "Hamm",
"addr:country": "DE",
waterway_name: "Datteln-Hamm-Kanal",
},
enrichmentProperties: {},
...overrides,
};
}
const matchingResult = {
url: "https://www.wsa.example/schleuse-werries",
title: "Schleuse Werries | WSA Westdeutsche Kanäle",
snippet: "Offizielle Informationen und Kontakt zur Schleuse Werries in Hamm am Datteln-Hamm-Kanal.",
};
const matchingPageHtml = `
Schleuse Werries | WSA Westdeutsche Kanäle
Schleuse Werries
Datteln-Hamm-Kanal in Hamm
`;
const identityOnlyPageHtml = `
Schleuse Werries | WSA Westdeutsche Kanäle
Schleuse Werries
Datteln-Hamm-Kanal in Hamm
`;
describe("marine facility search discovery", () => {
it("builds a stable, specific query and refuses generic facility names", () => {
const query = buildFacilitySearchQuery(searchCandidate());
expect(query).toContain('"Schleuse Werries"');
expect(query).toContain("Hamm");
expect(query).toContain("Datteln-Hamm-Kanal");
expect(query).toMatch(/Kontakt/iu);
expect(query).not.toMatch(/\b(?:undefined|null)\b/iu);
expect(
buildFacilitySearchQuery(
searchCandidate({
name: "Hafen",
layer: "harbours",
properties: { "addr:city": "Hamm" },
}),
),
).toBeNull();
expect(buildFacilitySearchQuery(searchCandidate({ name: "Schleuse", properties: {} }))).toBeNull();
expect(buildFacilitySearchQuery(searchCandidate({ name: null, properties: {} }))).toBeNull();
});
it("unwraps DuckDuckGo targets but rejects internal and unsafe links", () => {
const target = "https://www.wsa.example/schleuse-werries?view=contact";
const wrapped =
`//duckduckgo.com/l/?uddg=${encodeURIComponent(target)}` +
"&rut=0123456789";
expect(unwrapDuckDuckGoUrl(wrapped)).toBe(target);
expect(unwrapDuckDuckGoUrl(target)).toBe(target);
expect(unwrapDuckDuckGoUrl("/html/?q=schleuse+werries")).toBeNull();
expect(unwrapDuckDuckGoUrl("javascript:alert(1)")).toBeNull();
expect(unwrapDuckDuckGoUrl("mailto:test@example.test")).toBeNull();
});
it("parses organic DuckDuckGo results, decodes text and removes ads and duplicates", () => {
const wrappedTarget =
"//duckduckgo.com/l/?uddg=https%3A%2F%2Fwww.wsa.example%2Fschleuse-werries%23kontakt" +
"&rut=abc";
const html = `
Unsicher
`;
expect(parseDuckDuckGoResults(html, { limit: 10 })).toEqual([
{
url: "https://www.wsa.example/schleuse-werries",
title: "Schleuse Werries & Kontakt",
snippet: "Offizielle Informationen für Hamm. Telefon & E-Mail.",
rank: 1,
},
{
url: "https://hafen.example/kontakt?lang=de",
title: "Hafenservice Hamm",
snippet: "Ein zweiter organischer Treffer.",
rank: 2,
},
]);
expect(parseDuckDuckGoResults(html, { limit: 1 })).toHaveLength(1);
});
it("treats a DuckDuckGo browser challenge as a provider block, not as no results", async () => {
await expect(
searchDuckDuckGo("Schleuse Werries Hamm", {
fetchPageImpl: async () => ({
status: 202,
finalUrl: "https://html.duckduckgo.com/html/",
html: '',
}),
}),
).rejects.toMatchObject({ code: "PROVIDER_BLOCKED", status: 202 });
});
it("supports the authenticated Brave API through the same normalized result shape", async () => {
const fetchImpl = vi.fn(async (_url, init) => {
expect(init.headers["X-Subscription-Token"]).toBe("test-token");
return new Response(
JSON.stringify({
web: {
results: [
{
title: "Schleuse Werries",
url: "https://www.wsa.example/schleuse-werries",
description: "Kontakt in Hamm",
},
],
},
}),
{ status: 200, headers: { "content-type": "application/json" } },
);
});
await expect(
searchBrave("Schleuse Werries Hamm", {
apiKey: "test-token",
fetchImpl,
limit: 3,
}),
).resolves.toEqual([
{
title: "Schleuse Werries",
url: "https://www.wsa.example/schleuse-werries",
snippet: "Kontakt in Hamm",
rank: 1,
},
]);
});
it("requires a distinctive name plus matching place evidence before accepting a result", () => {
const accepted = scoreSearchResult(searchCandidate(), matchingResult);
const wrongPlace = scoreSearchResult(searchCandidate(), {
...matchingResult,
url: "https://tourismus.example/amsterdam/werries",
title: "Schleuse Werries in Amsterdam",
snippet: "Besuchen Sie die historische Schleuse in Amsterdam, Noord-Holland.",
});
const missingName = scoreSearchResult(searchCandidate(), {
url: "https://www.hamm.example/schleusen",
title: "Wasserstraßen und Schleusen in Hamm",
snippet: "Kontakt für den Datteln-Hamm-Kanal.",
});
const genericName = scoreSearchResult(
searchCandidate({ name: "Schleuse", properties: { "addr:city": "Hamm" } }),
matchingResult,
);
expect(accepted.accepted).toBe(true);
expect(accepted.score).toBeGreaterThanOrEqual(MATCH_THRESHOLD);
expect(accepted.evidence.length).toBeGreaterThan(0);
expect(wrongPlace.accepted).toBe(false);
expect(wrongPlace.score).toBeLessThan(MATCH_THRESHOLD);
expect(missingName.accepted).toBe(false);
expect(genericName).toMatchObject({ accepted: false, score: 0 });
});
it("validates the fetched page itself and rejects a misleading redirect or generic homepage", () => {
const accepted = evaluateFacilityPageMatch(searchCandidate(), {
html: matchingPageHtml,
finalUrl: "https://www.wsa.example/schleuse-werries",
});
const genericHomepage = evaluateFacilityPageMatch(searchCandidate(), {
html: `
WSA Westdeutsche Kanäle
Willkommen
Allgemeine Informationen zur Wasserstraßenverwaltung.
`,
finalUrl: "https://www.wsa.example/",
});
const wrongFacility = evaluateFacilityPageMatch(searchCandidate(), {
html: `
Schleuse Werries Amsterdam
Schleuse Werries
Amsterdam, NL
`,
finalUrl: "https://tourismus.example/amsterdam/werries",
});
expect(accepted.accepted).toBe(true);
expect(accepted.score).toBeGreaterThanOrEqual(MATCH_THRESHOLD);
expect(accepted.evidence.length).toBeGreaterThan(0);
expect(genericHomepage.accepted).toBe(false);
expect(wrongFacility.accepted).toBe(false);
});
it("stores the verified website with auditable provenance and never overwrites existing contacts", () => {
const candidate = searchCandidate({
properties: {
"addr:city": "Hamm",
phone: "+49 2381 100",
},
enrichmentProperties: {
operator: "Vorhandener Betreiber",
},
});
const query = buildFacilitySearchQuery(candidate);
const page = {
html: matchingPageHtml,
finalUrl: "https://www.wsa.example/anlagen/schleuse-werries",
};
const match = evaluateFacilityPageMatch(candidate, page);
const record = buildSearchEnrichmentRecord({
candidate,
query,
providerId: "duckduckgo",
searchResult: matchingResult,
page,
match,
extracted: {
phone: "+49 2381 999",
email: "schleuse-werries@example.test",
operator: "Anderer Betreiber",
address: null,
},
fetchedAt: FETCHED_AT,
});
expect(record).toMatchObject({
originalId: "42",
sourceId: "osm:w123",
properties: {
website: "https://www.wsa.example/anlagen/schleuse-werries",
phone: "+49 2381 100",
email: "schleuse-werries@example.test",
operator: "Vorhandener Betreiber",
original_source: "osm",
original_source_id: "w123",
enrichmentSource: "facility-search",
enrichmentProvider: "duckduckgo",
searchQuery: query,
searchResultUrl: matchingResult.url,
source_url: page.finalUrl,
fetched_at: FETCHED_AT,
},
});
expect(record.properties.searchScore).toBeGreaterThanOrEqual(MATCH_THRESHOLD);
expect(record.properties.enriched_fields).toEqual(
expect.arrayContaining(["website", "email"]),
);
expect(record.properties.enriched_fields).not.toContain("phone");
expect(record.properties.enriched_fields).not.toContain("operator");
});
it("persists a verified discovered website even when the page exposes no contact fields", async () => {
const search = vi.fn(async () => [matchingResult]);
const fetchPageImpl = vi.fn(async () => ({
html: identityOnlyPageHtml,
finalUrl: matchingResult.url,
redirects: 0,
}));
const result = await enrichSearchCandidates([searchCandidate()], {
searchProvider: { id: "duckduckgo", search },
fetchPageImpl,
concurrency: 1,
hostDelayMs: 0,
maxResults: 5,
maxPages: 2,
fetchedAt: FETCHED_AT,
logger: { warn: vi.fn() },
});
expect(search).toHaveBeenCalledTimes(1);
expect(search.mock.calls[0][0]).toContain("Schleuse Werries");
expect(fetchPageImpl).toHaveBeenCalledTimes(1);
expect(result.records).toHaveLength(1);
expect(result.records[0]).toMatchObject({
originalId: "42",
sourceId: "osm:w123",
properties: {
website: matchingResult.url,
enrichmentProvider: "duckduckgo",
fetched_at: FETCHED_AT,
},
});
expect(result.records[0].properties.enriched_fields).toContain("website");
});
it("opens the provider circuit after a block response and does not continue querying", async () => {
const blockedError = Object.assign(new Error("DuckDuckGo hat weitere Anfragen blockiert."), {
code: "SEARCH_PROVIDER_BLOCKED",
status: 429,
});
const search = vi.fn(async () => {
throw blockedError;
});
const fetchPageImpl = vi.fn();
const logger = { warn: vi.fn() };
const result = await enrichSearchCandidates(
[
searchCandidate(),
searchCandidate({
id: "43",
sourceId: "w124",
name: "Schleuse Uentrop",
properties: { "addr:city": "Hamm" },
}),
],
{
searchProvider: { id: "duckduckgo", search },
fetchPageImpl,
concurrency: 1,
hostDelayMs: 0,
fetchedAt: FETCHED_AT,
logger,
},
);
expect(result.records).toEqual([]);
expect(search).toHaveBeenCalledTimes(1);
expect(fetchPageImpl).not.toHaveBeenCalled();
expect(logger.warn).toHaveBeenCalled();
});
it("keeps the database untouched in the default dry-run", async () => {
const writer = vi.fn();
const search = vi.fn(async () => [matchingResult]);
const fetchPageImpl = vi.fn(async () => ({
html: identityOnlyPageHtml,
finalUrl: matchingResult.url,
redirects: 0,
}));
const result = await runSearchEnrichment({
candidates: [searchCandidate()],
writer,
enrichmentOptions: {
searchProvider: { id: "duckduckgo", search },
fetchPageImpl,
concurrency: 1,
hostDelayMs: 0,
fetchedAt: FETCHED_AT,
},
});
expect(result.dryRun).toBe(true);
expect(result.records).toHaveLength(1);
expect(writer).not.toHaveBeenCalled();
});
it("passes validated records and checkpoint attempts to the writer only when enabled", async () => {
const writer = vi.fn(async () => ({ enrichments: 1, attempts: 1 }));
const result = await runSearchEnrichment({
dryRun: false,
candidates: [searchCandidate()],
writer,
enrichmentOptions: {
searchProvider: { id: "duckduckgo", search: async () => [matchingResult] },
fetchPageImpl: async () => ({
html: identityOnlyPageHtml,
finalUrl: matchingResult.url,
redirects: 0,
}),
concurrency: 1,
hostDelayMs: 0,
fetchedAt: FETCHED_AT,
},
});
expect(result.records).toHaveLength(1);
expect(result.attempts).toHaveLength(1);
expect(result.attempts[0]).toMatchObject({
status: "success",
provider: "duckduckgo",
originalSource: "osm",
originalSourceId: "w123",
});
expect(writer).toHaveBeenCalledTimes(1);
expect(writer.mock.calls[0][0]).toMatchObject({
records: [expect.objectContaining({ sourceId: "osm:w123" })],
attempts: [expect.objectContaining({ status: "success" })],
});
});
});