Initial Watermaps import
This commit is contained in:
BuTzZ
@@ -0,0 +1,467 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
buildFacilitySearchQuery,
|
||||
buildSearchEnrichmentRecord,
|
||||
enrichSearchCandidates,
|
||||
evaluateFacilityPageMatch,
|
||||
parseDuckDuckGoResults,
|
||||
runSearchEnrichment,
|
||||
searchBrave,
|
||||
searchDuckDuckGo,
|
||||
scoreSearchResult,
|
||||
unwrapDuckDuckGoUrl,
|
||||
} from "../../../scripts/enrich-marine-search.mjs";
|
||||
|
||||
const MATCH_THRESHOLD = 70;
|
||||
const FETCHED_AT = "2026-07-23T09:30:00.000Z";
|
||||
|
||||
function searchCandidate(overrides = {}) {
|
||||
return {
|
||||
id: "42",
|
||||
layer: "locks",
|
||||
source: "osm",
|
||||
sourceId: "w123",
|
||||
name: "Schleuse Werries",
|
||||
properties: {
|
||||
"addr:city": "Hamm",
|
||||
"addr:country": "DE",
|
||||
waterway_name: "Datteln-Hamm-Kanal",
|
||||
},
|
||||
enrichmentProperties: {},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
const matchingResult = {
|
||||
url: "https://www.wsa.example/schleuse-werries",
|
||||
title: "Schleuse Werries | WSA Westdeutsche Kanäle",
|
||||
snippet: "Offizielle Informationen und Kontakt zur Schleuse Werries in Hamm am Datteln-Hamm-Kanal.",
|
||||
};
|
||||
|
||||
const matchingPageHtml = `
|
||||
<!doctype html>
|
||||
<html lang="de">
|
||||
<head>
|
||||
<title>Schleuse Werries | WSA Westdeutsche Kanäle</title>
|
||||
<script type="application/ld+json">
|
||||
{
|
||||
"@context": "https://schema.org",
|
||||
"@type": "GovernmentOrganization",
|
||||
"name": "WSA Westdeutsche Kanäle",
|
||||
"address": {
|
||||
"@type": "PostalAddress",
|
||||
"addressLocality": "Hamm",
|
||||
"addressCountry": "DE"
|
||||
}
|
||||
}
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<h1>Schleuse Werries</h1>
|
||||
<p>Datteln-Hamm-Kanal in Hamm</p>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
`;
|
||||
|
||||
const identityOnlyPageHtml = `
|
||||
<!doctype html>
|
||||
<html lang="de">
|
||||
<head><title>Schleuse Werries | WSA Westdeutsche Kanäle</title></head>
|
||||
<body>
|
||||
<main>
|
||||
<h1>Schleuse Werries</h1>
|
||||
<p>Datteln-Hamm-Kanal in Hamm</p>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
`;
|
||||
|
||||
describe("marine facility search discovery", () => {
|
||||
it("builds a stable, specific query and refuses generic facility names", () => {
|
||||
const query = buildFacilitySearchQuery(searchCandidate());
|
||||
|
||||
expect(query).toContain('"Schleuse Werries"');
|
||||
expect(query).toContain("Hamm");
|
||||
expect(query).toContain("Datteln-Hamm-Kanal");
|
||||
expect(query).toMatch(/Kontakt/iu);
|
||||
expect(query).not.toMatch(/\b(?:undefined|null)\b/iu);
|
||||
|
||||
expect(
|
||||
buildFacilitySearchQuery(
|
||||
searchCandidate({
|
||||
name: "Hafen",
|
||||
layer: "harbours",
|
||||
properties: { "addr:city": "Hamm" },
|
||||
}),
|
||||
),
|
||||
).toBeNull();
|
||||
expect(buildFacilitySearchQuery(searchCandidate({ name: "Schleuse", properties: {} }))).toBeNull();
|
||||
expect(buildFacilitySearchQuery(searchCandidate({ name: null, properties: {} }))).toBeNull();
|
||||
});
|
||||
|
||||
it("unwraps DuckDuckGo targets but rejects internal and unsafe links", () => {
|
||||
const target = "https://www.wsa.example/schleuse-werries?view=contact";
|
||||
const wrapped =
|
||||
`//duckduckgo.com/l/?uddg=${encodeURIComponent(target)}` +
|
||||
"&rut=0123456789";
|
||||
|
||||
expect(unwrapDuckDuckGoUrl(wrapped)).toBe(target);
|
||||
expect(unwrapDuckDuckGoUrl(target)).toBe(target);
|
||||
expect(unwrapDuckDuckGoUrl("/html/?q=schleuse+werries")).toBeNull();
|
||||
expect(unwrapDuckDuckGoUrl("javascript:alert(1)")).toBeNull();
|
||||
expect(unwrapDuckDuckGoUrl("mailto:test@example.test")).toBeNull();
|
||||
});
|
||||
|
||||
it("parses organic DuckDuckGo results, decodes text and removes ads and duplicates", () => {
|
||||
const wrappedTarget =
|
||||
"//duckduckgo.com/l/?uddg=https%3A%2F%2Fwww.wsa.example%2Fschleuse-werries%23kontakt" +
|
||||
"&rut=abc";
|
||||
const html = `
|
||||
<div class="result results_links results_links_deep web-result">
|
||||
<h2 class="result__title">
|
||||
<a rel="nofollow" class="result__a" href="${wrappedTarget}">
|
||||
Schleuse <b>Werries</b> & Kontakt
|
||||
</a>
|
||||
</h2>
|
||||
<a class="result__snippet">
|
||||
Offizielle Informationen für Hamm. Telefon & E-Mail.
|
||||
</a>
|
||||
</div>
|
||||
<div class="result result--ad">
|
||||
<h2>
|
||||
<a class="result__a" href="https://advertising.example/werries">
|
||||
Anzeige für Werries
|
||||
</a>
|
||||
</h2>
|
||||
<span class="result__badge">Ad</span>
|
||||
</div>
|
||||
<div class="result">
|
||||
<a class="result__a" href="https://www.wsa.example/schleuse-werries#anfahrt">
|
||||
Derselbe Treffer ein zweites Mal
|
||||
</a>
|
||||
<a class="result__snippet">Duplikat</a>
|
||||
</div>
|
||||
<div class="result">
|
||||
<a class="result__a" href="https://hafen.example/kontakt?lang=de">
|
||||
Hafenservice Hamm
|
||||
</a>
|
||||
<a class="result__snippet">Ein zweiter organischer Treffer.</a>
|
||||
</div>
|
||||
<a href="javascript:alert(1)" class="result__a">Unsicher</a>
|
||||
`;
|
||||
|
||||
expect(parseDuckDuckGoResults(html, { limit: 10 })).toEqual([
|
||||
{
|
||||
url: "https://www.wsa.example/schleuse-werries",
|
||||
title: "Schleuse Werries & Kontakt",
|
||||
snippet: "Offizielle Informationen für Hamm. Telefon & E-Mail.",
|
||||
rank: 1,
|
||||
},
|
||||
{
|
||||
url: "https://hafen.example/kontakt?lang=de",
|
||||
title: "Hafenservice Hamm",
|
||||
snippet: "Ein zweiter organischer Treffer.",
|
||||
rank: 2,
|
||||
},
|
||||
]);
|
||||
expect(parseDuckDuckGoResults(html, { limit: 1 })).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("treats a DuckDuckGo browser challenge as a provider block, not as no results", async () => {
|
||||
await expect(
|
||||
searchDuckDuckGo("Schleuse Werries Hamm", {
|
||||
fetchPageImpl: async () => ({
|
||||
status: 202,
|
||||
finalUrl: "https://html.duckduckgo.com/html/",
|
||||
html: '<form id="challenge-form"><script src="/anomaly.js"></script></form>',
|
||||
}),
|
||||
}),
|
||||
).rejects.toMatchObject({ code: "PROVIDER_BLOCKED", status: 202 });
|
||||
});
|
||||
|
||||
it("supports the authenticated Brave API through the same normalized result shape", async () => {
|
||||
const fetchImpl = vi.fn(async (_url, init) => {
|
||||
expect(init.headers["X-Subscription-Token"]).toBe("test-token");
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
web: {
|
||||
results: [
|
||||
{
|
||||
title: "Schleuse Werries",
|
||||
url: "https://www.wsa.example/schleuse-werries",
|
||||
description: "Kontakt in Hamm",
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
{ status: 200, headers: { "content-type": "application/json" } },
|
||||
);
|
||||
});
|
||||
|
||||
await expect(
|
||||
searchBrave("Schleuse Werries Hamm", {
|
||||
apiKey: "test-token",
|
||||
fetchImpl,
|
||||
limit: 3,
|
||||
}),
|
||||
).resolves.toEqual([
|
||||
{
|
||||
title: "Schleuse Werries",
|
||||
url: "https://www.wsa.example/schleuse-werries",
|
||||
snippet: "Kontakt in Hamm",
|
||||
rank: 1,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("requires a distinctive name plus matching place evidence before accepting a result", () => {
|
||||
const accepted = scoreSearchResult(searchCandidate(), matchingResult);
|
||||
const wrongPlace = scoreSearchResult(searchCandidate(), {
|
||||
...matchingResult,
|
||||
url: "https://tourismus.example/amsterdam/werries",
|
||||
title: "Schleuse Werries in Amsterdam",
|
||||
snippet: "Besuchen Sie die historische Schleuse in Amsterdam, Noord-Holland.",
|
||||
});
|
||||
const missingName = scoreSearchResult(searchCandidate(), {
|
||||
url: "https://www.hamm.example/schleusen",
|
||||
title: "Wasserstraßen und Schleusen in Hamm",
|
||||
snippet: "Kontakt für den Datteln-Hamm-Kanal.",
|
||||
});
|
||||
const genericName = scoreSearchResult(
|
||||
searchCandidate({ name: "Schleuse", properties: { "addr:city": "Hamm" } }),
|
||||
matchingResult,
|
||||
);
|
||||
|
||||
expect(accepted.accepted).toBe(true);
|
||||
expect(accepted.score).toBeGreaterThanOrEqual(MATCH_THRESHOLD);
|
||||
expect(accepted.evidence.length).toBeGreaterThan(0);
|
||||
|
||||
expect(wrongPlace.accepted).toBe(false);
|
||||
expect(wrongPlace.score).toBeLessThan(MATCH_THRESHOLD);
|
||||
expect(missingName.accepted).toBe(false);
|
||||
expect(genericName).toMatchObject({ accepted: false, score: 0 });
|
||||
});
|
||||
|
||||
it("validates the fetched page itself and rejects a misleading redirect or generic homepage", () => {
|
||||
const accepted = evaluateFacilityPageMatch(searchCandidate(), {
|
||||
html: matchingPageHtml,
|
||||
finalUrl: "https://www.wsa.example/schleuse-werries",
|
||||
});
|
||||
const genericHomepage = evaluateFacilityPageMatch(searchCandidate(), {
|
||||
html: `
|
||||
<html>
|
||||
<head><title>WSA Westdeutsche Kanäle</title></head>
|
||||
<body><h1>Willkommen</h1><p>Allgemeine Informationen zur Wasserstraßenverwaltung.</p></body>
|
||||
</html>
|
||||
`,
|
||||
finalUrl: "https://www.wsa.example/",
|
||||
});
|
||||
const wrongFacility = evaluateFacilityPageMatch(searchCandidate(), {
|
||||
html: `
|
||||
<html>
|
||||
<head><title>Schleuse Werries Amsterdam</title></head>
|
||||
<body><h1>Schleuse Werries</h1><address>Amsterdam, NL</address></body>
|
||||
</html>
|
||||
`,
|
||||
finalUrl: "https://tourismus.example/amsterdam/werries",
|
||||
});
|
||||
|
||||
expect(accepted.accepted).toBe(true);
|
||||
expect(accepted.score).toBeGreaterThanOrEqual(MATCH_THRESHOLD);
|
||||
expect(accepted.evidence.length).toBeGreaterThan(0);
|
||||
expect(genericHomepage.accepted).toBe(false);
|
||||
expect(wrongFacility.accepted).toBe(false);
|
||||
});
|
||||
|
||||
it("stores the verified website with auditable provenance and never overwrites existing contacts", () => {
|
||||
const candidate = searchCandidate({
|
||||
properties: {
|
||||
"addr:city": "Hamm",
|
||||
phone: "+49 2381 100",
|
||||
},
|
||||
enrichmentProperties: {
|
||||
operator: "Vorhandener Betreiber",
|
||||
},
|
||||
});
|
||||
const query = buildFacilitySearchQuery(candidate);
|
||||
const page = {
|
||||
html: matchingPageHtml,
|
||||
finalUrl: "https://www.wsa.example/anlagen/schleuse-werries",
|
||||
};
|
||||
const match = evaluateFacilityPageMatch(candidate, page);
|
||||
const record = buildSearchEnrichmentRecord({
|
||||
candidate,
|
||||
query,
|
||||
providerId: "duckduckgo",
|
||||
searchResult: matchingResult,
|
||||
page,
|
||||
match,
|
||||
extracted: {
|
||||
phone: "+49 2381 999",
|
||||
email: "schleuse-werries@example.test",
|
||||
operator: "Anderer Betreiber",
|
||||
address: null,
|
||||
},
|
||||
fetchedAt: FETCHED_AT,
|
||||
});
|
||||
|
||||
expect(record).toMatchObject({
|
||||
originalId: "42",
|
||||
sourceId: "osm:w123",
|
||||
properties: {
|
||||
website: "https://www.wsa.example/anlagen/schleuse-werries",
|
||||
phone: "+49 2381 100",
|
||||
email: "schleuse-werries@example.test",
|
||||
operator: "Vorhandener Betreiber",
|
||||
original_source: "osm",
|
||||
original_source_id: "w123",
|
||||
enrichmentSource: "facility-search",
|
||||
enrichmentProvider: "duckduckgo",
|
||||
searchQuery: query,
|
||||
searchResultUrl: matchingResult.url,
|
||||
source_url: page.finalUrl,
|
||||
fetched_at: FETCHED_AT,
|
||||
},
|
||||
});
|
||||
expect(record.properties.searchScore).toBeGreaterThanOrEqual(MATCH_THRESHOLD);
|
||||
expect(record.properties.enriched_fields).toEqual(
|
||||
expect.arrayContaining(["website", "email"]),
|
||||
);
|
||||
expect(record.properties.enriched_fields).not.toContain("phone");
|
||||
expect(record.properties.enriched_fields).not.toContain("operator");
|
||||
});
|
||||
|
||||
it("persists a verified discovered website even when the page exposes no contact fields", async () => {
|
||||
const search = vi.fn(async () => [matchingResult]);
|
||||
const fetchPageImpl = vi.fn(async () => ({
|
||||
html: identityOnlyPageHtml,
|
||||
finalUrl: matchingResult.url,
|
||||
redirects: 0,
|
||||
}));
|
||||
|
||||
const result = await enrichSearchCandidates([searchCandidate()], {
|
||||
searchProvider: { id: "duckduckgo", search },
|
||||
fetchPageImpl,
|
||||
concurrency: 1,
|
||||
hostDelayMs: 0,
|
||||
maxResults: 5,
|
||||
maxPages: 2,
|
||||
fetchedAt: FETCHED_AT,
|
||||
logger: { warn: vi.fn() },
|
||||
});
|
||||
|
||||
expect(search).toHaveBeenCalledTimes(1);
|
||||
expect(search.mock.calls[0][0]).toContain("Schleuse Werries");
|
||||
expect(fetchPageImpl).toHaveBeenCalledTimes(1);
|
||||
expect(result.records).toHaveLength(1);
|
||||
expect(result.records[0]).toMatchObject({
|
||||
originalId: "42",
|
||||
sourceId: "osm:w123",
|
||||
properties: {
|
||||
website: matchingResult.url,
|
||||
enrichmentProvider: "duckduckgo",
|
||||
fetched_at: FETCHED_AT,
|
||||
},
|
||||
});
|
||||
expect(result.records[0].properties.enriched_fields).toContain("website");
|
||||
});
|
||||
|
||||
it("opens the provider circuit after a block response and does not continue querying", async () => {
|
||||
const blockedError = Object.assign(new Error("DuckDuckGo hat weitere Anfragen blockiert."), {
|
||||
code: "SEARCH_PROVIDER_BLOCKED",
|
||||
status: 429,
|
||||
});
|
||||
const search = vi.fn(async () => {
|
||||
throw blockedError;
|
||||
});
|
||||
const fetchPageImpl = vi.fn();
|
||||
const logger = { warn: vi.fn() };
|
||||
|
||||
const result = await enrichSearchCandidates(
|
||||
[
|
||||
searchCandidate(),
|
||||
searchCandidate({
|
||||
id: "43",
|
||||
sourceId: "w124",
|
||||
name: "Schleuse Uentrop",
|
||||
properties: { "addr:city": "Hamm" },
|
||||
}),
|
||||
],
|
||||
{
|
||||
searchProvider: { id: "duckduckgo", search },
|
||||
fetchPageImpl,
|
||||
concurrency: 1,
|
||||
hostDelayMs: 0,
|
||||
fetchedAt: FETCHED_AT,
|
||||
logger,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.records).toEqual([]);
|
||||
expect(search).toHaveBeenCalledTimes(1);
|
||||
expect(fetchPageImpl).not.toHaveBeenCalled();
|
||||
expect(logger.warn).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps the database untouched in the default dry-run", async () => {
|
||||
const writer = vi.fn();
|
||||
const search = vi.fn(async () => [matchingResult]);
|
||||
const fetchPageImpl = vi.fn(async () => ({
|
||||
html: identityOnlyPageHtml,
|
||||
finalUrl: matchingResult.url,
|
||||
redirects: 0,
|
||||
}));
|
||||
|
||||
const result = await runSearchEnrichment({
|
||||
candidates: [searchCandidate()],
|
||||
writer,
|
||||
enrichmentOptions: {
|
||||
searchProvider: { id: "duckduckgo", search },
|
||||
fetchPageImpl,
|
||||
concurrency: 1,
|
||||
hostDelayMs: 0,
|
||||
fetchedAt: FETCHED_AT,
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.dryRun).toBe(true);
|
||||
expect(result.records).toHaveLength(1);
|
||||
expect(writer).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("passes validated records and checkpoint attempts to the writer only when enabled", async () => {
|
||||
const writer = vi.fn(async () => ({ enrichments: 1, attempts: 1 }));
|
||||
const result = await runSearchEnrichment({
|
||||
dryRun: false,
|
||||
candidates: [searchCandidate()],
|
||||
writer,
|
||||
enrichmentOptions: {
|
||||
searchProvider: { id: "duckduckgo", search: async () => [matchingResult] },
|
||||
fetchPageImpl: async () => ({
|
||||
html: identityOnlyPageHtml,
|
||||
finalUrl: matchingResult.url,
|
||||
redirects: 0,
|
||||
}),
|
||||
concurrency: 1,
|
||||
hostDelayMs: 0,
|
||||
fetchedAt: FETCHED_AT,
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.records).toHaveLength(1);
|
||||
expect(result.attempts).toHaveLength(1);
|
||||
expect(result.attempts[0]).toMatchObject({
|
||||
status: "success",
|
||||
provider: "duckduckgo",
|
||||
originalSource: "osm",
|
||||
originalSourceId: "w123",
|
||||
});
|
||||
expect(writer).toHaveBeenCalledTimes(1);
|
||||
expect(writer.mock.calls[0][0]).toMatchObject({
|
||||
records: [expect.objectContaining({ sourceId: "osm:w123" })],
|
||||
attempts: [expect.objectContaining({ status: "success" })],
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user