import { describe, expect, it, vi } from "vitest"; import { DEFAULT_LIMIT, MAX_HTML_BYTES, assertPublicHttpUrl, buildEnrichmentRecord, createHostLimiter, createPinnedLookup, extractContactsFromHtml, fetchHtmlPage, parseBooleanDefault, runWebsiteEnrichment, } from "../../../scripts/enrich-marine-websites.mjs"; const publicDns = async () => [{ address: "93.184.216.34", family: 4 }]; function htmlResponse(html, init = {}) { return new Response(html, { status: 200, headers: { "content-type": "text/html; charset=utf-8" }, ...init, }); } function candidate(overrides = {}) { return { id: "42", layer: "harbours", source: "osm", sourceId: "w123", name: "Testhafen", properties: { website: "https://marina.example/contact" }, ...overrides, }; } describe("marine facility website enrichment", () => { it("defaults to dry-run and validates explicit boolean values", () => { expect(DEFAULT_LIMIT).toBe(25); expect(parseBooleanDefault(undefined)).toBe(true); expect(parseBooleanDefault("false")).toBe(false); expect(() => parseBooleanDefault("maybe")).toThrow(/true oder false/u); }); it.each([ "http://127.0.0.1/admin", "http://[::1]/admin", "http://localhost/admin", "http://service.local/admin", ])("blocks local URL %s before fetching", async (url) => { await expect(assertPublicHttpUrl(url, { lookupImpl: publicDns })).rejects.toMatchObject({ name: "WebsiteEnrichmentError", }); }); it("rejects a public hostname if any DNS result is private", async () => { const lookupImpl = vi.fn(async () => [ { address: "93.184.216.34", family: 4 }, { address: "10.0.0.8", family: 4 }, ]); await expect( assertPublicHttpUrl("https://marina.example", { lookupImpl }), ).rejects.toMatchObject({ code: "SSRF_BLOCKED_DNS" }); }); it("pins the socket lookup to the already validated DNS addresses", async () => { const lookup = createPinnedLookup([{ address: "93.184.216.34", family: 4 }]); const addresses = await new Promise((resolve, reject) => { lookup("a-second-dns-name.example", { all: true }, (error, result) => { if (error) reject(error); else resolve(result); }); }); expect(addresses).toEqual([{ address: "93.184.216.34", family: 4 }]); expect(() => createPinnedLookup([{ address: "127.0.0.1", family: 4 }])).toThrow( /gepinnt/u, ); }); it("checks a redirect target again and never requests a private redirect", async () => { const fetchImpl = vi.fn(async () => new Response(null, { status: 302, headers: { location: "http://169.254.169.254/latest/meta-data" }, }), ); await expect( fetchHtmlPage("https://marina.example", { fetchImpl, lookupImpl: publicDns, beforeRequest: async () => {}, }), ).rejects.toMatchObject({ code: "SSRF_BLOCKED_IP" }); expect(fetchImpl).toHaveBeenCalledTimes(1); expect(fetchImpl.mock.calls[0][1]).toMatchObject({ redirect: "manual" }); }); it("requires HTML and stops streamed responses above 512 KiB", async () => { await expect( fetchHtmlPage("https://marina.example/file.pdf", { lookupImpl: publicDns, beforeRequest: async () => {}, fetchImpl: async () => new Response("pdf", { status: 200, headers: { "content-type": "application/pdf" } }), }), ).rejects.toMatchObject({ code: "UNSUPPORTED_CONTENT_TYPE" }); await expect( fetchHtmlPage("https://marina.example/huge", { lookupImpl: publicDns, beforeRequest: async () => {}, fetchImpl: async () => htmlResponse("x".repeat(MAX_HTML_BYTES + 1)), }), ).rejects.toMatchObject({ code: "BODY_TOO_LARGE" }); }); it("aborts a hanging HTTP request at the configured timeout", async () => { const fetchImpl = async (_url, init) => new Promise((_resolve, reject) => { init.signal.addEventListener("abort", () => reject(init.signal.reason), { once: true }); }); await expect( fetchHtmlPage("https://marina.example/hangs", { fetchImpl, lookupImpl: publicDns, beforeRequest: async () => {}, timeoutMs: 5, }), ).rejects.toMatchObject({ code: "FETCH_FAILED" }); }); it("also bounds a hanging DNS lookup", async () => { const fetchImpl = vi.fn(); await expect( fetchHtmlPage("https://marina.example/hangs", { fetchImpl, lookupImpl: async () => new Promise(() => {}), beforeRequest: async () => {}, timeoutMs: 5, }), ).rejects.toMatchObject({ code: "DNS_TIMEOUT" }); expect(fetchImpl).not.toHaveBeenCalled(); }); it("prefers JSON-LD and otherwise accepts only tel/mailto links", () => { const html = ` Alternative Alternative

Telefon 01234 567890

`; expect(extractContactsFromHtml(html)).toEqual({ phone: "+49 201 11111", email: "hafen@example.test", operator: "Hafenbetrieb Musterstadt", address: "Ufer 1, 12345 Musterstadt, DE", }); expect(extractContactsFromHtml("

Telefon 01234 567890

")).toEqual({ phone: null, email: null, operator: null, address: null, }); expect( extractContactsFromHtml( 'AnrufenMail', ), ).toMatchObject({ phone: "+49 201 777", email: "lock@example.test" }); expect( extractContactsFromHtml('Kein Kontaktlink'), ).toMatchObject({ phone: null }); expect(extractContactsFromHtml('X')).toMatchObject({ phone: null, }); expect( extractContactsFromHtml( '', ), ).toMatchObject({ email: null, operator: null }); expect( extractContactsFromHtml( 'Telefon', ), ).toMatchObject({ phone: "+49-201-555", operator: "Schleusenbetrieb Nord" }); }); it("keeps existing contact values and only fills missing fields", () => { const record = buildEnrichmentRecord({ candidate: candidate({ properties: { website: "https://marina.example/contact", phone: "+49 201 100", }, enrichmentProperties: { operator: "Vorhandener Hafenbetreiber" }, }), website: { key: "website", original: "https://marina.example/contact", url: "https://marina.example/contact", }, page: { finalUrl: "https://marina.example/kontakt" }, extracted: { phone: "+49 201 999", email: "hafen@example.test", }, fetchedAt: "2026-07-20T11:00:00.000Z", }); expect(record).toMatchObject({ sourceId: "osm:w123", properties: { website: "https://marina.example/contact", phone: "+49 201 100", email: "hafen@example.test", operator: "Vorhandener Hafenbetreiber", source_url: "https://marina.example/kontakt", fetchedAt: "2026-07-20T11:00:00.000Z", enriched_fields: ["email"], }, }); }); it("spaces starts to the same host while allowing a deterministic injected clock", async () => { let currentTime = 1_000; const waits = []; const limiter = createHostLimiter({ delayMs: 500, now: () => currentTime, sleepImpl: async (milliseconds) => { waits.push(milliseconds); currentTime += milliseconds; }, }); await limiter(new URL("https://marina.example/one")); await limiter(new URL("https://marina.example/two")); await limiter(new URL("https://other.example/one")); expect(waits).toEqual([500]); }); it("does not invoke the database writer during the default dry-run", async () => { const writer = vi.fn(); const fetchImpl = vi.fn(async (_url, init) => { expect(init.headers["User-Agent"]).toContain("Watermaps"); return htmlResponse('Schleuse anrufen'); }); const result = await runWebsiteEnrichment({ candidates: [ candidate(), candidate({ id: "43", sourceId: "w124", properties: { website: "https://other.example" } }), ], limit: 1, writer, enrichmentOptions: { fetchImpl, lookupImpl: publicDns, hostDelayMs: 0, fetchedAt: "2026-07-20T11:00:00.000Z", }, }); expect(result.dryRun).toBe(true); expect(result.records).toHaveLength(1); expect(result.records[0]).toMatchObject({ originalId: "42", sourceId: "osm:w123", properties: { phone: "+49-201-12345" }, }); expect(writer).not.toHaveBeenCalled(); }); });