Initial Watermaps import

2026-07-24 11:29:24 +02:00
commit 57f7b4dedb
129 changed files with 43136 additions and 0 deletions
@@ -0,0 +1,290 @@
+import { describe, expect, it, vi } from "vitest";
+import {
+  DEFAULT_LIMIT,
+  MAX_HTML_BYTES,
+  assertPublicHttpUrl,
+  buildEnrichmentRecord,
+  createHostLimiter,
+  createPinnedLookup,
+  extractContactsFromHtml,
+  fetchHtmlPage,
+  parseBooleanDefault,
+  runWebsiteEnrichment,
+} from "../../../scripts/enrich-marine-websites.mjs";
+
+const publicDns = async () => [{ address: "93.184.216.34", family: 4 }];
+
+function htmlResponse(html, init = {}) {
+  return new Response(html, {
+    status: 200,
+    headers: { "content-type": "text/html; charset=utf-8" },
+    ...init,
+  });
+}
+
+function candidate(overrides = {}) {
+  return {
+    id: "42",
+    layer: "harbours",
+    source: "osm",
+    sourceId: "w123",
+    name: "Testhafen",
+    properties: { website: "https://marina.example/contact" },
+    ...overrides,
+  };
+}
+
+describe("marine facility website enrichment", () => {
+  it("defaults to dry-run and validates explicit boolean values", () => {
+    expect(DEFAULT_LIMIT).toBe(25);
+    expect(parseBooleanDefault(undefined)).toBe(true);
+    expect(parseBooleanDefault("false")).toBe(false);
+    expect(() => parseBooleanDefault("maybe")).toThrow(/true oder false/u);
+  });
+
+  it.each([
+    "http://127.0.0.1/admin",
+    "http://[::1]/admin",
+    "http://localhost/admin",
+    "http://service.local/admin",
+  ])("blocks local URL %s before fetching", async (url) => {
+    await expect(assertPublicHttpUrl(url, { lookupImpl: publicDns })).rejects.toMatchObject({
+      name: "WebsiteEnrichmentError",
+    });
+  });
+
+  it("rejects a public hostname if any DNS result is private", async () => {
+    const lookupImpl = vi.fn(async () => [
+      { address: "93.184.216.34", family: 4 },
+      { address: "10.0.0.8", family: 4 },
+    ]);
+    await expect(
+      assertPublicHttpUrl("https://marina.example", { lookupImpl }),
+    ).rejects.toMatchObject({ code: "SSRF_BLOCKED_DNS" });
+  });
+
+  it("pins the socket lookup to the already validated DNS addresses", async () => {
+    const lookup = createPinnedLookup([{ address: "93.184.216.34", family: 4 }]);
+    const addresses = await new Promise((resolve, reject) => {
+      lookup("a-second-dns-name.example", { all: true }, (error, result) => {
+        if (error) reject(error);
+        else resolve(result);
+      });
+    });
+    expect(addresses).toEqual([{ address: "93.184.216.34", family: 4 }]);
+    expect(() => createPinnedLookup([{ address: "127.0.0.1", family: 4 }])).toThrow(
+      /gepinnt/u,
+    );
+  });
+
+  it("checks a redirect target again and never requests a private redirect", async () => {
+    const fetchImpl = vi.fn(async () =>
+      new Response(null, {
+        status: 302,
+        headers: { location: "http://169.254.169.254/latest/meta-data" },
+      }),
+    );
+
+    await expect(
+      fetchHtmlPage("https://marina.example", {
+        fetchImpl,
+        lookupImpl: publicDns,
+        beforeRequest: async () => {},
+      }),
+    ).rejects.toMatchObject({ code: "SSRF_BLOCKED_IP" });
+    expect(fetchImpl).toHaveBeenCalledTimes(1);
+    expect(fetchImpl.mock.calls[0][1]).toMatchObject({ redirect: "manual" });
+  });
+
+  it("requires HTML and stops streamed responses above 512 KiB", async () => {
+    await expect(
+      fetchHtmlPage("https://marina.example/file.pdf", {
+        lookupImpl: publicDns,
+        beforeRequest: async () => {},
+        fetchImpl: async () =>
+          new Response("pdf", { status: 200, headers: { "content-type": "application/pdf" } }),
+      }),
+    ).rejects.toMatchObject({ code: "UNSUPPORTED_CONTENT_TYPE" });
+
+    await expect(
+      fetchHtmlPage("https://marina.example/huge", {
+        lookupImpl: publicDns,
+        beforeRequest: async () => {},
+        fetchImpl: async () => htmlResponse("x".repeat(MAX_HTML_BYTES + 1)),
+      }),
+    ).rejects.toMatchObject({ code: "BODY_TOO_LARGE" });
+  });
+
+  it("aborts a hanging HTTP request at the configured timeout", async () => {
+    const fetchImpl = async (_url, init) =>
+      new Promise((_resolve, reject) => {
+        init.signal.addEventListener("abort", () => reject(init.signal.reason), { once: true });
+      });
+
+    await expect(
+      fetchHtmlPage("https://marina.example/hangs", {
+        fetchImpl,
+        lookupImpl: publicDns,
+        beforeRequest: async () => {},
+        timeoutMs: 5,
+      }),
+    ).rejects.toMatchObject({ code: "FETCH_FAILED" });
+  });
+
+  it("also bounds a hanging DNS lookup", async () => {
+    const fetchImpl = vi.fn();
+    await expect(
+      fetchHtmlPage("https://marina.example/hangs", {
+        fetchImpl,
+        lookupImpl: async () => new Promise(() => {}),
+        beforeRequest: async () => {},
+        timeoutMs: 5,
+      }),
+    ).rejects.toMatchObject({ code: "DNS_TIMEOUT" });
+    expect(fetchImpl).not.toHaveBeenCalled();
+  });
+
+  it("prefers JSON-LD and otherwise accepts only tel/mailto links", () => {
+    const html = `
+      <script type="application/ld+json">
+        {
+          "@context": "https://schema.org",
+          "@type": "Marina",
+          "name": "Hafenbetrieb Musterstadt",
+          "telephone": "+49 201 11111",
+          "email": "hafen@example.test",
+          "address": {
+            "@type": "PostalAddress",
+            "streetAddress": "Ufer 1",
+            "postalCode": "12345",
+            "addressLocality": "Musterstadt",
+            "addressCountry": "DE"
+          }
+        }
+      </script>
+      <a href="tel:+49-201-99999">Alternative</a>
+      <a href="mailto:other@example.test">Alternative</a>
+      <p>Telefon 01234 567890</p>
+    `;
+
+    expect(extractContactsFromHtml(html)).toEqual({
+      phone: "+49 201 11111",
+      email: "hafen@example.test",
+      operator: "Hafenbetrieb Musterstadt",
+      address: "Ufer 1, 12345 Musterstadt, DE",
+    });
+    expect(extractContactsFromHtml("<p>Telefon 01234 567890</p>")).toEqual({
+      phone: null,
+      email: null,
+      operator: null,
+      address: null,
+    });
+    expect(
+      extractContactsFromHtml(
+        '<a href="tel:0049%20201%20777">Anrufen</a><a href="mailto:lock%40example.test">Mail</a>',
+      ),
+    ).toMatchObject({ phone: "+49 201 777", email: "lock@example.test" });
+    expect(
+      extractContactsFromHtml('<a title="some href=\'tel:+49999\'">Kein Kontaktlink</a>'),
+    ).toMatchObject({ phone: null });
+    expect(extractContactsFromHtml('<a href="tel:+49201&#999999999999;">X</a>')).toMatchObject({
+      phone: null,
+    });
+    expect(
+      extractContactsFromHtml(
+        '<script type="application/ld+json">{"@type":"Marina","name":"Hafen\\u0000","email":"bad@example.test\\u0000"}</script>',
+      ),
+    ).toMatchObject({ email: null, operator: null });
+    expect(
+      extractContactsFromHtml(
+        '<script type="application/ld+json">{"@type":"Organization","name":"Schleusenbetrieb Nord"}</script><a href="tel:+49-201-555">Telefon</a>',
+      ),
+    ).toMatchObject({ phone: "+49-201-555", operator: "Schleusenbetrieb Nord" });
+  });
+
+  it("keeps existing contact values and only fills missing fields", () => {
+    const record = buildEnrichmentRecord({
+      candidate: candidate({
+        properties: {
+          website: "https://marina.example/contact",
+          phone: "+49 201 100",
+        },
+        enrichmentProperties: { operator: "Vorhandener Hafenbetreiber" },
+      }),
+      website: {
+        key: "website",
+        original: "https://marina.example/contact",
+        url: "https://marina.example/contact",
+      },
+      page: { finalUrl: "https://marina.example/kontakt" },
+      extracted: {
+        phone: "+49 201 999",
+        email: "hafen@example.test",
+      },
+      fetchedAt: "2026-07-20T11:00:00.000Z",
+    });
+
+    expect(record).toMatchObject({
+      sourceId: "osm:w123",
+      properties: {
+        website: "https://marina.example/contact",
+        phone: "+49 201 100",
+        email: "hafen@example.test",
+        operator: "Vorhandener Hafenbetreiber",
+        source_url: "https://marina.example/kontakt",
+        fetchedAt: "2026-07-20T11:00:00.000Z",
+        enriched_fields: ["email"],
+      },
+    });
+  });
+
+  it("spaces starts to the same host while allowing a deterministic injected clock", async () => {
+    let currentTime = 1_000;
+    const waits = [];
+    const limiter = createHostLimiter({
+      delayMs: 500,
+      now: () => currentTime,
+      sleepImpl: async (milliseconds) => {
+        waits.push(milliseconds);
+        currentTime += milliseconds;
+      },
+    });
+
+    await limiter(new URL("https://marina.example/one"));
+    await limiter(new URL("https://marina.example/two"));
+    await limiter(new URL("https://other.example/one"));
+    expect(waits).toEqual([500]);
+  });
+
+  it("does not invoke the database writer during the default dry-run", async () => {
+    const writer = vi.fn();
+    const fetchImpl = vi.fn(async (_url, init) => {
+      expect(init.headers["User-Agent"]).toContain("Watermaps");
+      return htmlResponse('<a href="tel:+49-201-12345">Schleuse anrufen</a>');
+    });
+
+    const result = await runWebsiteEnrichment({
+      candidates: [
+        candidate(),
+        candidate({ id: "43", sourceId: "w124", properties: { website: "https://other.example" } }),
+      ],
+      limit: 1,
+      writer,
+      enrichmentOptions: {
+        fetchImpl,
+        lookupImpl: publicDns,
+        hostDelayMs: 0,
+        fetchedAt: "2026-07-20T11:00:00.000Z",
+      },
+    });
+
+    expect(result.dryRun).toBe(true);
+    expect(result.records).toHaveLength(1);
+    expect(result.records[0]).toMatchObject({
+      originalId: "42",
+      sourceId: "osm:w123",
+      properties: { phone: "+49-201-12345" },
+    });
+    expect(writer).not.toHaveBeenCalled();
+  });
+});