watermaps/apps/api/tests/marine-website-enrichment.test.mjs

import { describe, expect, it, vi } from "vitest";
import {
  DEFAULT_LIMIT,
  MAX_HTML_BYTES,
  assertPublicHttpUrl,
  buildEnrichmentRecord,
  createHostLimiter,
  createPinnedLookup,
  extractContactsFromHtml,
  fetchHtmlPage,
  parseBooleanDefault,
  runWebsiteEnrichment,
} from "../../../scripts/enrich-marine-websites.mjs";

const publicDns = async () => [{ address: "93.184.216.34", family: 4 }];

function htmlResponse(html, init = {}) {
  return new Response(html, {
    status: 200,
    headers: { "content-type": "text/html; charset=utf-8" },
    ...init,
  });
}

function candidate(overrides = {}) {
  return {
    id: "42",
    layer: "harbours",
    source: "osm",
    sourceId: "w123",
    name: "Testhafen",
    properties: { website: "https://marina.example/contact" },
    ...overrides,
  };
}

describe("marine facility website enrichment", () => {
  it("defaults to dry-run and validates explicit boolean values", () => {
    expect(DEFAULT_LIMIT).toBe(25);
    expect(parseBooleanDefault(undefined)).toBe(true);
    expect(parseBooleanDefault("false")).toBe(false);
    expect(() => parseBooleanDefault("maybe")).toThrow(/true oder false/u);
  });

  it.each([
    "http://127.0.0.1/admin",
    "http://[::1]/admin",
    "http://localhost/admin",
    "http://service.local/admin",
  ])("blocks local URL %s before fetching", async (url) => {
    await expect(assertPublicHttpUrl(url, { lookupImpl: publicDns })).rejects.toMatchObject({
      name: "WebsiteEnrichmentError",
    });
  });

  it("rejects a public hostname if any DNS result is private", async () => {
    const lookupImpl = vi.fn(async () => [
      { address: "93.184.216.34", family: 4 },
      { address: "10.0.0.8", family: 4 },
    ]);
    await expect(
      assertPublicHttpUrl("https://marina.example", { lookupImpl }),
    ).rejects.toMatchObject({ code: "SSRF_BLOCKED_DNS" });
  });

  it("pins the socket lookup to the already validated DNS addresses", async () => {
    const lookup = createPinnedLookup([{ address: "93.184.216.34", family: 4 }]);
    const addresses = await new Promise((resolve, reject) => {
      lookup("a-second-dns-name.example", { all: true }, (error, result) => {
        if (error) reject(error);
        else resolve(result);
      });
    });
    expect(addresses).toEqual([{ address: "93.184.216.34", family: 4 }]);
    expect(() => createPinnedLookup([{ address: "127.0.0.1", family: 4 }])).toThrow(
      /gepinnt/u,
    );
  });

  it("checks a redirect target again and never requests a private redirect", async () => {
    const fetchImpl = vi.fn(async () =>
      new Response(null, {
        status: 302,
        headers: { location: "http://169.254.169.254/latest/meta-data" },
      }),
    );

    await expect(
      fetchHtmlPage("https://marina.example", {
        fetchImpl,
        lookupImpl: publicDns,
        beforeRequest: async () => {},
      }),
    ).rejects.toMatchObject({ code: "SSRF_BLOCKED_IP" });
    expect(fetchImpl).toHaveBeenCalledTimes(1);
    expect(fetchImpl.mock.calls[0][1]).toMatchObject({ redirect: "manual" });
  });

  it("requires HTML and stops streamed responses above 512 KiB", async () => {
    await expect(
      fetchHtmlPage("https://marina.example/file.pdf", {
        lookupImpl: publicDns,
        beforeRequest: async () => {},
        fetchImpl: async () =>
          new Response("pdf", { status: 200, headers: { "content-type": "application/pdf" } }),
      }),
    ).rejects.toMatchObject({ code: "UNSUPPORTED_CONTENT_TYPE" });

    await expect(
      fetchHtmlPage("https://marina.example/huge", {
        lookupImpl: publicDns,
        beforeRequest: async () => {},
        fetchImpl: async () => htmlResponse("x".repeat(MAX_HTML_BYTES + 1)),
      }),
    ).rejects.toMatchObject({ code: "BODY_TOO_LARGE" });
  });

  it("aborts a hanging HTTP request at the configured timeout", async () => {
    const fetchImpl = async (_url, init) =>
      new Promise((_resolve, reject) => {
        init.signal.addEventListener("abort", () => reject(init.signal.reason), { once: true });
      });

    await expect(
      fetchHtmlPage("https://marina.example/hangs", {
        fetchImpl,
        lookupImpl: publicDns,
        beforeRequest: async () => {},
        timeoutMs: 5,
      }),
    ).rejects.toMatchObject({ code: "FETCH_FAILED" });
  });

  it("also bounds a hanging DNS lookup", async () => {
    const fetchImpl = vi.fn();
    await expect(
      fetchHtmlPage("https://marina.example/hangs", {
        fetchImpl,
        lookupImpl: async () => new Promise(() => {}),
        beforeRequest: async () => {},
        timeoutMs: 5,
      }),
    ).rejects.toMatchObject({ code: "DNS_TIMEOUT" });
    expect(fetchImpl).not.toHaveBeenCalled();
  });

  it("prefers JSON-LD and otherwise accepts only tel/mailto links", () => {
    const html = `
      <script type="application/ld+json">
        {
          "@context": "https://schema.org",
          "@type": "Marina",
          "name": "Hafenbetrieb Musterstadt",
          "telephone": "+49 201 11111",
          "email": "hafen@example.test",
          "address": {
            "@type": "PostalAddress",
            "streetAddress": "Ufer 1",
            "postalCode": "12345",
            "addressLocality": "Musterstadt",
            "addressCountry": "DE"
          }
        }
      </script>
      <a href="tel:+49-201-99999">Alternative</a>
      <a href="mailto:other@example.test">Alternative</a>
      <p>Telefon 01234 567890</p>
    `;

    expect(extractContactsFromHtml(html)).toEqual({
      phone: "+49 201 11111",
      email: "hafen@example.test",
      operator: "Hafenbetrieb Musterstadt",
      address: "Ufer 1, 12345 Musterstadt, DE",
    });
    expect(extractContactsFromHtml("<p>Telefon 01234 567890</p>")).toEqual({
      phone: null,
      email: null,
      operator: null,
      address: null,
    });
    expect(
      extractContactsFromHtml(
        '<a href="tel:0049%20201%20777">Anrufen</a><a href="mailto:lock%40example.test">Mail</a>',
      ),
    ).toMatchObject({ phone: "+49 201 777", email: "lock@example.test" });
    expect(
      extractContactsFromHtml('<a title="some href=\'tel:+49999\'">Kein Kontaktlink</a>'),
    ).toMatchObject({ phone: null });
    expect(extractContactsFromHtml('<a href="tel:+49201&#999999999999;">X</a>')).toMatchObject({
      phone: null,
    });
    expect(
      extractContactsFromHtml(
        '<script type="application/ld+json">{"@type":"Marina","name":"Hafen\\u0000","email":"bad@example.test\\u0000"}</script>',
      ),
    ).toMatchObject({ email: null, operator: null });
    expect(
      extractContactsFromHtml(
        '<script type="application/ld+json">{"@type":"Organization","name":"Schleusenbetrieb Nord"}</script><a href="tel:+49-201-555">Telefon</a>',
      ),
    ).toMatchObject({ phone: "+49-201-555", operator: "Schleusenbetrieb Nord" });
  });

  it("keeps existing contact values and only fills missing fields", () => {
    const record = buildEnrichmentRecord({
      candidate: candidate({
        properties: {
          website: "https://marina.example/contact",
          phone: "+49 201 100",
        },
        enrichmentProperties: { operator: "Vorhandener Hafenbetreiber" },
      }),
      website: {
        key: "website",
        original: "https://marina.example/contact",
        url: "https://marina.example/contact",
      },
      page: { finalUrl: "https://marina.example/kontakt" },
      extracted: {
        phone: "+49 201 999",
        email: "hafen@example.test",
      },
      fetchedAt: "2026-07-20T11:00:00.000Z",
    });

    expect(record).toMatchObject({
      sourceId: "osm:w123",
      properties: {
        website: "https://marina.example/contact",
        phone: "+49 201 100",
        email: "hafen@example.test",
        operator: "Vorhandener Hafenbetreiber",
        source_url: "https://marina.example/kontakt",
        fetchedAt: "2026-07-20T11:00:00.000Z",
        enriched_fields: ["email"],
      },
    });
  });

  it("spaces starts to the same host while allowing a deterministic injected clock", async () => {
    let currentTime = 1_000;
    const waits = [];
    const limiter = createHostLimiter({
      delayMs: 500,
      now: () => currentTime,
      sleepImpl: async (milliseconds) => {
        waits.push(milliseconds);
        currentTime += milliseconds;
      },
    });

    await limiter(new URL("https://marina.example/one"));
    await limiter(new URL("https://marina.example/two"));
    await limiter(new URL("https://other.example/one"));
    expect(waits).toEqual([500]);
  });

  it("does not invoke the database writer during the default dry-run", async () => {
    const writer = vi.fn();
    const fetchImpl = vi.fn(async (_url, init) => {
      expect(init.headers["User-Agent"]).toContain("Watermaps");
      return htmlResponse('<a href="tel:+49-201-12345">Schleuse anrufen</a>');
    });

    const result = await runWebsiteEnrichment({
      candidates: [
        candidate(),
        candidate({ id: "43", sourceId: "w124", properties: { website: "https://other.example" } }),
      ],
      limit: 1,
      writer,
      enrichmentOptions: {
        fetchImpl,
        lookupImpl: publicDns,
        hostDelayMs: 0,
        fetchedAt: "2026-07-20T11:00:00.000Z",
      },
    });

    expect(result.dryRun).toBe(true);
    expect(result.records).toHaveLength(1);
    expect(result.records[0]).toMatchObject({
      originalId: "42",
      sourceId: "osm:w123",
      properties: { phone: "+49-201-12345" },
    });
    expect(writer).not.toHaveBeenCalled();
  });
});