feat(eval): add reproducible discovery evaluation reports

docs/docs/developer/development-setup.md

@@ -221,6 +221,8 @@ Tooling notes:
   `web` (the Next.js app and React-bound packages, in jsdom). All test commands
   run from the repo root:
   - `pnpm test` — run the whole suite once
+  - `pnpm discovery-eval --out /tmp/openmapx-eval` — run the versioned offline
+    [discovery evaluation](discovery-evaluation.md) and save a comparison report
   - `pnpm test:watch` — watch mode
   - `pnpm test:coverage` — run with a V8 coverage report (written to `coverage/`)
   - `pnpm test --project web` / `--project node` — scope to one environment

docs/docs/developer/map-comparison-baseline.md

@@ -199,3 +199,8 @@ rank boundaries, exclusions, duplicate prevention, layer priority, and validity.
 Repeat generation to check idempotence and repeat the visual matrix before
 expanding the thresholds. Hosted complete MapTiler styles are outside this
 owned-style policy.
+
+For versioned search, identity, enrichment and navigation evidence alongside these
+visual cases, follow [Discovery evaluation](discovery-evaluation.md). Store new
+capture archives externally with a manifest and checksums; attach selected review
+images directly to the PR.

package.json

@@ -8,7 +8,7 @@
     "lint": "biome check . && prettier --check \"**/*.{md,yml,yaml}\" && pnpm check-translations",
     "format": "biome format --write . && prettier --write \"**/*.{md,yml,yaml}\"",
     "clean": "turbo clean && rm -rf node_modules",
-    "check-types": "turbo run check-types",
+    "check-types": "turbo run check-types && pnpm discovery-eval:check-types",
     "check-dead-code": "knip --no-progress --reporter compact && pnpm check-unused-exports",
     "check-unused-exports": "knip --no-progress --reporter compact --workspace apps/ops-agent --workspace apps/transitous-runner --include exports,types --exclude files,dependencies",
     "check-duplicates": "jscpd --config .jscpd.json --cross-formats js-ts --no-tips apps packages integrations services scripts",
@@ -43,6 +43,8 @@
     "check-license-metadata": "pnpm -C packages/cli exec tsx ../../scripts/check-license-metadata.ts",
     "check-air-quality-release-gates": "node --disable-warning=MODULE_TYPELESS_PACKAGE_JSON --experimental-strip-types scripts/check-air-quality-release-gates.ts",
     "bench-air-quality": "pnpm -C packages/cli exec tsx ../../scripts/bench-air-quality.ts",
+    "discovery-eval:check-types": "tsc -p scripts/discovery-eval/tsconfig.json",
+    "discovery-eval": "pnpm -C packages/cli exec tsx ../../scripts/discovery-eval/run.ts",
     "search-eval:record": "pnpm -C packages/cli exec tsx ../../scripts/record-search-eval.mts",
     "check-toolchain-pins": "pnpm -C packages/cli exec tsx ../../scripts/check-toolchain-pins.ts",
     "check:policy": "pnpm check-legal-tables && pnpm check-legal-updated && pnpm check-data-flows && pnpm check-subject-data && pnpm check-license-metadata && pnpm check-air-quality-release-gates && pnpm check-toolchain-pins && pnpm check-feed-ids && pnpm check-credential-keys && pnpm check-dockerfile-sync && pnpm check-image-size-dos && pnpm check-docker-context-secrets && pnpm check-ops-authority",

scripts/discovery-eval/capture.test.ts

@@ -0,0 +1,48 @@
+import { describe, expect, it } from "vitest";
+import { captureMetadata } from "./capture.js";
+
+const revision = "a".repeat(40);
+const now = new Date("2026-10-07T00:00:00Z");
+
+describe("search capture provenance", () => {
+  it("identifies selected adapted API fields and leaves unobserved revisions unknown", () => {
+    expect(captureMetadata("https://openmapx.com/", revision, now)).toEqual({
+      protocolVersion: 1,
+      layer: "adapted-api",
+      representation: "selected-fields",
+      recordedAt: "2026-10-07T00:00:00.000Z",
+      captureCodeRevision: revision,
+      captureWorkingTreeDirty: false,
+      apiOrigin: "https://openmapx.com",
+      deploymentRevision: null,
+      sourceRevisions: null,
+      upstreamPayloadCaptured: false,
+    });
+  });
+
+  it.each([
+    "https://user:secret@example.com",
+    "https://example.com?key=secret",
+    "https://example.com#secret",
+    "file:///tmp/secret",
+    "not-a-url",
+  ])("rejects unsafe capture base %s without including its input in the error", (api) => {
+    expect(() => captureMetadata(api, revision, now)).toThrow("Capture API base");
+    try {
+      captureMetadata(api, revision, now);
+    } catch (error) {
+      expect(String(error)).not.toContain("secret");
+      expect(String(error)).not.toContain(api);
+    }
+  });
+
+  it("supports local capture without claiming that revision was deployed", () => {
+    const result = captureMetadata("http://localhost:3001/", null, now);
+    expect(result.captureCodeRevision).toBeNull();
+    expect(
+      captureMetadata("http://localhost:3001", revision, now, true).captureWorkingTreeDirty,
+    ).toBe(true);
+    expect(result.deploymentRevision).toBeNull();
+    expect(result.apiOrigin).toBe("http://localhost:3001");
+  });
+});

scripts/discovery-eval/capture.ts

@@ -0,0 +1,40 @@
+/** Provenance for filtered API recordings; never represents raw upstream data. */
+export function captureMetadata(
+  api: string,
+  revision: string | null,
+  now = new Date(),
+  workingTreeDirty = false,
+) {
+  let base: URL;
+  try {
+    base = new URL(api);
+  } catch {
+    throw new Error(
+      "Capture API base must be a public HTTP(S) URL without credentials or parameters",
+    );
+  }
+  if (
+    !["http:", "https:"].includes(base.protocol) ||
+    base.username ||
+    base.password ||
+    base.search ||
+    base.hash
+  ) {
+    // Do not include the input: rejected URLs can contain secrets.
+    throw new Error(
+      "Capture API base must be a public HTTP(S) URL without credentials or parameters",
+    );
+  }
+  return {
+    protocolVersion: 1 as const,
+    layer: "adapted-api" as const,
+    representation: "selected-fields" as const,
+    recordedAt: now.toISOString(),
+    captureCodeRevision: revision,
+    captureWorkingTreeDirty: workingTreeDirty,
+    apiOrigin: base.origin,
+    deploymentRevision: null,
+    sourceRevisions: null,
+    upstreamPayloadCaptured: false,
+  };
+}

scripts/discovery-eval/catalog.test.ts

@@ -0,0 +1,35 @@
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { describe, expect, it } from "vitest";
+import { CATALOG, INPUT_FILES } from "./catalog.js";
+
+describe("discovery input inventory", () => {
+  it("runs selected-place partial states as well as list-card enrichment", () => {
+    expect(
+      CATALOG.some(
+        (entry) =>
+          entry.suite === "apps/web/src/components/panels/place/PlaceDetailContent.test.tsx",
+      ),
+    ).toBe(true);
+  });
+  it("fingerprints the recorded upstream station results used by cold/warm ranking", () => {
+    const entry = CATALOG.find((entry) => entry.id === "search/station-synonyms-cache-order");
+    expect(entry?.fixtures).toContain(
+      "integrations/geocoding-maptiler/__fixtures__/station-search.json",
+    );
+    expect(INPUT_FILES).toContain(
+      "integrations/geocoding-maptiler/__fixtures__/station-search.json",
+    );
+  });
+
+  it("includes external JSON imports from every selected suite", () => {
+    for (const entry of CATALOG.filter((entry) => entry.suite)) {
+      const suite = entry.suite!;
+      const source = readFileSync(suite, "utf8");
+      for (const match of source.matchAll(/from\s+["'](\.[^"']+\.json)["']/g)) {
+        const path = resolve(suite, "..", match[1]).slice(`${process.cwd()}/`.length);
+        expect(INPUT_FILES, `${suite} reads ${path}`).toContain(path);
+      }
+    }
+  });
+});

scripts/discovery-eval/catalog.ts

@@ -0,0 +1,128 @@
+import { EVAL_CASES } from "../../packages/core/src/utils/__tests__/search-eval/cases.js";
+import { OVERTURE_QUALITY_BASELINE_RELEASE } from "../../services/data-manager/src/jobs/overture/eval/quality-baseline.js";
+import type { EvalCase } from "./report.js";
+
+const search = "packages/core/src/utils/__tests__/search-eval";
+const navigation = "packages/core/src/navigation";
+const overture = "services/data-manager/__tests__/overture";
+const cards = "apps/web/src/components/panels/category/CategoryResultsContent.test.tsx";
+
+/** Existing assertions stay authoritative; this is a catalog, not a second ranker. */
+export const CATALOG: EvalCase[] = [
+  ...EVAL_CASES.map((entry) => ({
+    id: `search/${entry.id}`,
+    layer: "client-ranking/recorded-adapted-api",
+    suite: `${search}/search-eval.test.ts`,
+    assertions: [`${entry.id}:`],
+    fixtures: [`${search}/fixtures/${entry.id}.json`, `${search}/fixtures/_shared.json`],
+    expected: entry,
+    ...(entry.knownGap ? { knownGap: entry.knownGap } : {}),
+  })),
+  {
+    id: "search/aggregate-ranking-budget",
+    layer: "client-ranking/recorded-adapted-api",
+    suite: `${search}/search-eval.test.ts`,
+    assertions: ["keeps the first expectation at rank 1 for most cases"],
+    expected: { hitAt1: 0.8, reciprocalRank: 0.85 },
+  },
+  {
+    id: "search/station-synonyms-cache-order",
+    layer: "adapted-api/mocked-upstream",
+    suite: "integrations/geocoding/__tests__/forward-ranking.test.ts",
+    fixtures: ["integrations/geocoding-maptiler/__fixtures__/station-search.json"],
+    note: "Cold/warm query order, station aliases, language/proximity isolation and provider-order fallback; mocked upstream, not provider coverage.",
+  },
+  {
+    id: "search/location-cache-isolation",
+    layer: "adapted-api/mocked-upstream",
+    suite: "integrations/geocoding/__tests__/routes.test.ts",
+  },
+  {
+    id: "identity/co-located-tenants",
+    layer: "conflation/synthetic",
+    suite: "packages/core/src/utils/__tests__/poiConflation.test.ts",
+    note: "Includes plural-per-address restaurants, contradictory address/phone and shared switchboard guards; does not establish real mall floor identity.",
+  },
+  {
+    id: "place/partial-enrichment-loading",
+    layer: "ui/contract-fixtures",
+    suite: cards,
+    note: "Partial photos/ratings, independent credits, bounded retries, stale searches and missing data; no real-provider availability or production latency claim.",
+  },
+  {
+    id: "place/selected-sheet-partial-states",
+    layer: "ui/contract-fixtures",
+    suite: "apps/web/src/components/panels/place/PlaceDetailContent.test.tsx",
+    note: "Absent/uncertain hours, photo detents, retained selection across enrichment and independent actions; no availability claim.",
+  },
+  {
+    id: "coverage/overture-reviewed-gate-contract",
+    layer: "dataset-gate/unit-fixtures",
+    suite: `${overture}/eval/quality-gate.test.ts`,
+    expected: { reviewedRelease: OVERTURE_QUALITY_BASELINE_RELEASE, resultWindow: 50 },
+    note: "Berlin/Aachen/Monschau/Maastricht anchor gate contracts; this command does not query a deployed dataset or measure current regional recall.",
+  },
+  {
+    id: "coverage/overture-metric-contract",
+    layer: "dataset-gate/unit-fixtures",
+    suite: `${overture}/eval/metrics.test.ts`,
+  },
+  {
+    id: "coverage/overture-search-quality-contract",
+    layer: "dataset-gate/unit-fixtures",
+    suite: `${overture}/eval/search-quality.test.ts`,
+  },
+  {
+    id: "navigation/transit-recovery-gps-gaps",
+    layer: "navigation-engine/synthetic-replay",
+    suite: `${navigation}/mobileReplay.test.ts`,
+    fixtures: ["ground-basic", "transit-basic", "transit-tunnel", "transit-transfer"].map(
+      (name) => `${navigation}/__fixtures__/mobile/${name}.json`,
+    ),
+    note: "Deterministic transfers, serialization/recovery and GPS gap confidence. Not installed shell/device verification.",
+  },
+  {
+    id: "navigation/ground-off-route-arrival",
+    layer: "navigation-engine/synthetic",
+    suite: `${navigation}/__tests__/processFix.test.ts`,
+  },
+  {
+    id: "navigation/alternative-route",
+    layer: "navigation-engine/synthetic",
+    suite: `${navigation}/fasterRoute.test.ts`,
+  },
+  {
+    id: "unavailable/installed-navigation",
+    layer: "installed-device",
+    unavailable:
+      "Installed navigation composition and device evidence pending #398; shared-engine replays are separate.",
+  },
+  {
+    id: "unavailable/offline-place-search",
+    layer: "installed-device/offline",
+    unavailable:
+      "Offline place/address search pending #403; downloaded map tiles are not offline search.",
+  },
+  {
+    id: "unavailable/offline-rerouting",
+    layer: "installed-device/offline",
+    unavailable:
+      "Android offline routing/rerouting prototype pending #404; no airplane-mode engine measurement in this corpus.",
+  },
+];
+
+export const INPUT_FILES = [
+  ...new Set([
+    ...CATALOG.flatMap((entry) => [
+      ...(entry.fixtures ?? []),
+      ...(entry.suite ? [entry.suite] : []),
+    ]),
+    `${search}/cases.ts`,
+    "scripts/discovery-eval/reviewed.ts",
+    "scripts/discovery-eval/examples/control-before.json",
+    "scripts/discovery-eval/examples/control-after.json",
+    "services/data-manager/src/jobs/overture/eval/quality-baseline.ts",
+    "apps/web/public/styles/openmapx-streets.json",
+    "apps/web/public/styles/openmapx-dark.json",
+  ]),
+].sort();