fix(middleware): public pages and lead capture were behind the login wall

.github/workflows/ci.yml

@@ -54,6 +54,14 @@ jobs:
       - name: Lockfile guard (PRD-209)
         run: pnpm check:lockfile
 
+      # isPublicRoute is an allowlist, and a page missing from it 307s every
+      # anonymous visitor and crawler to the Clerk login while looking fine in
+      # development. Four occurrences to date — robots/sitemap, the Automatos
+      # ingest route, the legal pages + lead endpoint, and the whole /documents
+      # guide hub. Clean on this tree after that fix, so it gates.
+      - name: Public route guard (allowlist vs route tree)
+        run: pnpm check:public-routes
+
       # PRD-216 AC-7: PRD-200 string-aware grep gates (CSS sanitize +
       # error-message-leak). These already pass on this tree, so they gate
       # the build (blocking).

nextjs_space/middleware.ts

@@ -18,6 +18,13 @@ const isPublicRoute = createRouteMatcher([
   "/sitemap.xml",
   "/auth/login(.*)",
   "/auth/signup(.*)",
+  // Account recovery, and the only routes here whose absence LOCKED USERS OUT
+  // rather than merely hiding a page: a visitor who has forgotten their
+  // password was redirected to the login they cannot complete. Every one of
+  // these is reached with no session by definition.
+  "/auth/forgot-password",
+  "/auth/reset-password(.*)",
+  "/auth/callback",
   "/store/(.*)", // Storefronts are public
   "/api/webhooks(.*)",
   "/api/uploadthing(.*)",
@@ -51,6 +58,33 @@ const isPublicRoute = createRouteMatcher([
   "/terms", // Terms of service
   "/privacy", // Privacy policy
   "/cookies", // Cookie policy
+  // These three shipped as public legal pages but were never added here, so a
+  // signed-out visitor or crawler got a 307 to /auth/login. Confirmed against
+  // production before the fix: /terms and /privacy returned 200 while /dpa,
+  // /aup and /regulatory returned 307. /regulatory is also linked from the
+  // storefront footer, so the dead end was reachable from tenant sites.
+  "/dpa", // Data processing agreement
+  "/aup", // Acceptable use policy
+  "/regulatory", // Regulatory information
+  "/faq", // Public FAQ
+  // Public compliance pages. /legal/subprocessors in particular is a GDPR
+  // transparency obligation and is linked from the DPA — a login wall in front
+  // of it is a compliance problem, not just a broken link.
+  "/legal/changelog",
+  "/legal/subprocessors",
+  // The BudStacks Guide (#246/#249/#251) — 18 illustrated guide pages carrying
+  // 16 embedded videos, built as top-of-funnel marketing and then reachable
+  // only by signed-in users. Both the index and every guide beneath it.
+  "/documents",
+  "/documents/(.*)",
+  // Public lead capture for the homepage CTA and the Operator 101 download.
+  // UNAUTHENTICATED BY DESIGN — a prospect has no account and no tenant, which
+  // is why it is not the storefront newsletter endpoint (see the route's own
+  // header). Consent, honeypot and IP rate-limiting are enforced inside it.
+  // Without this entry the endpoint answered every submission with a 307 to
+  // Clerk, so PRD Phase 1 lead capture recorded nothing from the moment #254
+  // deployed until this landed.
+  "/api/platform/leads",
   "/accept-invite(.*)", // PRD-301 team invitation acceptance (logged-out invitees)
   "/api/team/invitation(.*)", // PRD-301 public token-gated invitation preview
 ]);

nextjs_space/package.json

@@ -23,6 +23,7 @@
     "check:grep-gates:report": "node scripts/ci/check-no-tenant-context-escapes.mjs || true; node scripts/ci/check-no-pii-in-logs.mjs || true",
     "check:docs-lint": "node scripts/ci/check-docs-lint.mjs",
     "check:lockfile": "node scripts/ci/check-single-lockfile.mjs",
+    "check:public-routes": "node scripts/ci/check-public-routes-allowlisted.mjs",
     "test:e2e": "playwright test",
     "test:e2e:ui": "playwright test --ui",
     "test:e2e:headed": "playwright test --headed",

nextjs_space/scripts/ci/check-public-routes-allowlisted.mjs

@@ -0,0 +1,209 @@
+#!/usr/bin/env node
+/**
+ * Public-route guard: a public page must be in `isPublicRoute`, or it answers
+ * anonymous traffic with a redirect to the Clerk login.
+ *
+ * `isPublicRoute` in middleware.ts is an ALLOWLIST. A new marketing or legal
+ * page renders correctly in development (where you are signed in) and 307s for
+ * every real visitor and every crawler. Nothing in the build catches it.
+ *
+ * This has now happened four times:
+ *   1. /robots.txt and /sitemap.xml            (SEO US-006)
+ *   2. /api/integrations/automatos/posts       (fix f59ac74)
+ *   3. /dpa, /aup, /regulatory, /faq           (found by PRD review, 2026-08-15)
+ *   4. /documents + every guide beneath it     (found while fixing 3 — the whole
+ *      18-page guide hub, built as top-of-funnel marketing, had never been
+ *      reachable logged-out)
+ *
+ * …and #3 also took POST /api/platform/leads with it, so platform lead capture
+ * recorded nothing from the moment it shipped.
+ *
+ * WHY THIS IS A SOURCE CHECK AND NOT AN HTTP PROBE. CI never starts the app
+ * (typecheck, lint and build only), so there is no origin to curl. Comparing
+ * the route tree against the allowlist needs no server and fails at PR time
+ * rather than after deploy.
+ *
+ * Exits non-zero (CI failure) if a public-looking page is not allowlisted.
+ */
+import { readdirSync, readFileSync, existsSync } from "node:fs";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const here = dirname(fileURLToPath(import.meta.url));
+const appRoot = join(here, "..", "..");
+
+/**
+ * Top-level route segments that are NOT public, with the reason. Anything else
+ * carrying a page.tsx is treated as public and must be allowlisted.
+ *
+ * Deliberately an explicit list: a new segment defaults to "must be public",
+ * so forgetting to classify it fails the build instead of silently shipping
+ * behind a login wall. If you add a genuinely private area, name it here.
+ */
+const INTENTIONALLY_PRIVATE = new Map([
+  ["tenant-admin", "tenant admin console — auth required"],
+  ["super-admin", "platform admin console — auth required"],
+]);
+
+/** Extract the string literals passed to createRouteMatcher([...]). */
+function readPublicPatterns(middlewareSource) {
+  const start = middlewareSource.indexOf("const isPublicRoute = createRouteMatcher([");
+  if (start === -1) {
+    console.error("✗ Could not find isPublicRoute in middleware.ts.");
+    process.exit(1);
+  }
+  const end = middlewareSource.indexOf("]);", start);
+  const block = middlewareSource.slice(start, end);
+
+  return [...block.matchAll(/"([^"]+)"/g)].map((m) => m[1]);
+}
+
+/**
+ * Clerk's matcher accepts path-to-regexp style patterns. Only `(.*)` appears in
+ * this file, so translating that one form is enough — and being narrow here is
+ * deliberate: a pattern this does not understand should fail loudly rather than
+ * quietly match everything.
+ */
+function patternToRegExp(pattern) {
+  const WILDCARD = "(.*)";
+  // Split on the wildcard FIRST, escape each literal piece completely, then
+  // rejoin with a real `.*`. Escaping the whole string and un-escaping after
+  // is the obvious approach and is wrong: `*` has to be in the escape class to
+  // handle a literal asterisk, which leaves `(\.\*)` to detect — and getting
+  // that dance slightly wrong yields `(\.*)`, "zero or more literal dots",
+  // which still matches the bare path and so passes its own test.
+  const escaped = pattern
+    .split(WILDCARD)
+    .map((part) => part.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))
+    .join(".*");
+  return new RegExp(`^${escaped}$`);
+}
+
+const middlewarePath = join(appRoot, "middleware.ts");
+const patterns = readPublicPatterns(readFileSync(middlewarePath, "utf8"));
+const matchers = patterns.map(patternToRegExp);
+const isPublic = (path) => matchers.some((re) => re.test(path));
+
+/**
+ * Unauthenticated API handlers that must STAY unauthenticated. A route.ts gives
+ * no signal about whether it expects a session, so these are declared rather
+ * than discovered — the point is that dropping one from the allowlist fails CI.
+ * `/api/platform/leads` is here because that is precisely what happened: it
+ * shipped in #254 and answered every submission with a redirect to Clerk.
+ */
+const PUBLIC_API_HANDLERS = [
+  {
+    path: "/api/platform/leads",
+    file: "app/api/platform/leads/route.ts",
+    why: "homepage CTA + Operator 101 download — a prospect has no account and no tenant",
+  },
+];
+
+/**
+ * Every page in the App Router tree, as an addressable path.
+ *
+ * Recursive, not top-level only: `app/documents/[slug]/page.tsx` is 18 guide
+ * pages, and a top-level scan cannot see that dropping `/documents/(.*)` puts
+ * all of them back behind the login wall. Walking the whole tree is also what
+ * surfaced /auth/forgot-password, /auth/callback, /legal/changelog and
+ * /legal/subprocessors as already broken.
+ *
+ * Dynamic segments become a probe value so the matcher test means something.
+ * An OPTIONAL catch-all `[[...rest]]` also matches its parent, so it yields the
+ * parent path — that is the only reason /auth/login resolves at all, since
+ * there is no app/auth/login/page.tsx, just [[...rest]].
+ */
+function collectRoutes(dir, segments = []) {
+  const routes = [];
+
+  if (existsSync(join(dir, "page.tsx"))) {
+    routes.push(`/${segments.join("/")}`.replace(/\/+$/, "") || "/");
+  }
+
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    if (!entry.isDirectory()) continue;
+    const name = entry.name;
+    if (name.startsWith("_") || name === "node_modules") continue;
+
+    let next;
+    if (name.startsWith("(") && name.endsWith(")")) {
+      next = segments; // route group — organisational, not addressable
+    } else if (name.startsWith("[[") && name.endsWith("]]")) {
+      next = segments; // optional catch-all — also matches the parent
+    } else if (name.startsWith("[")) {
+      next = [...segments, "__probe__"];
+    } else {
+      next = [...segments, name];
+    }
+
+    routes.push(...collectRoutes(join(dir, name), next));
+  }
+
+  return routes;
+}
+
+const violations = [];
+
+const routes = [...new Set(collectRoutes(join(appRoot, "app")))].sort();
+
+for (const route of routes) {
+  if (route === "/") continue; // the root page, allowlisted as "/"
+  const topSegment = route.split("/")[1];
+  if (INTENTIONALLY_PRIVATE.has(topSegment)) continue;
+  if (!isPublic(route)) {
+    violations.push({
+      path: route,
+      hint: `a page.tsx renders ${route} but no isPublicRoute pattern matches it`,
+    });
+  }
+}
+
+for (const handler of PUBLIC_API_HANDLERS) {
+  // A removed route is not this guard's business — only a still-present
+  // unauthenticated handler that has lost its allowlist entry.
+  if (!existsSync(join(appRoot, handler.file))) continue;
+  if (!isPublic(handler.path)) {
+    violations.push({
+      path: handler.path,
+      hint: `${handler.file} is unauthenticated by design (${handler.why}) but is not in isPublicRoute`,
+    });
+  }
+}
+
+/**
+ * Every URL the platform sitemap advertises must also be fetchable. Listing a
+ * path for crawlers that answers them with a login redirect is worse than not
+ * listing it.
+ */
+const sitemapPath = join(appRoot, "app", "sitemap.ts");
+if (existsSync(sitemapPath)) {
+  const sitemapSource = readFileSync(sitemapPath, "utf8");
+  const block = sitemapSource.slice(
+    sitemapSource.indexOf("MARKETING_PATHS"),
+    sitemapSource.indexOf("];", sitemapSource.indexOf("MARKETING_PATHS")),
+  );
+  for (const [, path] of block.matchAll(/path:\s*"([^"]*)"/g)) {
+    const url = path === "" ? "/" : path;
+    if (!isPublic(url)) {
+      violations.push({
+        path: url,
+        hint: `app/sitemap.ts advertises ${url} to crawlers but it is not in isPublicRoute`,
+      });
+    }
+  }
+}
+
+if (violations.length > 0) {
+  console.error("✗ Public-route guard failed — these answer anonymous requests with a login redirect:");
+  for (const v of violations) {
+    console.error(`  - ${v.path}`);
+    console.error(`      ${v.hint}`);
+  }
+  console.error("\n  Add them to isPublicRoute in middleware.ts, or — if the page really is");
+  console.error("  private — name it in INTENTIONALLY_PRIVATE in this script with a reason.");
+  process.exit(1);
+}
+
+console.log(
+  `✓ Public-route guard: ${routes.length} pages, ${PUBLIC_API_HANDLERS.length} public API handler(s) and every sitemap path are allowlisted.`,
+);