Rewrite old forum links in posts and redirect legacy domain

web/DEPLOY.md

@@ -51,14 +51,28 @@ requests; it holds no Cloudflare credentials.
    `npm run upload:attachments:remote`
 5. Make the first admin: `npm run admin -- grant <email> --remote`.
 6. Check: a sign-in email arrives, an upload works, a screenshot shows its
-   resized variant, an old Tender URL redirects.
+   resized variant, an old Tender path (`/discussions/…`) on the forum redirects.
 
 ## Read-only until the move
 
 `READ_ONLY` in `wrangler.jsonc` is `"true"` while Tender is still the forum:
 everyone can read and sign in, only admins can post, and every page says so.
 At the switch, do the final import first, then remove `READ_ONLY` and push.
 
+## The old domain
+
+Once Tender is switched off, point `support.nodebox.net` at the Worker so old
+links keep working: delete its DNS record (the CNAME to Tender) and add it as a
+second custom domain of the Worker. Every request on a host in `LEGACY_HOSTS`
+gets a 301 to the same page on `PUBLIC_URL`: old discussion URLs go to their
+topic (found by the discussion number, so moved and renamed discussions
+resolve too), category pages to their category, anything else to the same
+path, where it shows the forum's not-found page. Links to `*.tenderapp.com`
+can't be redirected; the import rewrites those inside posts instead.
+
+Check: `curl -sI https://support.nodebox.net/discussions/<category>/<number>-<slug>`
+answers 301 with the topic's URL.
+
 ## Deploying by hand
 
 When Workers Builds can't run, `npm run deploy` does the same as a build: it

web/README.md

@@ -59,6 +59,11 @@ Run it against an empty, freshly migrated database. If the spam filter
 view comes in with status `spam` (hidden, restorable from `/admin`). To start
 over, delete `.wrangler/state` and run `npm run db:migrate:local` first.
 
+Links in posts to the old forum (`support.nodebox.net`, `*.tenderapp.com`)
+are rewritten to the new topics, categories and files. A bare URL becomes an
+absolute link on `https://forum.nodebox.net`; set `FORUM_URL` to import for
+another host (`FORUM_URL=http://localhost:8787 npm run import:tender`).
+
 ## Tests
 
 `npm test` runs the importer against a small Tender fixture, and the app itself

web/scripts/import-tender.ts

@@ -11,13 +11,15 @@
 // What comes over: public categories; open/assigned/resolved public discussions;
 // comments that aren't internal notes or system messages; the users who wrote at least one visible post (deduplicated on
 // email); attachment metadata (files are uploaded separately); old URLs for
-// redirects.
+// redirects. Links in bodies to old attachments, discussions and category pages
+// point at their new place.
 // Items the spam filter flagged (the `spam_excluded` view, if present) are
 // imported with status 'spam': hidden, but admins can restore them.
 import { DatabaseSync } from "node:sqlite";
 import { createHash } from "node:crypto";
 import fs from "node:fs";
 import path from "node:path";
+import { isLegacyHost, tenderCategoryPermalink, tenderDiscussionNumber } from "../src/legacy.ts";
 
 export interface ImportStats {
   categories: number;
@@ -31,6 +33,12 @@ export interface ImportStats {
   skippedDiscussions: number;
 }
 
+/** Where bare links to old discussions point after the import. FORUM_URL overrides it. */
+const DEFAULT_FORUM_URL = "https://forum.nodebox.net";
+/** The old forum's custom domain; *.tenderapp.com and the hosts in the export's own URLs count too. */
+const LEGACY_HOSTS = ["support.nodebox.net"];
+/** Candidates for old links in a body: absolute URLs, and paths to old attachments and discussions. */
+const OLD_LINK = /https?:\/\/[^\s)"'<>\]]+|(?<![\w/.-])\/(?:assets|discussions)\/[^\s)"'<>\]]+/g;
 const IMAGE_TYPES = new Set(["image/png", "image/jpeg", "image/gif", "image/webp"]);
 /** Ids for placeholder users (authors without a Tender user record) start here, clear of Tender's ids. */
 const PLACEHOLDER_BASE = 2_000_000_000;
@@ -104,7 +112,7 @@ function htmlToText(html: string): string {
     .trim();
 }
 
-export function convert(src: DatabaseSync, imagesPrefix = "nodebox-forum-"): { statements: string[]; stats: ImportStats } {
+export function convert(src: DatabaseSync, imagesPrefix = "nodebox-forum-", forumUrl = DEFAULT_FORUM_URL): { statements: string[]; stats: ImportStats } {
   const out: string[] = [];
   const stats: ImportStats = { categories: 0, users: 0, topics: 0, posts: 0, spamTopics: 0, spamPosts: 0, attachments: 0, redirects: 0, skippedDiscussions: 0 };
 
@@ -125,12 +133,14 @@ export function convert(src: DatabaseSync, imagesPrefix = "nodebox-forum-"): { s
   const categories = src.prepare("SELECT id, name, permalink, summary, public FROM categories ORDER BY id").all() as any[];
   const catIds = new Set<number>();
   const usedSlugs = new Set<string>();
+  const categoryPathByPermalink = new Map<string, string>();
   categories.forEach((c, i) => {
     if (!isPublic(c.public)) return;
     let slug = slugify(c.permalink || c.name, 40);
     while (usedSlugs.has(slug)) slug += "-x";
     usedSlugs.add(slug);
     catIds.add(c.id);
+    categoryPathByPermalink.set(slugify(c.permalink || c.name, 40), `/c/${slug}`);
     out.push(insert("categories", { id: c.id, slug, name: c.name, description: c.summary ?? "", position: i, topic_count: 0, tender_id: c.id }));
     stats.categories++;
   });
@@ -175,8 +185,11 @@ export function convert(src: DatabaseSync, imagesPrefix = "nodebox-forum-"): { s
     "SELECT download_path, comment_number, filename, content_type, size, width, height, created_at, local_path FROM attachments WHERE discussion_id = ?",
   );
 
-  // Old URLs -> new topic paths, for rewriting links inside bodies.
+  // Old URLs -> new topic paths, for rewriting links inside bodies. Keyed by the
+  // discussion number too, since old links may carry an earlier category or slug.
   const topicPathByOldUrl = new Map<string, string>();
+  const topicPathByNumber = new Map<number, string>();
+  const legacyHosts = new Set(LEGACY_HOSTS);
   const kept: { d: any; comments: any[]; slug: string }[] = [];
   for (const d of discussions) {
     if (!isPublic(d.public) || !catIds.has(d.category_id)) {
@@ -190,7 +203,14 @@ export function convert(src: DatabaseSync, imagesPrefix = "nodebox-forum-"): { s
     }
     const slug = slugify(d.title ?? "topic");
     kept.push({ d, comments, slug });
-    if (d.html_href) topicPathByOldUrl.set(new URL(d.html_href).pathname.replace(/\/$/, ""), `/t/${d.id}/${slug}`);
+    if (d.html_href) {
+      const old = new URL(d.html_href);
+      const p = old.pathname.replace(/\/$/, "");
+      topicPathByOldUrl.set(p, `/t/${d.id}/${slug}`);
+      const n = tenderDiscussionNumber(p);
+      if (n !== null) topicPathByNumber.set(n, `/t/${d.id}/${slug}`);
+      legacyHosts.add(old.hostname.toLowerCase());
+    }
   }
 
   const categoryCounts = new Map<number, number>();
@@ -212,15 +232,34 @@ export function convert(src: DatabaseSync, imagesPrefix = "nodebox-forum-"): { s
     for (const f of files) fileUrlByPath.set(f.download_path, `/files/${attachmentId(d.id, f.comment_number, f.download_path)}/${encodeURIComponent(f.filename ?? "file")}`);
 
     const rewrite = (text: string) =>
-      text.replace(/(https?:\/\/[^\s)"'<>]+|\/assets\/[^\s)"'<>]+|\/discussions\/[^\s)"'<>]+)/g, (url) => {
-        let p: string;
+      text.replace(OLD_LINK, (match: string, offset: number) => {
+        // A sentence's full stop or a Markdown emphasis mark after a bare URL is not part of it.
+        const url = match.replace(/[.,;:!?*_]+$/, "");
+        const tail = match.slice(url.length);
+        let u: URL;
         try {
-          p = new URL(url, "https://x").pathname;
+          u = new URL(url, "https://legacy.invalid");
         } catch {
-          return url;
+          return match;
+        }
+        const absolute = /^https?:/i.test(url);
+        const p = u.pathname.replace(/\/$/, "");
+        let found: string | undefined;
+        for (const [dp, fu] of fileUrlByPath) if (u.pathname === new URL(dp, "https://x").pathname) found = fu;
+        if (!found && (!absolute || isLegacyHost(u.hostname, legacyHosts))) {
+          const n = tenderDiscussionNumber(p);
+          found = topicPathByOldUrl.get(p) ?? (n !== null ? topicPathByNumber.get(n) : undefined);
+          // Category pages and the front page, only for links that name the old forum's host.
+          if (!found && absolute) {
+            const permalink = tenderCategoryPermalink(p);
+            if (permalink) found = categoryPathByPermalink.get(slugify(permalink, 40));
+            else if (p === "" || p === "/discussions") found = "/";
+          }
         }
-        for (const [dp, fu] of fileUrlByPath) if (p === new URL(dp, "https://x").pathname) return fu;
-        return topicPathByOldUrl.get(p.replace(/\/$/, "")) ?? url;
+        if (!found) return match;
+        // A bare URL was a link only because it was absolute, so it stays absolute; a Markdown link target can be a path.
+        const inTarget = text.slice(Math.max(0, offset - 2), offset) === "](";
+        return (absolute && !inTarget ? forumUrl + found : found) + tail;
       });
 
     const topicSpam = spamDiscussions.get(d.id) ?? (spamComments.has(`${d.id}:${comments[0].number}`) ? spamComments.get(`${d.id}:${comments[0].number}`)! : null);
@@ -338,7 +377,7 @@ if (import.meta.url === `file://${process.argv[1]}`) {
     process.exit(1);
   }
   const db = new DatabaseSync(input, { readOnly: true });
-  const { statements, stats } = convert(db);
+  const { statements, stats } = convert(db, undefined, (process.env.FORUM_URL ?? DEFAULT_FORUM_URL).replace(/\/$/, ""));
   fs.writeFileSync(output, statements.join("\n") + "\n");
   console.log(`Wrote ${statements.length} statements to ${output}`);
   console.table(stats);

web/src/app.ts

@@ -3,6 +3,7 @@ import { csrf } from "hono/csrf";
 import { HTTPException } from "hono/http-exception";
 import { getCookie } from "hono/cookie";
 import { secureHeaders } from "hono/secure-headers";
+import { parseHosts, resolveLegacyPath } from "./legacy.ts";
 import type { AppEnv } from "./env.ts";
 import {
   REACTIONS,
@@ -78,6 +79,15 @@ const themeFromCookie = (v: string | undefined) => (v === "light" || v === "dark
 
 export const app = new Hono<AppEnv>();
 
+// The old forum's own domain (support.nodebox.net), once it points at this Worker: every
+// request moves to the same page here, old discussion URLs to their topics.
+app.use(async (c, next) => {
+  const url = new URL(c.req.url);
+  if (!parseHosts(c.env.LEGACY_HOSTS).includes(url.hostname)) return next();
+  const target = (await resolveLegacyPath(c.env.DB, url.pathname)) ?? url.pathname + url.search;
+  // Concatenated, not resolved with new URL(): a path like //evil.example would leave the site.
+  return c.redirect(c.env.PUBLIC_URL.replace(/\/$/, "") + target, 301);
+});
 app.use(secureHeaders({ crossOriginEmbedderPolicy: false, crossOriginResourcePolicy: false }));
 // Rejects cross-site form posts by checking the Origin header; with SameSite=Lax
 // session cookies this is the CSRF protection.
@@ -845,14 +855,10 @@ app.route("/admin", admin);
 // --- Fallbacks ----------------------------------------------------------------
 
 app.notFound(async (c) => {
-  // Old Tender links: /discussions/<category>/<id>-<slug> and friends.
-  const path = new URL(c.req.url).pathname.replace(/\/$/, "");
-  const legacy = await c.env.DB.prepare(
-    "SELECT t.id, t.slug FROM legacy_redirects r JOIN topics t ON t.id = r.topic_id WHERE r.path = ?",
-  )
-    .bind(path)
-    .first<{ id: number; slug: string }>();
-  if (legacy) return c.redirect(topicUrl(legacy), 301);
+  // Old Tender links: /discussions/<category>/<number>-<slug> and category pages.
+  const path = new URL(c.req.url).pathname;
+  const legacy = path.startsWith("/discussions") ? await resolveLegacyPath(c.env.DB, path) : null;
+  if (legacy) return c.redirect(legacy, 301);
   return page(c, messagePage("Not found", "That page doesn't exist, or it moved.", { href: "/", label: "Go to the forum" }), {
     title: "Not found",
     status: 404,

web/src/env.ts

@@ -15,6 +15,8 @@ export interface Bindings {
   MAIL_FROM: string;
   /** Git commit of this deploy. Workers Builds passes it in (`npm run ci:deploy`); absent in development. */
   GIT_COMMIT?: string;
+  /** The old forum's domains (comma-separated). Requests on them redirect to the same page on PUBLIC_URL. */
+  LEGACY_HOSTS?: string;
   /** "true" while the forum waits for the move from Tender: members can read and sign in, only admins can write. */
   READ_ONLY?: string;
   GOOGLE_CLIENT_ID?: string;

web/src/legacy.ts

@@ -0,0 +1,58 @@
+// Old Tender URLs and where they live now. Used by the Worker (redirects) and by
+// the importer (links inside post bodies).
+//
+// Tender's discussion URLs are /discussions/<category>/<number>-<slug>. The category
+// and the slug changed when a discussion was moved or renamed; the number did not.
+import { slugify } from "./util.ts";
+
+/** Hosts the old forum answered on. LEGACY_HOSTS (comma-separated) adds custom domains such as support.nodebox.net. */
+export function isLegacyHost(host: string, extra: Iterable<string> = []): boolean {
+  const h = host.toLowerCase();
+  if (h === "tenderapp.com" || h.endsWith(".tenderapp.com")) return true;
+  for (const e of extra) if (e.trim().toLowerCase() === h) return true;
+  return false;
+}
+
+export function parseHosts(list: string | undefined): string[] {
+  return (list ?? "").split(",").map((h) => h.trim().toLowerCase()).filter(Boolean);
+}
+
+/** The discussion number in a Tender discussion path (/discussions/<category>/<number>-<slug>), or null. */
+export function tenderDiscussionNumber(path: string): number | null {
+  const m = /^\/discussions\/(?:[^/]+\/)?(\d+)(?:-[^/]*)?\/?$/.exec(path);
+  return m ? Number(m[1]) : null;
+}
+
+/** The category permalink in a Tender category path (/discussions/<category>), or null. */
+export function tenderCategoryPermalink(path: string): string | null {
+  const m = /^\/discussions\/([^/]+)\/?$/.exec(path);
+  return m && !/^\d+(-|$)/.test(m[1]!) ? decodeURIComponent(m[1]!).toLowerCase() : null;
+}
+
+/**
+ * Where an old Tender path lives on the forum, or null when it has no counterpart
+ * (knowledge-base articles, private or deleted discussions, user pages).
+ */
+export async function resolveLegacyPath(db: D1Database, rawPath: string): Promise<string | null> {
+  const path = rawPath.replace(/\/+$/, "") || "/";
+  if (path === "/" || path === "/discussions") return "/";
+  const number = tenderDiscussionNumber(path);
+  if (number !== null) {
+    // The exact old URL first; otherwise the same discussion number under another category or slug.
+    const topic = await db
+      .prepare(
+        `SELECT t.id, t.slug FROM legacy_redirects r JOIN topics t ON t.id = r.topic_id
+         WHERE r.path = ?1 OR r.path GLOB '/discussions/*/' || ?2 || '-*' OR r.path GLOB '/discussions/*/' || ?2
+         ORDER BY r.path = ?1 DESC LIMIT 1`,
+      )
+      .bind(path, String(number))
+      .first<{ id: number; slug: string }>();
+    return topic ? `/t/${topic.id}/${topic.slug}` : null;
+  }
+  const permalink = tenderCategoryPermalink(path);
+  if (permalink) {
+    const category = await db.prepare("SELECT slug FROM categories WHERE slug = ?").bind(slugify(permalink, 40)).first<{ slug: string }>();
+    return category ? `/c/${category.slug}` : null;
+  }
+  return null;
+}

web/test/app.test.ts

@@ -168,6 +168,17 @@ test("legacy URLs redirect", async () => {
   const res = await req("/discussions/problems/4711-old/");
   assert.equal(res.status, 301);
   assert.equal(res.headers.get("location"), "/t/4711/old");
+  // Moved or renamed since: the discussion number still finds it.
+  assert.equal((await req("/discussions/questions/4711-renamed")).headers.get("location"), "/t/4711/old");
+  assert.equal((await req("/discussions/problems/47110-other")).status, 404);
+  // The old domain, once it points here: everything moves to the same page on the forum.
+  const legacyEnv = { ...env, LEGACY_HOSTS: "support.nodebox.net" };
+  const old = (p: string) => app.request("https://support.nodebox.net" + p, {}, legacyEnv);
+  assert.equal((await old("/discussions/problems/4711-old")).headers.get("location"), "http://localhost/t/4711/old");
+  assert.equal((await old("/discussions/questions")).headers.get("location"), "http://localhost/c/questions");
+  assert.equal((await old("/search?q=svg")).headers.get("location"), "http://localhost/search?q=svg");
+  assert.equal((await old("/")).headers.get("location"), "http://localhost/");
+  assert.equal((await old("//evil.example/x")).headers.get("location"), "http://localhost//evil.example/x");
 });
 
 test("watching, grouped digests, hearts, your topics", async () => {

web/test/import.test.ts

@@ -129,6 +129,21 @@ test("converts a Tender export into forum rows", () => {
   assert.equal(stats.spamPosts, 2);
 });
 
+test("points links to the old forum at their new place", () => {
+  const src = fixture();
+  src.exec(`INSERT INTO comments (discussion_id, number, user_id, author_name, author_email, body, internal, created_at, raw) VALUES
+    (105, 3, NULL, 'Guest', 'guest@example.com', 'See http://support.nodebox.net/discussions/problems/100-crash-on-import. Also [this](https://nodebox.tenderapp.com/discussions/old-category/100-old-title#comment_123), https://support.nodebox.net/discussions/problems and https://github.com/discussions/problems/100-crash-on-import', 0, '2014-05-05T11:00:00Z', '{}');`);
+  const db = new DatabaseSync(":memory:");
+  db.exec(forumSchema);
+  db.exec(convert(src).statements.join("\n"));
+  const links = (db.prepare("SELECT body FROM posts WHERE topic_id = 105 AND tender_number = 3").get() as any).body;
+  assert.equal(
+    links,
+    "See https://forum.nodebox.net/t/100/crash-on-import. Also [this](/t/100/crash-on-import), https://forum.nodebox.net/c/problems and https://github.com/discussions/problems/100-crash-on-import",
+    "bare links stay absolute, moved and renamed discussions are found by number, other hosts are left alone",
+  );
+});
+
 test("works without spam verdicts", () => {
   const db = fixture();
   db.exec("DROP VIEW spam_excluded; DROP TABLE spam_verdicts; DROP TABLE spam_decisions;");

web/wrangler.jsonc

@@ -24,6 +24,9 @@
     // Absolute origin, used in sign-in emails and the Google redirect URI.
     // These are the production values; `npm run dev` overrides PUBLIC_URL and MAILER.
     "PUBLIC_URL": "https://forum.nodebox.net",
+    // The old Tender forum's domain. Once it is a custom domain of this Worker, every request on it
+    // redirects here: old discussion URLs to their topics, the rest to the same path (web/DEPLOY.md).
+    "LEGACY_HOSTS": "support.nodebox.net",
     // Cloudflare Images: image ids are global to the account, so they are prefixed. The variants are the account's shared ones.
     "CF_IMAGES_PREFIX": "nodebox-forum-",
     "CF_IMAGES_VARIANT_POST": "medium",