diff --git a/web/DEPLOY.md b/web/DEPLOY.md index 45c03e2..3fd74ae 100644 --- a/web/DEPLOY.md +++ b/web/DEPLOY.md @@ -51,7 +51,7 @@ requests; it holds no Cloudflare credentials. `npm run upload:attachments:remote` 5. Make the first admin: `npm run admin -- grant --remote`. 6. Check: a sign-in email arrives, an upload works, a screenshot shows its - resized variant, an old Tender URL redirects. + resized variant, an old Tender path (`/discussions/…`) on the forum redirects. ## Read-only until the move @@ -59,6 +59,20 @@ requests; it holds no Cloudflare credentials. everyone can read and sign in, only admins can post, and every page says so. At the switch, do the final import first, then remove `READ_ONLY` and push. +## The old domain + +Once Tender is switched off, point `support.nodebox.net` at the Worker so old +links keep working: delete its DNS record (the CNAME to Tender) and add it as a +second custom domain of the Worker. Every request on a host in `LEGACY_HOSTS` +gets a 301 to the same page on `PUBLIC_URL`: old discussion URLs go to their +topic (found by the discussion number, so moved and renamed discussions +resolve too), category pages to their category, anything else to the same +path, where it shows the forum's not-found page. Links to `*.tenderapp.com` +can't be redirected; the import rewrites those inside posts instead. + +Check: `curl -sI https://support.nodebox.net/discussions//-` +answers 301 with the topic's URL. + ## Deploying by hand When Workers Builds can't run, `npm run deploy` does the same as a build: it diff --git a/web/README.md b/web/README.md index cefb61b..8715f32 100644 --- a/web/README.md +++ b/web/README.md @@ -59,6 +59,11 @@ Run it against an empty, freshly migrated database. If the spam filter view comes in with status `spam` (hidden, restorable from `/admin`). To start over, delete `.wrangler/state` and run `npm run db:migrate:local` first. +Links in posts to the old forum (`support.nodebox.net`, `*.tenderapp.com`) +are rewritten to the new topics, categories and files. A bare URL becomes an +absolute link on `https://forum.nodebox.net`; set `FORUM_URL` to import for +another host (`FORUM_URL=http://localhost:8787 npm run import:tender`). + ## Tests `npm test` runs the importer against a small Tender fixture, and the app itself diff --git a/web/scripts/import-tender.ts b/web/scripts/import-tender.ts index c0fc2e7..c7060bb 100644 --- a/web/scripts/import-tender.ts +++ b/web/scripts/import-tender.ts @@ -11,13 +11,15 @@ // What comes over: public categories; open/assigned/resolved public discussions; // comments that aren't internal notes or system messages; the users who wrote at least one visible post (deduplicated on // email); attachment metadata (files are uploaded separately); old URLs for -// redirects. +// redirects. Links in bodies to old attachments, discussions and category pages +// point at their new place. // Items the spam filter flagged (the `spam_excluded` view, if present) are // imported with status 'spam': hidden, but admins can restore them. import { DatabaseSync } from "node:sqlite"; import { createHash } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; +import { isLegacyHost, tenderCategoryPermalink, tenderDiscussionNumber } from "../src/legacy.ts"; export interface ImportStats { categories: number; @@ -31,6 +33,12 @@ export interface ImportStats { skippedDiscussions: number; } +/** Where bare links to old discussions point after the import. FORUM_URL overrides it. */ +const DEFAULT_FORUM_URL = "https://forum.nodebox.net"; +/** The old forum's custom domain; *.tenderapp.com and the hosts in the export's own URLs count too. */ +const LEGACY_HOSTS = ["support.nodebox.net"]; +/** Candidates for old links in a body: absolute URLs, and paths to old attachments and discussions. */ +const OLD_LINK = /https?:\/\/[^\s)"'<>\]]+|(?\]]+/g; const IMAGE_TYPES = new Set(["image/png", "image/jpeg", "image/gif", "image/webp"]); /** Ids for placeholder users (authors without a Tender user record) start here, clear of Tender's ids. */ const PLACEHOLDER_BASE = 2_000_000_000; @@ -104,7 +112,7 @@ function htmlToText(html: string): string { .trim(); } -export function convert(src: DatabaseSync, imagesPrefix = "nodebox-forum-"): { statements: string[]; stats: ImportStats } { +export function convert(src: DatabaseSync, imagesPrefix = "nodebox-forum-", forumUrl = DEFAULT_FORUM_URL): { statements: string[]; stats: ImportStats } { const out: string[] = []; const stats: ImportStats = { categories: 0, users: 0, topics: 0, posts: 0, spamTopics: 0, spamPosts: 0, attachments: 0, redirects: 0, skippedDiscussions: 0 }; @@ -125,12 +133,14 @@ export function convert(src: DatabaseSync, imagesPrefix = "nodebox-forum-"): { s const categories = src.prepare("SELECT id, name, permalink, summary, public FROM categories ORDER BY id").all() as any[]; const catIds = new Set(); const usedSlugs = new Set(); + const categoryPathByPermalink = new Map(); categories.forEach((c, i) => { if (!isPublic(c.public)) return; let slug = slugify(c.permalink || c.name, 40); while (usedSlugs.has(slug)) slug += "-x"; usedSlugs.add(slug); catIds.add(c.id); + categoryPathByPermalink.set(slugify(c.permalink || c.name, 40), `/c/${slug}`); out.push(insert("categories", { id: c.id, slug, name: c.name, description: c.summary ?? "", position: i, topic_count: 0, tender_id: c.id })); stats.categories++; }); @@ -175,8 +185,11 @@ export function convert(src: DatabaseSync, imagesPrefix = "nodebox-forum-"): { s "SELECT download_path, comment_number, filename, content_type, size, width, height, created_at, local_path FROM attachments WHERE discussion_id = ?", ); - // Old URLs -> new topic paths, for rewriting links inside bodies. + // Old URLs -> new topic paths, for rewriting links inside bodies. Keyed by the + // discussion number too, since old links may carry an earlier category or slug. const topicPathByOldUrl = new Map(); + const topicPathByNumber = new Map(); + const legacyHosts = new Set(LEGACY_HOSTS); const kept: { d: any; comments: any[]; slug: string }[] = []; for (const d of discussions) { if (!isPublic(d.public) || !catIds.has(d.category_id)) { @@ -190,7 +203,14 @@ export function convert(src: DatabaseSync, imagesPrefix = "nodebox-forum-"): { s } const slug = slugify(d.title ?? "topic"); kept.push({ d, comments, slug }); - if (d.html_href) topicPathByOldUrl.set(new URL(d.html_href).pathname.replace(/\/$/, ""), `/t/${d.id}/${slug}`); + if (d.html_href) { + const old = new URL(d.html_href); + const p = old.pathname.replace(/\/$/, ""); + topicPathByOldUrl.set(p, `/t/${d.id}/${slug}`); + const n = tenderDiscussionNumber(p); + if (n !== null) topicPathByNumber.set(n, `/t/${d.id}/${slug}`); + legacyHosts.add(old.hostname.toLowerCase()); + } } const categoryCounts = new Map(); @@ -212,15 +232,34 @@ export function convert(src: DatabaseSync, imagesPrefix = "nodebox-forum-"): { s for (const f of files) fileUrlByPath.set(f.download_path, `/files/${attachmentId(d.id, f.comment_number, f.download_path)}/${encodeURIComponent(f.filename ?? "file")}`); const rewrite = (text: string) => - text.replace(/(https?:\/\/[^\s)"'<>]+|\/assets\/[^\s)"'<>]+|\/discussions\/[^\s)"'<>]+)/g, (url) => { - let p: string; + text.replace(OLD_LINK, (match: string, offset: number) => { + // A sentence's full stop or a Markdown emphasis mark after a bare URL is not part of it. + const url = match.replace(/[.,;:!?*_]+$/, ""); + const tail = match.slice(url.length); + let u: URL; try { - p = new URL(url, "https://x").pathname; + u = new URL(url, "https://legacy.invalid"); } catch { - return url; + return match; + } + const absolute = /^https?:/i.test(url); + const p = u.pathname.replace(/\/$/, ""); + let found: string | undefined; + for (const [dp, fu] of fileUrlByPath) if (u.pathname === new URL(dp, "https://x").pathname) found = fu; + if (!found && (!absolute || isLegacyHost(u.hostname, legacyHosts))) { + const n = tenderDiscussionNumber(p); + found = topicPathByOldUrl.get(p) ?? (n !== null ? topicPathByNumber.get(n) : undefined); + // Category pages and the front page, only for links that name the old forum's host. + if (!found && absolute) { + const permalink = tenderCategoryPermalink(p); + if (permalink) found = categoryPathByPermalink.get(slugify(permalink, 40)); + else if (p === "" || p === "/discussions") found = "/"; + } } - for (const [dp, fu] of fileUrlByPath) if (p === new URL(dp, "https://x").pathname) return fu; - return topicPathByOldUrl.get(p.replace(/\/$/, "")) ?? url; + if (!found) return match; + // A bare URL was a link only because it was absolute, so it stays absolute; a Markdown link target can be a path. + const inTarget = text.slice(Math.max(0, offset - 2), offset) === "]("; + return (absolute && !inTarget ? forumUrl + found : found) + tail; }); const topicSpam = spamDiscussions.get(d.id) ?? (spamComments.has(`${d.id}:${comments[0].number}`) ? spamComments.get(`${d.id}:${comments[0].number}`)! : null); @@ -338,7 +377,7 @@ if (import.meta.url === `file://${process.argv[1]}`) { process.exit(1); } const db = new DatabaseSync(input, { readOnly: true }); - const { statements, stats } = convert(db); + const { statements, stats } = convert(db, undefined, (process.env.FORUM_URL ?? DEFAULT_FORUM_URL).replace(/\/$/, "")); fs.writeFileSync(output, statements.join("\n") + "\n"); console.log(`Wrote ${statements.length} statements to ${output}`); console.table(stats); diff --git a/web/src/app.ts b/web/src/app.ts index 43900ce..5c19aad 100644 --- a/web/src/app.ts +++ b/web/src/app.ts @@ -3,6 +3,7 @@ import { csrf } from "hono/csrf"; import { HTTPException } from "hono/http-exception"; import { getCookie } from "hono/cookie"; import { secureHeaders } from "hono/secure-headers"; +import { parseHosts, resolveLegacyPath } from "./legacy.ts"; import type { AppEnv } from "./env.ts"; import { REACTIONS, @@ -78,6 +79,15 @@ const themeFromCookie = (v: string | undefined) => (v === "light" || v === "dark export const app = new Hono(); +// The old forum's own domain (support.nodebox.net), once it points at this Worker: every +// request moves to the same page here, old discussion URLs to their topics. +app.use(async (c, next) => { + const url = new URL(c.req.url); + if (!parseHosts(c.env.LEGACY_HOSTS).includes(url.hostname)) return next(); + const target = (await resolveLegacyPath(c.env.DB, url.pathname)) ?? url.pathname + url.search; + // Concatenated, not resolved with new URL(): a path like //evil.example would leave the site. + return c.redirect(c.env.PUBLIC_URL.replace(/\/$/, "") + target, 301); +}); app.use(secureHeaders({ crossOriginEmbedderPolicy: false, crossOriginResourcePolicy: false })); // Rejects cross-site form posts by checking the Origin header; with SameSite=Lax // session cookies this is the CSRF protection. @@ -845,14 +855,10 @@ app.route("/admin", admin); // --- Fallbacks ---------------------------------------------------------------- app.notFound(async (c) => { - // Old Tender links: /discussions//- and friends. - const path = new URL(c.req.url).pathname.replace(/\/$/, ""); - const legacy = await c.env.DB.prepare( - "SELECT t.id, t.slug FROM legacy_redirects r JOIN topics t ON t.id = r.topic_id WHERE r.path = ?", - ) - .bind(path) - .first<{ id: number; slug: string }>(); - if (legacy) return c.redirect(topicUrl(legacy), 301); + // Old Tender links: /discussions//- and category pages. + const path = new URL(c.req.url).pathname; + const legacy = path.startsWith("/discussions") ? await resolveLegacyPath(c.env.DB, path) : null; + if (legacy) return c.redirect(legacy, 301); return page(c, messagePage("Not found", "That page doesn't exist, or it moved.", { href: "/", label: "Go to the forum" }), { title: "Not found", status: 404, diff --git a/web/src/env.ts b/web/src/env.ts index dd973b7..0423393 100644 --- a/web/src/env.ts +++ b/web/src/env.ts @@ -15,6 +15,8 @@ export interface Bindings { MAIL_FROM: string; /** Git commit of this deploy. Workers Builds passes it in (`npm run ci:deploy`); absent in development. */ GIT_COMMIT?: string; + /** The old forum's domains (comma-separated). Requests on them redirect to the same page on PUBLIC_URL. */ + LEGACY_HOSTS?: string; /** "true" while the forum waits for the move from Tender: members can read and sign in, only admins can write. */ READ_ONLY?: string; GOOGLE_CLIENT_ID?: string; diff --git a/web/src/legacy.ts b/web/src/legacy.ts new file mode 100644 index 0000000..c98f4d5 --- /dev/null +++ b/web/src/legacy.ts @@ -0,0 +1,58 @@ +// Old Tender URLs and where they live now. Used by the Worker (redirects) and by +// the importer (links inside post bodies). +// +// Tender's discussion URLs are /discussions//-. The category +// and the slug changed when a discussion was moved or renamed; the number did not. +import { slugify } from "./util.ts"; + +/** Hosts the old forum answered on. LEGACY_HOSTS (comma-separated) adds custom domains such as support.nodebox.net. */ +export function isLegacyHost(host: string, extra: Iterable = []): boolean { + const h = host.toLowerCase(); + if (h === "tenderapp.com" || h.endsWith(".tenderapp.com")) return true; + for (const e of extra) if (e.trim().toLowerCase() === h) return true; + return false; +} + +export function parseHosts(list: string | undefined): string[] { + return (list ?? "").split(",").map((h) => h.trim().toLowerCase()).filter(Boolean); +} + +/** The discussion number in a Tender discussion path (/discussions//-), or null. */ +export function tenderDiscussionNumber(path: string): number | null { + const m = /^\/discussions\/(?:[^/]+\/)?(\d+)(?:-[^/]*)?\/?$/.exec(path); + return m ? Number(m[1]) : null; +} + +/** The category permalink in a Tender category path (/discussions/), or null. */ +export function tenderCategoryPermalink(path: string): string | null { + const m = /^\/discussions\/([^/]+)\/?$/.exec(path); + return m && !/^\d+(-|$)/.test(m[1]!) ? decodeURIComponent(m[1]!).toLowerCase() : null; +} + +/** + * Where an old Tender path lives on the forum, or null when it has no counterpart + * (knowledge-base articles, private or deleted discussions, user pages). + */ +export async function resolveLegacyPath(db: D1Database, rawPath: string): Promise { + const path = rawPath.replace(/\/+$/, "") || "/"; + if (path === "/" || path === "/discussions") return "/"; + const number = tenderDiscussionNumber(path); + if (number !== null) { + // The exact old URL first; otherwise the same discussion number under another category or slug. + const topic = await db + .prepare( + `SELECT t.id, t.slug FROM legacy_redirects r JOIN topics t ON t.id = r.topic_id + WHERE r.path = ?1 OR r.path GLOB '/discussions/*/' || ?2 || '-*' OR r.path GLOB '/discussions/*/' || ?2 + ORDER BY r.path = ?1 DESC LIMIT 1`, + ) + .bind(path, String(number)) + .first<{ id: number; slug: string }>(); + return topic ? `/t/${topic.id}/${topic.slug}` : null; + } + const permalink = tenderCategoryPermalink(path); + if (permalink) { + const category = await db.prepare("SELECT slug FROM categories WHERE slug = ?").bind(slugify(permalink, 40)).first<{ slug: string }>(); + return category ? `/c/${category.slug}` : null; + } + return null; +} diff --git a/web/test/app.test.ts b/web/test/app.test.ts index ff95847..72d36ca 100644 --- a/web/test/app.test.ts +++ b/web/test/app.test.ts @@ -168,6 +168,17 @@ test("legacy URLs redirect", async () => { const res = await req("/discussions/problems/4711-old/"); assert.equal(res.status, 301); assert.equal(res.headers.get("location"), "/t/4711/old"); + // Moved or renamed since: the discussion number still finds it. + assert.equal((await req("/discussions/questions/4711-renamed")).headers.get("location"), "/t/4711/old"); + assert.equal((await req("/discussions/problems/47110-other")).status, 404); + // The old domain, once it points here: everything moves to the same page on the forum. + const legacyEnv = { ...env, LEGACY_HOSTS: "support.nodebox.net" }; + const old = (p: string) => app.request("https://support.nodebox.net" + p, {}, legacyEnv); + assert.equal((await old("/discussions/problems/4711-old")).headers.get("location"), "http://localhost/t/4711/old"); + assert.equal((await old("/discussions/questions")).headers.get("location"), "http://localhost/c/questions"); + assert.equal((await old("/search?q=svg")).headers.get("location"), "http://localhost/search?q=svg"); + assert.equal((await old("/")).headers.get("location"), "http://localhost/"); + assert.equal((await old("//evil.example/x")).headers.get("location"), "http://localhost//evil.example/x"); }); test("watching, grouped digests, hearts, your topics", async () => { diff --git a/web/test/import.test.ts b/web/test/import.test.ts index ddca251..c8c2141 100644 --- a/web/test/import.test.ts +++ b/web/test/import.test.ts @@ -129,6 +129,21 @@ test("converts a Tender export into forum rows", () => { assert.equal(stats.spamPosts, 2); }); +test("points links to the old forum at their new place", () => { + const src = fixture(); + src.exec(`INSERT INTO comments (discussion_id, number, user_id, author_name, author_email, body, internal, created_at, raw) VALUES + (105, 3, NULL, 'Guest', 'guest@example.com', 'See http://support.nodebox.net/discussions/problems/100-crash-on-import. Also [this](https://nodebox.tenderapp.com/discussions/old-category/100-old-title#comment_123), https://support.nodebox.net/discussions/problems and https://github.com/discussions/problems/100-crash-on-import', 0, '2014-05-05T11:00:00Z', '{}');`); + const db = new DatabaseSync(":memory:"); + db.exec(forumSchema); + db.exec(convert(src).statements.join("\n")); + const links = (db.prepare("SELECT body FROM posts WHERE topic_id = 105 AND tender_number = 3").get() as any).body; + assert.equal( + links, + "See https://forum.nodebox.net/t/100/crash-on-import. Also [this](/t/100/crash-on-import), https://forum.nodebox.net/c/problems and https://github.com/discussions/problems/100-crash-on-import", + "bare links stay absolute, moved and renamed discussions are found by number, other hosts are left alone", + ); +}); + test("works without spam verdicts", () => { const db = fixture(); db.exec("DROP VIEW spam_excluded; DROP TABLE spam_verdicts; DROP TABLE spam_decisions;"); diff --git a/web/wrangler.jsonc b/web/wrangler.jsonc index 670b993..d8d99ed 100644 --- a/web/wrangler.jsonc +++ b/web/wrangler.jsonc @@ -24,6 +24,9 @@ // Absolute origin, used in sign-in emails and the Google redirect URI. // These are the production values; `npm run dev` overrides PUBLIC_URL and MAILER. "PUBLIC_URL": "https://forum.nodebox.net", + // The old Tender forum's domain. Once it is a custom domain of this Worker, every request on it + // redirects here: old discussion URLs to their topics, the rest to the same path (web/DEPLOY.md). + "LEGACY_HOSTS": "support.nodebox.net", // Cloudflare Images: image ids are global to the account, so they are prefixed. The variants are the account's shared ones. "CF_IMAGES_PREFIX": "nodebox-forum-", "CF_IMAGES_VARIANT_POST": "medium",