From 86848d793c9c149b5dd84bc020cee6d5ec45b214 Mon Sep 17 00:00:00 2001 From: phorcys420 <57866459+phorcys420@users.noreply.github.com> Date: Mon, 24 Aug 2026 18:14:46 +0000 Subject: [PATCH] feat(src): configure Discord to Linear backfills --- src/bridge/core/mirror.ts | 5 +++ src/bridge/discord/index.ts | 86 ++++++++++++++++++++++++------------- src/lib/config.ts | 58 ++++++++++++++++--------- 3 files changed, 98 insertions(+), 51 deletions(-) diff --git a/src/bridge/core/mirror.ts b/src/bridge/core/mirror.ts index a6b3ed8..1a3c5e9 100644 --- a/src/bridge/core/mirror.ts +++ b/src/bridge/core/mirror.ts @@ -21,6 +21,11 @@ export class Mirror { private readonly source: Pick, ) {} + // Whether the post already maps to a hub issue. + isMirrored(post: Post): Promise { + return this.target.findIssueId(post.ref).then((id) => id !== null); + } + // Mirrors a post: issue (with the opening message as its body), linking // attachment, and labels. Announces the issue back to the source unless // suppressed (e.g. during startup backfill of old posts). diff --git a/src/bridge/discord/index.ts b/src/bridge/discord/index.ts index 9dabfc4..9ce194d 100644 --- a/src/bridge/discord/index.ts +++ b/src/bridge/discord/index.ts @@ -63,7 +63,9 @@ export class DiscordConnector implements Source { if (!message.inGuild() || !(await isHelpPost(message.channel))) return; if (!isHumanMessage(message) || isStarter(message)) return; try { - const post = await this.postFor(message.channel as ThreadChannel); + const thread = message.channel as ThreadChannel; + const post = await this.postFor(thread); + if (await this.caughtUp(thread, post)) return; await this.mirror.addMessage(post, await toMessage(message)); } catch (err) { console.error("[bridge]", "message create failed", err); @@ -179,7 +181,9 @@ export class DiscordConnector implements Source { flush = debounce(1000, async (thread: ThreadChannel) => { flushers.delete(thread.id); try { - await this.mirror.syncStatus(await this.postFor(thread)); + const post = await this.postFor(thread); + if (await this.caughtUp(thread, post)) return; + await this.mirror.syncStatus(post); } catch (err) { console.error("[bridge]", "thread update failed", err); } @@ -210,13 +214,15 @@ export class DiscordConnector implements Source { ); } - // Mirrors #help threads that aren't fully in the hub yet, so threads and - // messages from while the bridge was off still land as issues. With backfillAll - // it imports every thread, paging through all archived threads and waiting out - // rate limits. + // Startup import of #help threads not fully in the hub yet, so threads and + // messages from while the bridge was off still land as issues. `days` bounds + // it to a recency window; -1 imports everything, paging through all archived + // threads and waiting out rate limits. async backfill(): Promise { - const { backfillAll, backfillLimit, backfillDays } = config.linearBridge; - if (!backfillAll && backfillLimit <= 0) return; + const { days, limit } = config.linearBridge.deepBackfill; + if (days === 0 || limit === 0) return; + const all = days < 0; + const cutoff = all ? 0 : Date.now() - days * 24 * 60 * 60 * 1000; const forum = await this.client.channels.fetch(config.helpChannel.id); if (!forum || forum.type !== ChannelType.GuildForum) return; @@ -225,41 +231,40 @@ export class DiscordConnector implements Source { const active = await forum.threads.fetchActive(); for (const thread of active.threads.values()) byId.set(thread.id, thread); - // Pull archived threads too. For a full import, page through every archived - // thread; otherwise a single page bounded by the limit is enough. + // Page archived threads (ordered by archive time, newest first). A full + // import walks every page; a windowed import stops once a page ends past the + // cutoff, since older pages can only be older still. let before: Date | undefined; do { - const page = await forum.threads.fetchArchived({ - limit: backfillAll ? 100 : backfillLimit, - before, - }); - const last = [...page.threads.values()].at(-1); - for (const thread of page.threads.values()) byId.set(thread.id, thread); + const page = await forum.threads.fetchArchived({ limit: 100, before }); + const threads = [...page.threads.values()]; + for (const thread of threads) byId.set(thread.id, thread); + const oldest = threads.at(-1); + const reachedCutoff = + !all && (oldest?.archivedAt?.getTime() ?? 0) < cutoff; before = - backfillAll && page.hasMore - ? (last?.archivedAt ?? undefined) + page.hasMore && !reachedCutoff + ? (oldest?.archivedAt ?? undefined) : undefined; } while (before); const sorted = [...byId.values()].sort((a, b) => (b.lastMessageId ?? "").localeCompare(a.lastMessageId ?? ""), ); - // A normal backfill is bounded by both a count and a recency window, so it - // can't reach ancient threads in a low-traffic channel. A full import takes - // everything. - const cutoff = Date.now() - backfillDays * 24 * 60 * 60 * 1000; - const threads = backfillAll + const windowed = all ? sorted - : sorted.filter((t) => lastActivity(t) >= cutoff).slice(0, backfillLimit); + : sorted.filter((t) => lastActivity(t) >= cutoff); + const threads = limit >= 0 ? windowed.slice(0, limit) : windowed; + const scope = all + ? "(full import)" + : `within ${days}d of ${byId.size} fetched`; + const capped = limit >= 0 ? ` (limit ${limit})` : ""; console.log( "[bridge]", "startup backfill:", threads.length, - "thread(s)", - backfillAll - ? "(full import)" - : `of ${byId.size} fetched (limit ${backfillLimit}, ${backfillDays}d)`, + `thread(s) ${scope}${capped}`, ); for (const thread of threads) { try { @@ -274,9 +279,28 @@ export class DiscordConnector implements Source { console.log("[bridge]", "startup backfill complete"); } + // Mirrors a thread's full history when live backfill is on and it has no issue + // yet, so a thread whose start the bridge missed lands complete on its first + // live event. Returns true when it handled the thread, so the caller skips its + // per-event mirror. + private async caughtUp(thread: ThreadChannel, post: Post): Promise { + if (!config.linearBridge.backfill.enabled) return false; + if (await this.mirror.isMirrored(post)) return false; + await withRateLimitRetry( + () => this.backfillThread(thread, true), + isRateLimited, + ); + return true; + } + // Mirrors a thread: ensures the issue exists, fills in missing messages, then - // reconciles state. Safe to re-run over already-mirrored threads. - private async backfillThread(thread: ThreadChannel): Promise { + // reconciles state. Safe to re-run over already-mirrored threads. Announces + // the hub link back to the thread only when asked: off for the startup import + // of many old threads, on for a live catch-up of an active one. + private async backfillThread( + thread: ThreadChannel, + announce = false, + ): Promise { console.log("[bridge]", "backfilling thread", thread.id, thread.name); const help = new HelpThread(thread); @@ -290,7 +314,7 @@ export class DiscordConnector implements Source { const starter = await thread.fetchStarterMessage().catch(() => null); const post = await toPost(help, starter); - await this.mirror.createPost(post, false); + await this.mirror.createPost(post, announce); const fetched = await thread.messages.fetch({ limit: 100 }); const messages = await Promise.all( diff --git a/src/lib/config.ts b/src/lib/config.ts index 27e32c7..758b48e 100644 --- a/src/lib/config.ts +++ b/src/lib/config.ts @@ -47,17 +47,22 @@ interface Config { teamId?: string; // Optional Linear project that mirrored thread issues are filed under. projectId?: string; - // Number of most recently active help threads to mirror on startup. 0 to - // disable. Threads already mirrored are skipped. - backfillLimit: number; - // Only mirror threads active within this many days on a normal (non-full) - // startup backfill, so it can't reach ancient threads. Ignored by - // backfillAll. - backfillDays: number; - // Mirror every #help thread on startup (all archived pages, ignoring - // backfillLimit), retrying through Linear rate limits. Slow; intended for - // the initial bulk import. - backfillAll: boolean; + // Catch up threads whose start we missed: when a live event lands on a + // #help thread that has no issue yet, mirror the thread's full history + // instead of only that event, so a thread opened while the bridge was off + // still lands complete. + backfill: { + enabled: boolean; + }; + // Startup import that walks back through #help history. `days` mirrors every + // thread active within that many days (-1 imports everything, paging all + // archived threads and retrying through rate limits; 0 disables it). `limit` + // caps how many threads are mirrored, most recent first; -1 is unlimited. + // Threads already mirrored are skipped. + deepBackfill: { + days: number; + limit: number; + }; // Attribute mirrored comments to the Discord author via Linear's // createAsUser. Requires the app-actor token; turn off to post as the app. createAsUser: boolean; @@ -86,9 +91,13 @@ export const { config, layers } = await loadConfig({ linearBridge: { enabled: false, createAsUser: false, - backfillLimit: 50, - backfillDays: 14, - backfillAll: false, + backfill: { + enabled: true, + }, + deepBackfill: { + days: 90, + limit: -1, + }, labels: { // Label creation runs on the user token, which can manage the team's // labels. Each #help tag becomes a flat label named " > tag"; @@ -123,9 +132,9 @@ export const { config, layers } = await loadConfig({ }); // configmasher does not coerce types: values from env files or process.env -// arrive as strings, so a boolean like `backfillAll=false` would be the truthy -// string "false". Coerce the env-overridable booleans and numbers to their real -// types after loading. +// arrive as strings, so a boolean like `backfill.enabled=false` would be the +// truthy string "false". Coerce the env-overridable booleans and numbers to +// their real types after loading. function bool(value: unknown, fallback: boolean): boolean { if (typeof value === "boolean") return value; if (value === "true") return true; @@ -145,9 +154,18 @@ config.linearBridge.createAsUser = bool( config.linearBridge.createAsUser, false, ); -config.linearBridge.backfillAll = bool(config.linearBridge.backfillAll, false); -config.linearBridge.backfillLimit = num(config.linearBridge.backfillLimit, 50); -config.linearBridge.backfillDays = num(config.linearBridge.backfillDays, 14); +config.linearBridge.backfill.enabled = bool( + config.linearBridge.backfill.enabled, + true, +); +config.linearBridge.deepBackfill.days = num( + config.linearBridge.deepBackfill.days, + 90, +); +config.linearBridge.deepBackfill.limit = num( + config.linearBridge.deepBackfill.limit, + -1, +); config.linearBridge.labels.enabled = bool( config.linearBridge.labels.enabled, true,