feat(src): configure Discord to Linear backfills

src/bridge/core/mirror.ts

@@ -21,6 +21,11 @@ export class Mirror {
     private readonly source: Pick<Source, "announce">,
   ) {}
 
+  // Whether the post already maps to a hub issue.
+  isMirrored(post: Post): Promise<boolean> {
+    return this.target.findIssueId(post.ref).then((id) => id !== null);
+  }
+
   // Mirrors a post: issue (with the opening message as its body), linking
   // attachment, and labels. Announces the issue back to the source unless
   // suppressed (e.g. during startup backfill of old posts).

src/bridge/discord/index.ts

@@ -63,7 +63,9 @@ export class DiscordConnector implements Source {
       if (!message.inGuild() || !(await isHelpPost(message.channel))) return;
       if (!isHumanMessage(message) || isStarter(message)) return;
       try {
-        const post = await this.postFor(message.channel as ThreadChannel);
+        const thread = message.channel as ThreadChannel;
+        const post = await this.postFor(thread);
+        if (await this.caughtUp(thread, post)) return;
         await this.mirror.addMessage(post, await toMessage(message));
       } catch (err) {
         console.error("[bridge]", "message create failed", err);
@@ -179,7 +181,9 @@ export class DiscordConnector implements Source {
         flush = debounce(1000, async (thread: ThreadChannel) => {
           flushers.delete(thread.id);
           try {
-            await this.mirror.syncStatus(await this.postFor(thread));
+            const post = await this.postFor(thread);
+            if (await this.caughtUp(thread, post)) return;
+            await this.mirror.syncStatus(post);
           } catch (err) {
             console.error("[bridge]", "thread update failed", err);
           }
@@ -210,13 +214,15 @@ export class DiscordConnector implements Source {
     );
   }
 
-  // Mirrors #help threads that aren't fully in the hub yet, so threads and
-  // messages from while the bridge was off still land as issues. With backfillAll
-  // it imports every thread, paging through all archived threads and waiting out
-  // rate limits.
+  // Startup import of #help threads not fully in the hub yet, so threads and
+  // messages from while the bridge was off still land as issues. `days` bounds
+  // it to a recency window; -1 imports everything, paging through all archived
+  // threads and waiting out rate limits.
   async backfill(): Promise<void> {
-    const { backfillAll, backfillLimit, backfillDays } = config.linearBridge;
-    if (!backfillAll && backfillLimit <= 0) return;
+    const { days, limit } = config.linearBridge.deepBackfill;
+    if (days === 0 || limit === 0) return;
+    const all = days < 0;
+    const cutoff = all ? 0 : Date.now() - days * 24 * 60 * 60 * 1000;
 
     const forum = await this.client.channels.fetch(config.helpChannel.id);
     if (!forum || forum.type !== ChannelType.GuildForum) return;
@@ -225,41 +231,40 @@ export class DiscordConnector implements Source {
     const active = await forum.threads.fetchActive();
     for (const thread of active.threads.values()) byId.set(thread.id, thread);
 
-    // Pull archived threads too. For a full import, page through every archived
-    // thread; otherwise a single page bounded by the limit is enough.
+    // Page archived threads (ordered by archive time, newest first). A full
+    // import walks every page; a windowed import stops once a page ends past the
+    // cutoff, since older pages can only be older still.
     let before: Date | undefined;
     do {
-      const page = await forum.threads.fetchArchived({
-        limit: backfillAll ? 100 : backfillLimit,
-        before,
-      });
-      const last = [...page.threads.values()].at(-1);
-      for (const thread of page.threads.values()) byId.set(thread.id, thread);
+      const page = await forum.threads.fetchArchived({ limit: 100, before });
+      const threads = [...page.threads.values()];
+      for (const thread of threads) byId.set(thread.id, thread);
+      const oldest = threads.at(-1);
+      const reachedCutoff =
+        !all && (oldest?.archivedAt?.getTime() ?? 0) < cutoff;
       before =
-        backfillAll && page.hasMore
-          ? (last?.archivedAt ?? undefined)
+        page.hasMore && !reachedCutoff
+          ? (oldest?.archivedAt ?? undefined)
           : undefined;
     } while (before);
 
     const sorted = [...byId.values()].sort((a, b) =>
       (b.lastMessageId ?? "").localeCompare(a.lastMessageId ?? ""),
     );
-    // A normal backfill is bounded by both a count and a recency window, so it
-    // can't reach ancient threads in a low-traffic channel. A full import takes
-    // everything.
-    const cutoff = Date.now() - backfillDays * 24 * 60 * 60 * 1000;
-    const threads = backfillAll
+    const windowed = all
       ? sorted
-      : sorted.filter((t) => lastActivity(t) >= cutoff).slice(0, backfillLimit);
+      : sorted.filter((t) => lastActivity(t) >= cutoff);
+    const threads = limit >= 0 ? windowed.slice(0, limit) : windowed;
 
+    const scope = all
+      ? "(full import)"
+      : `within ${days}d of ${byId.size} fetched`;
+    const capped = limit >= 0 ? ` (limit ${limit})` : "";
     console.log(
       "[bridge]",
       "startup backfill:",
       threads.length,
-      "thread(s)",
-      backfillAll
-        ? "(full import)"
-        : `of ${byId.size} fetched (limit ${backfillLimit}, ${backfillDays}d)`,
+      `thread(s) ${scope}${capped}`,
     );
     for (const thread of threads) {
       try {
@@ -274,9 +279,28 @@ export class DiscordConnector implements Source {
     console.log("[bridge]", "startup backfill complete");
   }
 
+  // Mirrors a thread's full history when live backfill is on and it has no issue
+  // yet, so a thread whose start the bridge missed lands complete on its first
+  // live event. Returns true when it handled the thread, so the caller skips its
+  // per-event mirror.
+  private async caughtUp(thread: ThreadChannel, post: Post): Promise<boolean> {
+    if (!config.linearBridge.backfill.enabled) return false;
+    if (await this.mirror.isMirrored(post)) return false;
+    await withRateLimitRetry(
+      () => this.backfillThread(thread, true),
+      isRateLimited,
+    );
+    return true;
+  }
+
   // Mirrors a thread: ensures the issue exists, fills in missing messages, then
-  // reconciles state. Safe to re-run over already-mirrored threads.
-  private async backfillThread(thread: ThreadChannel): Promise<void> {
+  // reconciles state. Safe to re-run over already-mirrored threads. Announces
+  // the hub link back to the thread only when asked: off for the startup import
+  // of many old threads, on for a live catch-up of an active one.
+  private async backfillThread(
+    thread: ThreadChannel,
+    announce = false,
+  ): Promise<void> {
     console.log("[bridge]", "backfilling thread", thread.id, thread.name);
     const help = new HelpThread(thread);
 
@@ -290,7 +314,7 @@ export class DiscordConnector implements Source {
 
     const starter = await thread.fetchStarterMessage().catch(() => null);
     const post = await toPost(help, starter);
-    await this.mirror.createPost(post, false);
+    await this.mirror.createPost(post, announce);
 
     const fetched = await thread.messages.fetch({ limit: 100 });
     const messages = await Promise.all(

src/lib/config.ts

@@ -47,17 +47,22 @@ interface Config {
     teamId?: string;
     // Optional Linear project that mirrored thread issues are filed under.
     projectId?: string;
-    // Number of most recently active help threads to mirror on startup. 0 to
-    // disable. Threads already mirrored are skipped.
-    backfillLimit: number;
-    // Only mirror threads active within this many days on a normal (non-full)
-    // startup backfill, so it can't reach ancient threads. Ignored by
-    // backfillAll.
-    backfillDays: number;
-    // Mirror every #help thread on startup (all archived pages, ignoring
-    // backfillLimit), retrying through Linear rate limits. Slow; intended for
-    // the initial bulk import.
-    backfillAll: boolean;
+    // Catch up threads whose start we missed: when a live event lands on a
+    // #help thread that has no issue yet, mirror the thread's full history
+    // instead of only that event, so a thread opened while the bridge was off
+    // still lands complete.
+    backfill: {
+      enabled: boolean;
+    };
+    // Startup import that walks back through #help history. `days` mirrors every
+    // thread active within that many days (-1 imports everything, paging all
+    // archived threads and retrying through rate limits; 0 disables it). `limit`
+    // caps how many threads are mirrored, most recent first; -1 is unlimited.
+    // Threads already mirrored are skipped.
+    deepBackfill: {
+      days: number;
+      limit: number;
+    };
     // Attribute mirrored comments to the Discord author via Linear's
     // createAsUser. Requires the app-actor token; turn off to post as the app.
     createAsUser: boolean;
@@ -86,9 +91,13 @@ export const { config, layers } = await loadConfig<Config>({
     linearBridge: {
       enabled: false,
       createAsUser: false,
-      backfillLimit: 50,
-      backfillDays: 14,
-      backfillAll: false,
+      backfill: {
+        enabled: true,
+      },
+      deepBackfill: {
+        days: 90,
+        limit: -1,
+      },
       labels: {
         // Label creation runs on the user token, which can manage the team's
         // labels. Each #help tag becomes a flat label named "<namespace> > tag";
@@ -123,9 +132,9 @@ export const { config, layers } = await loadConfig<Config>({
 });
 
 // configmasher does not coerce types: values from env files or process.env
-// arrive as strings, so a boolean like `backfillAll=false` would be the truthy
-// string "false". Coerce the env-overridable booleans and numbers to their real
-// types after loading.
+// arrive as strings, so a boolean like `backfill.enabled=false` would be the
+// truthy string "false". Coerce the env-overridable booleans and numbers to
+// their real types after loading.
 function bool(value: unknown, fallback: boolean): boolean {
   if (typeof value === "boolean") return value;
   if (value === "true") return true;
@@ -145,9 +154,18 @@ config.linearBridge.createAsUser = bool(
   config.linearBridge.createAsUser,
   false,
 );
-config.linearBridge.backfillAll = bool(config.linearBridge.backfillAll, false);
-config.linearBridge.backfillLimit = num(config.linearBridge.backfillLimit, 50);
-config.linearBridge.backfillDays = num(config.linearBridge.backfillDays, 14);
+config.linearBridge.backfill.enabled = bool(
+  config.linearBridge.backfill.enabled,
+  true,
+);
+config.linearBridge.deepBackfill.days = num(
+  config.linearBridge.deepBackfill.days,
+  90,
+);
+config.linearBridge.deepBackfill.limit = num(
+  config.linearBridge.deepBackfill.limit,
+  -1,
+);
 config.linearBridge.labels.enabled = bool(
   config.linearBridge.labels.enabled,
   true,