Compute the slow breakdown where nothing is waiting for it

apps/poller/src/index.js

@@ -1,4 +1,13 @@
-import { connect, createWriteWorker, migrate, q, accounts, alerts, takeWriteTally } from '@rssamplifier/db';
+import {
+  connect,
+  createWriteWorker,
+  migrate,
+  q,
+  accounts,
+  alerts,
+  takeWriteTally,
+  warmStatsCache,
+} from '@rssamplifier/db';
 import {
   crawlDue,
   enrichDue,
@@ -135,6 +144,16 @@ const authorIntervalMs = (Number(env['AUTHOR_INTERVAL_SECONDS']) || 60) * 1000;
 // minutes stale on a page that refreshes every fifteen seconds.
 const queueSampleMs = (Number(env['QUEUE_SAMPLE_SECONDS']) || 600) * 1000;
 
+// How often the category breakdown is recomputed into Redis for /crawlstats.
+//
+// It is a ~59-second read of every feed, which is why it cannot live on a
+// request: with a 30-second deadline in front of it, it could only ever fail,
+// so the cache it was supposed to fill stayed empty and every visitor paid the
+// full timeout. Five minutes is far more often than the number moves -- it is
+// the shape of a directory of half a million feeds -- and the point is only
+// that Redis is never empty, not that it is current to the second.
+const statsWarmMs = (Number(env['STATS_WARM_SECONDS']) || 300) * 1000;
+
 // Where the write queue lives. Absent, `connect()` falls back to the
 // in-process serialiser and this daemon starts no worker — the system as it
 // shipped before the queue existed, which is the right thing to degrade to.
@@ -831,6 +850,34 @@ async function queueTick() {
   }
 }
 
+/** Guards against a warm that outruns its interval starting a second one. */
+let warming = false;
+
+/**
+ * Recompute the category breakdown into Redis, so no page has to.
+ *
+ * This read takes about a minute against half a million feeds and cannot be
+ * made cheap: the columns it groups by are rewritten on every crawl, so an
+ * index covering them would be paid for on the write path, which is the one
+ * thing this system has none of to spare. Doing it here instead costs the
+ * crawler nothing -- it is a read, and reads do not queue behind the single
+ * writer everything else contends for.
+ *
+ * `warmStatsCache` never throws; the result is logged rather than acted on,
+ * because there is nothing to do about a warm that did not happen except serve
+ * the previous answer, which is what the cache already does.
+ */
+async function statsTick() {
+  if (warming) return;
+  warming = true;
+  try {
+    const result = await warmStatsCache({ log });
+    if (!result.ok) return;
+  } finally {
+    warming = false;
+  }
+}
+
 /**
  * The one process that actually writes to Turso.
  *
@@ -881,6 +928,7 @@ const cardTimer = setInterval(cardTick, cardIntervalMs);
 const alertTimer = setInterval(alertTick, alertIntervalMs);
 const enrichTimer = setInterval(enrichTick, authorIntervalMs);
 const queueTimer = setInterval(queueTick, queueSampleMs);
+const statsTimer = setInterval(statsTick, statsWarmMs);
 const searchTimer = setInterval(searchTick, searchIntervalMs);
 // Same cadence as the queue sample: long enough that the line is a summary
 // rather than a stream, short enough to bracket an experiment against.
@@ -891,6 +939,9 @@ void cardTick();
 void alertTick();
 void enrichTick();
 void queueTick();
+// Run once at boot, so a deploy does not leave the page slow until the first
+// interval comes round.
+void statsTick();
 
 log('started', {
   intervalSeconds: intervalMs / 1000,
@@ -922,6 +973,7 @@ function shutdown(signal) {
   clearInterval(enrichTimer);
   clearInterval(alertTimer);
   clearInterval(queueTimer);
+  clearInterval(statsTimer);
   clearInterval(searchTimer);
   clearInterval(tallyTimer);
 

apps/web/src/app/api/crawlstats/route.js

@@ -1,7 +1,14 @@
-import { q, discovery, alerts } from '@rssamplifier/db';
+import { q, discovery } from '@rssamplifier/db';
 
 import { db } from '../../../lib/db.js';
-import { categoryStats, indexingHistory, jobBacklogs } from '../../../lib/crawlstats.js';
+import {
+  categoryStats,
+  indexingHistory,
+  jobBacklogs,
+  liveStats,
+  failingFeeds,
+  alertingAccounts,
+} from '../../../lib/crawlstats.js';
 import { toLine } from '../../../lib/crawlLog.js';
 import { jobRows } from '../../../lib/jobs.js';
 
@@ -15,10 +22,15 @@ export const dynamic = 'force-dynamic';
  * backlog that should drain within a tick or two, `stale` is active feeds the
  * crawler has not successfully read in a day.
  *
- * Deliberately uncached — a status endpoint that answers from a cache reports
- * that everything was fine a minute ago, which is the one thing it must not do.
- * The two additions that are cached, briefly, are the ones nothing would alert
- * on: the hourly history and the category breakdown. See lib/crawlstats.js.
+ * Every read here is cached, and the distinction that keeps that honest is
+ * between a fact and a derivation. A count is a fact and may be ten seconds
+ * old; `idleMinutes` is `now - lastSuccessAt`, so caching it would freeze the
+ * one number a monitor alerts on. `liveStats` caches the timestamp and redoes
+ * the subtraction — see lib/crawlstats.js.
+ *
+ * Before this, the endpoint answered in 118 seconds: `categoryStats` no longer
+ * completes inside the client's 30s deadline, and since a cache that only
+ * stores successes never stored it, every request paid the full timeout.
  */
 export async function GET() {
   const client = db();
@@ -36,8 +48,8 @@ export async function GET() {
     alertAccounts,
     operationalErrors,
   ] = await Promise.all([
-    q.crawlStats(client),
-    q.failingFeeds(client, 20),
+    liveStats(),
+    failingFeeds(),
     q.recentlyCrawled(client, 20),
     discovery.countQueuedCandidates(client),
     discovery.countQueuedKeywords(client),
@@ -57,7 +69,7 @@ export async function GET() {
     q.logActivity(client, 1),
     // See the page: this only tells a sender with nobody to serve from one that
     // has stopped, which the log alone cannot say.
-    alerts.alertingAccountCount(client),
+    alertingAccounts(),
     q.crawlOperationalErrors(client, { limit: 20, hours: 24 }),
   ]);
 

apps/web/src/app/crawlstats/page.jsx

@@ -1,11 +1,14 @@
-import { q, discovery, alerts } from '@rssamplifier/db';
+import { q, discovery } from '@rssamplifier/db';
 
 import { db } from '../../lib/db.js';
 import {
   categoryStats,
   indexingHistory,
   jobBacklogs,
   queueHistory,
+  liveStats,
+  failingFeeds,
+  alertingAccounts,
   GROWTH_DAYS,
 } from '../../lib/crawlstats.js';
 import { describe, toLine } from '../../lib/crawlLog.js';
@@ -62,8 +65,8 @@ export default async function CrawlStatsPage() {
     operationalErrors,
     queues,
   ] = await Promise.all([
-    q.crawlStats(client),
-    q.failingFeeds(client, 50),
+    liveStats(),
+    failingFeeds(50),
     q.recentlyCrawled(client, 15),
     discovery.countQueuedCandidates(client),
     discovery.countQueuedKeywords(client),
@@ -90,7 +93,7 @@ export default async function CrawlStatsPage() {
     // Only to tell a sender with nothing to do from one that has stopped: the
     // alert pass writes no log line at all when nobody is subscribed, and a
     // silent job is otherwise indistinguishable from a dead one.
-    alerts.alertingAccountCount(client),
+    alertingAccounts(),
     // Kept separately from the rolling live-log window. At crawler throughput,
     // 400 successful feed lines can evict an operational failure in minutes.
     q.crawlOperationalErrors(client, { limit: 20, hours: 24 }),