Find the people behind the feeds when they never wrote themselves down

apps/poller/src/index.js

@@ -2,6 +2,7 @@ import { connect, migrate, q, accounts, alerts } from '@rssamplifier/db';
 import {
   crawlDue,
   enrichDue,
+  searchDue,
   notifyFinishedSubmissions,
   notifyFinishedDiscoveries,
   drainDiscoveryQueue,
@@ -145,6 +146,32 @@ const authorVerify = env['AUTHOR_VERIFY'] === '1' || env['AUTHOR_VERIFY'] === 't
 // is the switch to reach for if a site ever objects to the extra fetches.
 const authorEnabled = env['AUTHOR_ENRICH'] !== '0' && env['AUTHOR_ENRICH'] !== 'false';
 
+// A GitHub credential, and the difference between the profile lookups working
+// and not working at all: unauthenticated, the API allows **60 requests an hour
+// per IP**, which a single batch spends. With a token it is 5,000, which is
+// more than this pass can use. Optional only in the sense that the rest of the
+// enrichment carries on without it -- profile resolution simply stops finding
+// anything once the hour's 60 are gone, and does so quietly, as a 403.
+const authorGithubToken = String(env['GITHUB_TOKEN'] ?? '').trim();
+
+// Buying searches for the people who left no trail.
+//
+// **Off unless every one of these is set**, and that is the design rather than
+// caution. The credits are metered, they come from an account shared with
+// another product, and there are 369,056 feeds here -- one query each would be
+// fifteen times the monthly allowance. So it takes a key, a non-zero budget,
+// and an explicit switch, and the budget is counted from the ledger in the
+// database rather than from a variable, because this process restarts on every
+// deploy and a budget that resets with it is not a budget.
+const searchEnabled = env['AUTHOR_SEARCH'] === '1' || env['AUTHOR_SEARCH'] === 'true';
+const searchApiKey = String(env['VALUESERP_API_KEY'] ?? '').trim();
+const searchMonthlyBudget = Number(env['AUTHOR_SEARCH_BUDGET']) || 0;
+const searchPerAuthor = Number(env['AUTHOR_SEARCH_PER_AUTHOR']) || 2;
+const searchBatch = Number(env['AUTHOR_SEARCH_BATCH']) || 5;
+// Slow on purpose: this is the one pass that costs money per unit of work, so
+// its default cadence spends at most a few credits an hour even misconfigured.
+const searchIntervalMs = (Number(env['AUTHOR_SEARCH_INTERVAL_SECONDS']) || 900) * 1000;
+
 // Accounts considered per alert pass, and how often a pass runs. Its own timer
 // for the same reason the card and cluster passes have one: a tick spends
 // minutes inside the crawl, and work queued behind that only happens if the
@@ -590,6 +617,7 @@ async function enrichTick() {
       verify: authorVerify,
       recheckDays: authorRecheckDays,
       concurrency: authorConcurrency,
+      githubToken: authorGithubToken,
       onEvent: publishLog ? recorder.record : null,
     });
     // Silent when nothing was due, which is the steady state once the
@@ -602,6 +630,38 @@ async function enrichTick() {
   }
 }
 
+/**
+ * Spend a little of the month's search allowance on the unreachable.
+ *
+ * Guarded twice over. It does nothing without a key, a budget and the switch;
+ * and `searchDue` re-reads the ledger every pass, so two pollers or a restarted
+ * one cannot between them spend more than the month allows.
+ */
+let searching = false;
+
+async function searchTick() {
+  if (!searchEnabled || !searchApiKey || searchMonthlyBudget <= 0) return;
+  if (searching) return;
+  searching = true;
+
+  try {
+    const result = await searchDue(db, {
+      apiKey: searchApiKey,
+      monthlyBudget: searchMonthlyBudget,
+      perAuthor: searchPerAuthor,
+      batchSize: searchBatch,
+    });
+    // Logged whenever anything was bought, even when it found nothing --
+    // especially then. A pass that spends credits and stores no links is the
+    // signal that the gate needs tightening, and it is invisible otherwise.
+    if (result.spent) log('author-search', result);
+  } catch (err) {
+    log('author-search-error', { message: String(err?.message ?? err) });
+  } finally {
+    searching = false;
+  }
+}
+
 /**
  * Write down how deep each queue is, so /crawlstats can draw the slope.
  *
@@ -647,6 +707,7 @@ const cardTimer = setInterval(cardTick, cardIntervalMs);
 const alertTimer = setInterval(alertTick, alertIntervalMs);
 const enrichTimer = setInterval(enrichTick, authorIntervalMs);
 const queueTimer = setInterval(queueTick, queueSampleMs);
+const searchTimer = setInterval(searchTick, searchIntervalMs);
 void tick();
 void backfillTick();
 void cardTick();
@@ -684,6 +745,7 @@ function shutdown(signal) {
   clearInterval(enrichTimer);
   clearInterval(alertTimer);
   clearInterval(queueTimer);
+  clearInterval(searchTimer);
 
   // Recorded before the buffer is closed, so the live log's last line is the
   // daemon saying it stopped rather than the log simply going quiet — which is

apps/web/src/lib/jobs.js

@@ -102,6 +102,20 @@ export function jobRows({
       // The one row where a backlog is a problem rather than a workload.
       alarmAbove: 500,
     },
+    {
+      key: 'authors',
+      label: 'Author enrichment',
+      what: 'Finds who writes each feed, and how to reach them',
+      backlog: backlogs.authorsPending ?? 0,
+      // Genuinely a queue rather than a fire: it walks the directory once and
+      // then rechecks on a 90-day cycle, so a large number here is the work
+      // remaining and not a stall.
+      expectFull: true,
+      rate: backlogs.authorsLastHour ?? 0,
+      rateNote: 'publishers looked at',
+      done: `${backlogs.authorsDone ?? 0} looked at so far`,
+      events: ['author', 'authors-error', 'author-search', 'author-search-error'],
+    },
     {
       key: 'cards',
       label: 'Feed pictures',

apps/web/test/jobs.test.js

@@ -21,6 +21,9 @@ function busy(overrides = {}) {
       cardsNone: 283,
       cardsError: 31,
       cardsLastHour: 1_320,
+      authorsPending: 81_123,
+      authorsDone: 3_275,
+      authorsLastHour: 283,
       ...overrides.backlogs,
     },
     activity: {
@@ -32,6 +35,7 @@ function busy(overrides = {}) {
       'cluster-backfill': { lines: 360, errors: 0, amount: 12_000, lastAt: at, ms: 90 },
       topics: { lines: 4, errors: 0, amount: 0, lastAt: at, ms: 300 },
       alerts: { lines: 30, errors: 0, amount: 48, lastAt: at, ms: 600 },
+      author: { lines: 283, errors: 4, amount: 60, lastAt: at, ms: 2_400 },
       ...overrides.activity,
     },
     alertAccounts: overrides.alertAccounts ?? 4,

packages/db/migrations/20260819081824_author_searches.sql

@@ -0,0 +1,38 @@
+-- What we spent looking for people, and on whom.
+--
+-- Author enrichment reads what publishers put where we could find it, and that
+-- is free. Buying a search is not: the credits come from a metered account
+-- shared with another product, and the directory is large enough that an
+-- unbounded pass would spend a month's allowance in an afternoon.
+--
+-- A counter in the process would not do. The poller restarts on every deploy,
+-- and a budget that resets when the process does is not a budget — it is a
+-- rate limit with a hole in it. So the spend is written down, and the ledger is
+-- the authority.
+--
+-- It is also the audit trail, which matters as much. "Which people did we spend
+-- money looking for, and did it find anything" is a question worth being able
+-- to answer, both to tune the gate that chooses them and to justify the line
+-- item.
+create table if not exists author_searches (
+  id        text primary key,
+
+  -- Null once an author is deleted: the spend still happened, and dropping the
+  -- row would make the month's total disagree with what was billed.
+  author_id text references authors (id) on delete set null,
+
+  at        text not null,
+
+  -- Credits actually spent, which is not the same as results found. A query
+  -- that returns nothing is still billed, and a budget that counted only hits
+  -- would overspend precisely on the people who are hardest to find.
+  queries   integer not null default 0,
+
+  -- Links stored as a result, so the gate can be judged on its yield rather
+  -- than on how it felt.
+  found     integer not null default 0
+);
+
+-- The only query this table answers in the hot path: how much has been spent
+-- since the start of the current billing period.
+create index if not exists author_searches_at_idx on author_searches (at);

packages/db/src/authors.js

@@ -43,6 +43,46 @@ export async function dueForAuthors(db, limit = 25, recheckBefore = '') {
   return /** @type {any} */ (rows);
 }
 
+/**
+ * Record that a feed's enrichment *failed*, so it is tried again sooner.
+ *
+ * `markAuthorsChecked` is right for a miss — a site that genuinely names nobody
+ * should not be re-read tomorrow — and wrong for a failure. They were the same
+ * call, which meant a DNS hiccup, a timeout or a 503 cost that publisher its
+ * enrichment for the **full recheck cycle**, ninety days, on the strength of one
+ * bad afternoon. On a pass that has so far reached 3,275 of 369,056 feeds, that
+ * is a quiet way to lose the ones on flaky hosts permanently.
+ *
+ * Done by back-dating the stamp rather than by adding an attempts column, and
+ * that is a deliberate trade. Writes on this database serialize and the crawl
+ * is already write-bound (see the notes in `crawl.js`), so the cheap fix that
+ * costs one UPDATE beats the tidy one that costs a migration and a second
+ * column to read. The feed still counts as "looked at" for the backlog, and
+ * still comes due again in `retryDays`.
+ *
+ * It must never back-date past *now*: a stamp in the future would hide the feed
+ * from a pass whose recheck window is shorter than this one's.
+ *
+ * @param {Client} db
+ * @param {string} feedId
+ * @param {{ retryDays?: number, recheckDays?: number }} [opts]
+ * @returns {Promise<void>}
+ */
+export async function markAuthorsFailed(db, feedId, opts = {}) {
+  const retryDays = Math.max(0, Number(opts.retryDays ?? 3));
+  const recheckDays = Math.max(retryDays, Number(opts.recheckDays ?? 90));
+
+  // Stamped as though it were checked (recheckDays - retryDays) ago, so the
+  // ordinary due test brings it back in retryDays without knowing why.
+  const backdated = (recheckDays - retryDays) * 86_400_000;
+  const at = new Date(Date.now() - backdated).toISOString();
+
+  await db.execute({
+    sql: 'update feeds set authors_checked_at = ? where id = ?',
+    args: [at, feedId],
+  });
+}
+
 /**
  * Record that a feed has been looked at, whether or not anyone was found.
  *
@@ -812,3 +852,108 @@ export async function postsByAuthor(db, feedIds, limit = 12) {
 
   return rows;
 }
+
+/* ------------------------------------------------------------------ *
+ * Bought searches
+ * ------------------------------------------------------------------ */
+
+/**
+ * The people it would be worth buying a search for.
+ *
+ * The gate is mean because the budget is small. What it selects for is the
+ * person we are confident *is* a person, who writes here, and whom nobody can
+ * currently contact — which is the only case where a paid query buys something
+ * the free sources could not.
+ *
+ * Ordered by how much they publish, so a limited budget is spent on the
+ * publishers a reader is most likely to want to reach.
+ *
+ * Anyone searched for already is excluded outright rather than re-searched on a
+ * schedule: a second query for somebody the web did not know about the first
+ * time is the easiest way to spend a month's credits on nothing.
+ *
+ * @param {Client} db
+ * @param {number} [limit]
+ * @param {number} [minConfidence] the floor the caller publishes at
+ * @returns {Promise<object[]>}
+ */
+export async function authorsWithoutContact(db, limit = 10, minConfidence = 0.8) {
+  const { rows } = await db.execute({
+    sql: `select a.id, a.slug, a.name, a.site_url as site, a.confidence,
+                 count(distinct fa.feed_id) as feed_count
+            from authors a
+            join feed_authors fa on fa.author_id = a.id
+           where a.confidence >= ?
+             and not exists (select 1 from author_links l where l.author_id = a.id)
+             and not exists (select 1 from author_searches s where s.author_id = a.id)
+             -- A single word is not a searchable name: it returns the world.
+             and instr(trim(a.name), ' ') > 0
+           group by a.id
+           order by feed_count desc, a.confidence desc
+           limit ?`,
+    args: [minConfidence, limit],
+  });
+
+  return rows;
+}
+
+/**
+ * Write down what a search cost, whether or not it found anything.
+ *
+ * @param {Client} db
+ * @param {{ authorId: string|null, queries: number, found: number }} spend
+ * @returns {Promise<void>}
+ */
+export async function recordAuthorSearch(db, spend) {
+  await db.execute({
+    sql: `insert into author_searches (id, author_id, at, queries, found)
+          values (?, ?, ?, ?, ?)`,
+    args: [
+      newId(),
+      spend.authorId ?? null,
+      nowIso(),
+      Math.max(0, Math.floor(Number(spend.queries) || 0)),
+      Math.max(0, Math.floor(Number(spend.found) || 0)),
+    ],
+  });
+}
+
+/**
+ * Credits spent since a moment, which is how much of the budget is gone.
+ *
+ * @param {Client} db
+ * @param {string} since ISO 8601
+ * @returns {Promise<number>}
+ */
+export async function searchSpendSince(db, since) {
+  const { rows } = await db.execute({
+    sql: 'select coalesce(sum(queries), 0) as spent from author_searches where at >= ?',
+    args: [String(since)],
+  });
+
+  return Number(rows[0]?.spent ?? 0);
+}
+
+/**
+ * The start of the current billing period.
+ *
+ * The provider's month does not begin on the first: ValueSERP resets this
+ * account's allowance on the **13th**, so a budget counted per calendar month
+ * would let the allowance be spent twice across a reset and refuse spending
+ * that is actually available just after one.
+ *
+ * @param {Date} [now]
+ * @param {number} [resetDay]
+ * @returns {string} ISO 8601
+ */
+export function billingPeriodStart(now = new Date(), resetDay = 13) {
+  const at = new Date(now.getTime());
+  const start = new Date(
+    Date.UTC(at.getUTCFullYear(), at.getUTCMonth(), resetDay, 0, 0, 0, 0),
+  );
+
+  // Before this month's reset day, the period began last month.
+  if (at.getTime() < start.getTime()) start.setUTCMonth(start.getUTCMonth() - 1);
+
+  return start.toISOString();
+}

packages/db/src/queries.js

@@ -1382,7 +1382,7 @@ export async function jobBacklogs(db) {
   // The fix is not "stop using conditional aggregates" — two of them survive
   // below. It is to make sure the scan they force is over a *covering index*
   // rather than over the table, at which point the same CASE is cheap.
-  const [byStatus, backlog, submitted, cards] = await Promise.all([
+  const [byStatus, backlog, submitted, cards, enriched] = await Promise.all([
     // One covering scan of feeds_status_success_idx (0028) answering three
     // questions at once: the status breakdown, and how many feeds in each state
     // have never once been read successfully.
@@ -1410,6 +1410,20 @@ export async function jobBacklogs(db) {
             from feeds group by card_state`,
       args: [hourAgo],
     }),
+
+    // How far the author enrichment has walked, read off the partial index
+    // 0024 already built for it (`feeds (authors_checked_at) where status =
+    // 'active'`). Counted as the *stamped* set rather than the unstamped one
+    // for the reason this whole function exists: 3,275 of 369,056 feeds carry a
+    // stamp, so this touches a few thousand index entries, while asking for the
+    // complement would visit every row. The backlog is arithmetic afterwards.
+    db.execute({
+      sql: `select count(*) as n,
+                   sum(case when authors_checked_at >= ? then 1 else 0 end) as hour
+            from feeds
+           where status = 'active' and authors_checked_at is not null`,
+      args: [hourAgo],
+    }),
   ]);
 
   // Deliberately no "first crawls completed this hour". Nothing records when a
@@ -1443,6 +1457,9 @@ export async function jobBacklogs(db) {
     cardsNone: card('none'),
     cardsError: card('error'),
     cardsLastHour: cards.rows.reduce((a, r) => a + Number(r.hour ?? 0), 0),
+    authorsDone: Number(enriched.rows[0]?.n ?? 0),
+    authorsPending: Math.max(0, n('active') - Number(enriched.rows[0]?.n ?? 0)),
+    authorsLastHour: Number(enriched.rows[0]?.hour ?? 0),
   };
 }
 

packages/feed/index.js

@@ -67,6 +67,7 @@ export {
   feedContacts,
   feedCredits,
   identityFromHtml,
+  identityFromHumansTxt,
   identityKey,
   isRoleEmail,
   linksBackTo,
@@ -78,3 +79,4 @@ export {
   personalEmail,
   splitBylines,
 } from './src/identity.js';
+export { hostIdentity, identityFromProfile, profileRequest } from './src/platforms.js';