From 855cbecc3679902e29a88d05d5d52016bbcbf67d Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Wed, 19 Aug 2026 08:09:26 +0000 Subject: [PATCH 1/5] Find the author when the blog names nobody, by reading the address MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /authors/pierre-felgines is a name and nothing else: no email, no site, no accounts. The pass that produced it was not broken. It fetched felginep.github.io, found no rel="me", no h-card, no JSON-LD and exactly one outbound link — to the Jekyll theme its author happened to use — correctly concluded there was nobody to credit, and stamped the feed as checked. Meanwhile the author's GitHub account was named in the hostname the feed is served from, and one request to it returns "Pierre Felgines". That is the gap this closes, and it is not a corner. Of 369,056 feeds only 3,275 have ever been enriched, and 77% of the identity we hold came free out of feed documents rather than from visiting anything. Reading what publishers marked up works beautifully on the part of the small web that marks itself up; this reads the two things nobody had to publish. **The host.** A blog on .github.io, .substack.com, medium.com/@user and seven more names its owner's account in the address. Deriving it is string arithmetic on URLs already in hand, so a feed on an unrecognised platform costs nothing extra at all. **The profile behind it.** GitHub returns a real name, an avatar, a homepage and sometimes an email its owner chose to publish; GitLab a name and public email; Codeberg the same through Gitea; and a fediverse account returns its profile fields with the instance's own rel="me" verification already performed — the same handshake enrichFeedAuthors spends up to three fetches proving, arriving done. Three rules keep this from inventing people, which is the expensive direction: - An organisation is not a person. GitHub and Gitea serve both from one endpoint, so jekyll.github.io resolves to an account whose name is a product. The account is still stored as a link — a link is not a claim about who somebody is — but no author row is created. - A derived account is evidence, not proof, and sits below the 0.6 publishing floor until a profile answers. A 404 leaves no trace rather than half a person. - `verified` keeps meaning what the schema says. It is set when the profile links *back* at the site being enriched, which is the IndieWeb handshake in the other direction, and never merely because an account exists. safeFetch grew a headers option rather than being bypassed: a profile URL is built from a hostname read out of somebody else's feed, so it is exactly as untrusted as a page URL and must keep the private-address guard and the timeout. GITHUB_TOKEN is wired through the poller and is not optional in practice — the anonymous GitHub API allows 60 requests an hour per IP, which one batch spends, against 5,000 with a token. Measured against the live APIs while building, not assumed: GitLab's unauthenticated user search answers a reduced object (name and public_email only, the contact fields needing a token), and Mastodon bios are rendered HTML, where stripping every tag to a space turns "InfoSec." into "InfoSec ." — the test caught that before it reached anybody's page. --- apps/poller/src/index.js | 9 + packages/feed/index.js | 1 + packages/feed/src/fetch.js | 15 +- packages/feed/src/platforms.js | 522 ++++++++++++++++++ packages/feed/test/platforms.test.js | 157 ++++++ packages/ingest/src/enrich.js | 168 +++++- packages/ingest/test/enrich-platforms.test.js | 220 ++++++++ 7 files changed, 1087 insertions(+), 5 deletions(-) create mode 100644 packages/feed/src/platforms.js create mode 100644 packages/feed/test/platforms.test.js create mode 100644 packages/ingest/test/enrich-platforms.test.js diff --git a/apps/poller/src/index.js b/apps/poller/src/index.js index 0e12f75..88835d0 100644 --- a/apps/poller/src/index.js +++ b/apps/poller/src/index.js @@ -145,6 +145,14 @@ const authorVerify = env['AUTHOR_VERIFY'] === '1' || env['AUTHOR_VERIFY'] === 't // is the switch to reach for if a site ever objects to the extra fetches. const authorEnabled = env['AUTHOR_ENRICH'] !== '0' && env['AUTHOR_ENRICH'] !== 'false'; +// A GitHub credential, and the difference between the profile lookups working +// and not working at all: unauthenticated, the API allows **60 requests an hour +// per IP**, which a single batch spends. With a token it is 5,000, which is +// more than this pass can use. Optional only in the sense that the rest of the +// enrichment carries on without it -- profile resolution simply stops finding +// anything once the hour's 60 are gone, and does so quietly, as a 403. +const authorGithubToken = String(env['GITHUB_TOKEN'] ?? '').trim(); + // Accounts considered per alert pass, and how often a pass runs. Its own timer // for the same reason the card and cluster passes have one: a tick spends // minutes inside the crawl, and work queued behind that only happens if the @@ -590,6 +598,7 @@ async function enrichTick() { verify: authorVerify, recheckDays: authorRecheckDays, concurrency: authorConcurrency, + githubToken: authorGithubToken, onEvent: publishLog ? recorder.record : null, }); // Silent when nothing was due, which is the steady state once the diff --git a/packages/feed/index.js b/packages/feed/index.js index 96ceaca..2852bc1 100644 --- a/packages/feed/index.js +++ b/packages/feed/index.js @@ -78,3 +78,4 @@ export { personalEmail, splitBylines, } from './src/identity.js'; +export { hostIdentity, identityFromProfile, profileRequest } from './src/platforms.js'; diff --git a/packages/feed/src/fetch.js b/packages/feed/src/fetch.js index f2cf3ca..29fc5fb 100644 --- a/packages/feed/src/fetch.js +++ b/packages/feed/src/fetch.js @@ -102,7 +102,8 @@ export async function isPublicHost(hostname) { * keep revalidating against a stale one for ever. * * @param {string} url - * @param {{ etag?: string|null, lastModified?: string|null }} [conditional] + * @param {{ etag?: string|null, lastModified?: string|null, + * headers?: Record }} [conditional] * @returns {Promise<{ ok: boolean, status: number, contentType: string, body: string, url: string, notModified?: boolean, etag?: string|null, lastModified?: string|null, error?: string }>} */ export async function safeFetch(url, conditional = {}) { @@ -132,6 +133,18 @@ export async function safeFetch(url, conditional = {}) { if (conditional?.etag) headers['if-none-match'] = String(conditional.etag); if (conditional?.lastModified) headers['if-modified-since'] = String(conditional.lastModified); + // Caller-supplied headers, for the JSON APIs the author enrichment asks + // about a person. They go through this function rather than around it so + // an API host is still checked against the private-address guard and still + // bounded by the same timeout -- a profile URL is built from a hostname we + // read out of somebody else's feed, so it is exactly as untrusted as a + // page URL. The user-agent stays ours and cannot be overridden. + for (const [key, value] of Object.entries(conditional?.headers ?? {})) { + const name = String(key).toLowerCase(); + if (name === 'user-agent') continue; + headers[name] = String(value); + } + const res = await fetch(normalized, { headers, redirect: 'follow', diff --git a/packages/feed/src/platforms.js b/packages/feed/src/platforms.js new file mode 100644 index 0000000..863f23a --- /dev/null +++ b/packages/feed/src/platforms.js @@ -0,0 +1,522 @@ +/* + * The identity a publisher never wrote down. + * + * `identity.js` reads what a page *says*: rel="me", an h-card, a JSON-LD + * `sameAs`, a footer link. That works beautifully on the part of the small web + * that marks itself up, and it finds exactly nothing on the part that does not. + * + * The case that made this necessary is the ordinary one. `felginep.github.io` + * publishes a blog with no rel="me", no h-card, and one outbound link — to the + * Jekyll theme its author happened to use. The old pass fetched the page, + * correctly found nobody, and stamped the feed as checked. Meanwhile the + * author's GitHub account was named *in the hostname the feed was served from*, + * and the profile behind it confirms his real name. + * + * So this module reads two things nobody had to publish: + * + * - **The host.** A blog on `.github.io`, `.substack.com` or + * `medium.com/@` names its owner's account in the address. That costs + * no request at all — it is a fact about the URL we already have. + * - **The profile behind it.** Given the account, its platform will describe + * the person: GitHub returns a real name, a homepage and sometimes a public + * email; GitLab returns a name and a public email without a token, and its + * contact fields with one; a Mastodon account returns its profile fields + * *with the instance's own rel="me" verification already done for us*. + * + * **Pure by construction.** Nothing here fetches. The functions describe a + * request (`profileRequest`) and read a response (`identityFromProfile`), which + * is the same split `identity.js` keeps and the reason both can be tested + * without a network. `enrich.js` owns the sockets. + * + * **The confidence rule.** A host-derived account is evidence, not proof: + * `blog.example.com` on a shared host is not a person, and a project site can + * live under an organisation's `github.io`. So a derivation is only made where + * the platform gives each *user* their own subdomain, and the account it + * proposes is confirmed the moment the profile answers with a matching name. + * Anything unconfirmed stays below the publishing floor. + */ + +/** + * Platforms that put a user's handle in the hostname. + * + * Only the ones where a subdomain is *a person's account* rather than a site + * somebody happens to host. `.wordpress.com` and `.blogspot.com` + * are deliberately absent: they name a blog, and we already have the blog — + * deriving "a website link to the feed we are enriching" is a row that teaches + * nobody anything. + * + * `reserved` are the subdomains the platform itself uses, which are never + * people. Without it, `www.github.io` and `pages.github.com` become authors. + */ +const HOST_ACCOUNTS = [ + { + network: 'github', + host: /^([a-z0-9](?:[a-z0-9-]{0,38}))\.github\.io$/i, + profile: (handle) => `https://github.com/${handle}`, + }, + { + network: 'gitlab', + host: /^([a-z0-9][\w.-]{1,60})\.gitlab\.io$/i, + profile: (handle) => `https://gitlab.com/${handle}`, + }, + { + network: 'codeberg', + host: /^([a-z0-9][\w-]{0,38})\.codeberg\.page$/i, + profile: (handle) => `https://codeberg.org/${handle}`, + }, + { + network: 'sourcehut', + host: /^([\w.-]{1,64})\.srht\.site$/i, + profile: (handle) => `https://git.sr.ht/~${handle}`, + }, + { + network: 'substack', + host: /^([\w-]{1,60})\.substack\.com$/i, + profile: (handle) => `https://${handle}.substack.com`, + }, + { + network: 'medium', + host: /^([\w-]{1,60})\.medium\.com$/i, + profile: (handle) => `https://medium.com/@${handle}`, + }, + { + network: 'tumblr', + host: /^([\w-]{1,32})\.tumblr\.com$/i, + profile: (handle) => `https://${handle}.tumblr.com`, + }, + { + network: 'microblog', + host: /^([\w.-]{1,60})\.micro\.blog$/i, + profile: (handle) => `https://micro.blog/${handle}`, + }, + { + network: 'bearblog', + host: /^([\w-]{1,60})\.bearblog\.dev$/i, + profile: (handle) => `https://bearblog.dev/${handle}`, + }, +]; + +/** + * Path-shaped platforms, where the handle is the first segment. + * + * Same idea as above for hosts that give everybody one domain. Only applied to + * the *feed's own* address, never to an arbitrary link — `classifyLink` already + * reads links, and this is about the address the feed itself lives at. + */ +const PATH_ACCOUNTS = [ + { network: 'medium', host: /^(?:www\.)?medium\.com$/i, path: /^\/@([\w.-]{1,60})(?:\/|$)/ }, + { network: 'devto', host: /^dev\.to$/i, path: /^\/([\w-]{1,60})(?:\/|$)/ }, + { network: 'hashnode', host: /^hashnode\.com$/i, path: /^\/@([\w-]{1,60})(?:\/|$)/ }, + { network: 'microblog', host: /^(?:www\.)?micro\.blog$/i, path: /^\/([\w.-]{1,60})(?:\/|$)/ }, + { network: 'substack', host: /^(?:www\.)?substack\.com$/i, path: /^\/@([\w-]{1,60})(?:\/|$)/ }, +]; + +/** + * Subdomains that belong to the platform rather than to a person. + * + * Kept deliberately short. A false *positive* here costs an author page for a + * person who does not exist, which is the expensive mistake; a false negative + * costs one blogger a link we would have found on their page anyway. + */ +const RESERVED_HANDLES = new Set([ + 'www', + 'blog', + 'docs', + 'help', + 'api', + 'status', + 'about', + 'support', + 'mail', + 'admin', + 'pages', + 'app', + 'static', + 'assets', + 'cdn', + 'media', + 'news', + 'test', + 'demo', + 'example', +]); + +/** + * Accounts named by the addresses a feed already has. + * + * Costs no request: this is arithmetic on strings we were given. Both the feed + * URL and the site URL are read, because they disagree often — a blog on its + * own domain whose feed is proxied through Substack names its author in the + * feed URL and nowhere else. + * + * @param {...unknown} urls the feed URL, the site URL, in any order + * @returns {Array<{ network: string, url: string, handle: string, source: string, confidence: number }>} + */ +export function hostIdentity(...urls) { + /** @type {Map} */ + const found = new Map(); + + for (const raw of urls) { + let parsed; + try { + parsed = new URL(String(raw ?? '')); + } catch { + continue; + } + if (!/^https?:$/.test(parsed.protocol)) continue; + + const host = parsed.hostname.toLowerCase(); + + for (const rule of HOST_ACCOUNTS) { + const match = host.match(rule.host); + if (!match) continue; + add(found, rule.network, match[1], rule.profile); + break; + } + + for (const rule of PATH_ACCOUNTS) { + if (!rule.host.test(host)) continue; + const match = parsed.pathname.match(rule.path); + if (!match) continue; + add(found, rule.network, match[1], (handle) => + rule.network === 'medium' + ? `https://medium.com/@${handle}` + : `https://${host}/${rule.network === 'hashnode' || rule.network === 'substack' ? '@' : ''}${handle}`, + ); + break; + } + } + + return [...found.values()]; +} + +/** + * @param {Map} found + * @param {string} network + * @param {string} rawHandle + * @param {(handle: string) => string} profile + */ +function add(found, network, rawHandle, profile) { + const handle = String(rawHandle ?? '').toLowerCase(); + if (!handle || RESERVED_HANDLES.has(handle)) return; + + const url = profile(handle); + const key = `${network}:${handle}`; + if (found.has(key)) return; + + found.set(key, { + network, + url, + handle, + source: 'host-derived', + // Below the 0.6 publishing floor on purpose. The hostname is a strong hint + // and not a fact: it becomes one when `identityFromProfile` finds a profile + // that agrees, and that is what raises it. + confidence: 0.5, + }); +} + +/** + * The platforms that will describe a person if asked, and how to ask. + * + * Only APIs that answer unauthenticated, return JSON, and publish a *person* + * rather than a repository. Bluesky and LinkedIn are absent for the reason they + * are absent from the rel="me" verification list — one renders client-side and + * the other refuses robots outright. + */ +const PROFILE_APIS = { + github: (handle) => `https://api.github.com/users/${encodeURIComponent(handle)}`, + gitlab: (handle) => `https://gitlab.com/api/v4/users?username=${encodeURIComponent(handle)}`, + codeberg: (handle) => `https://codeberg.org/api/v1/users/${encodeURIComponent(handle)}`, +}; + +/** + * How to ask a platform about one of its people. + * + * Returns null for a link no API here can resolve, which is most of them — the + * caller treats that as "nothing more to learn from this one" rather than as an + * error. + * + * `token` is the caller's GitHub credential, and it matters more than it looks: + * **the unauthenticated GitHub API allows 60 requests an hour per IP**, which is + * roughly one enrichment batch. With a token it is 5,000. A pass over this + * directory without one is not slow, it is stopped. + * + * @param {{ network: string, handle?: string, url?: string }} link + * @param {{ token?: string }} [opts] + * @returns {{ url: string, network: string, headers: Record }|null} + */ +export function profileRequest(link, opts = {}) { + const network = String(link?.network ?? ''); + const handle = String(link?.handle ?? '').replace(/^[@~]/, ''); + if (!handle) return null; + + // The fediverse has no host list, so its endpoint is derived from the account + // rather than looked up: `@user@host` is served by `host`. + if (network === 'fediverse') { + const parts = handle.replace(/^@/, '').split('@'); + if (parts.length !== 2 || !parts[0] || !parts[1]) return null; + return { + network, + url: `https://${parts[1]}/api/v1/accounts/lookup?acct=${encodeURIComponent(parts[0])}`, + headers: { accept: 'application/json' }, + }; + } + + const build = PROFILE_APIS[network]; + if (!build) return null; + + /** @type {Record} */ + const headers = { accept: 'application/json' }; + + if (network === 'github') { + headers.accept = 'application/vnd.github+json'; + if (opts.token) headers.authorization = `Bearer ${opts.token}`; + } + + return { network, url: build(handle), headers }; +} + +/** + * Read a profile document into the credits and links it implies. + * + * Each platform is mapped explicitly rather than by a shared field list, + * because the one field that matters is different on each: GitHub calls the + * homepage `blog`, GitLab calls it `website_url` and publishes the email as + * `public_email`, and Mastodon does not have named fields at all — it has an + * array the user filled in themselves, of which the *verified* entries are + * worth more than anything else this module can find. + * + * `kind` is the one field the caller must not ignore. GitHub and Gitea both + * serve organisations from the same endpoint as people, so a project site on + * `someproject.github.io` resolves to an account whose "name" is a product. + * Turning that into an author row would publish a company as a person, which is + * the one mistake this directory has been careful not to make since the role + * filters went in. + * + * @param {string} network + * @param {unknown} body the parsed JSON + * @param {{ handle?: string, url?: string }} [link] the account this describes + * @returns {{ name: string, bio: string, avatar: string, kind: string, links: Array }} + */ +export function identityFromProfile(network, body, link = {}) { + const empty = { name: '', bio: '', avatar: '', kind: 'unknown', links: [] }; + if (!body || typeof body !== 'object') return empty; + + switch (network) { + case 'github': + return fromGithub(/** @type {any} */ (body)); + case 'gitlab': + // The users endpoint answers with an array, because it is a search. + return fromGitlab(Array.isArray(body) ? body[0] : body); + case 'codeberg': + return fromCodeberg(/** @type {any} */ (body)); + case 'fediverse': + return fromMastodon(/** @type {any} */ (body), link); + default: + return empty; + } +} + +/** + * GitHub. + * + * `email` is only ever populated when the account holder ticked the box that + * publishes it, so it is a contact address they chose to make public rather + * than one scraped out of commit metadata — which is a different thing with + * different rules, and is not collected here. + * + * @param {any} p + */ +function fromGithub(p) { + const links = []; + if (p.blog) links.push(profileLink('website', p.blog, 'github-profile')); + if (p.email) links.push(profileLink('email', `mailto:${p.email}`, 'github-profile')); + if (p.twitter_username) { + links.push(profileLink('twitter', `https://x.com/${p.twitter_username}`, 'github-profile')); + } + + return { + name: str(p.name), + bio: str(p.bio), + avatar: str(p.avatar_url), + // "User" or "Organization"; anything else is a shape we do not know and is + // treated as unknown rather than assumed to be a person. + kind: p.type === 'Organization' ? 'org' : p.type === 'User' ? 'user' : 'unknown', + links: links.filter(Boolean), + }; +} + +/** + * GitLab. + * + * Measured rather than assumed: the *unauthenticated* `/users?username=` search + * answers a reduced object — `name`, `public_email`, `avatar_url`, `web_url` + * and nothing else. The richer profile (`website_url`, `bio`, `twitter`, + * `linkedin`) needs a token, and `/users/:id` refuses outright without one. The + * mappings below cover both shapes, so a deployment that sets a GitLab token + * gets the contact fields and one that does not still gets a name and a public + * email, which is the part that matters. + * + * @param {any} p + */ +function fromGitlab(p) { + if (!p || typeof p !== 'object') return { name: '', bio: '', avatar: '', kind: 'unknown', links: [] }; + + const links = []; + if (p.website_url) links.push(profileLink('website', p.website_url, 'gitlab-profile')); + if (p.public_email) links.push(profileLink('email', `mailto:${p.public_email}`, 'gitlab-profile')); + if (p.twitter) links.push(profileLink('twitter', `https://x.com/${strip(p.twitter)}`, 'gitlab-profile')); + if (p.linkedin) { + links.push(profileLink('linkedin', `https://www.linkedin.com/in/${strip(p.linkedin)}`, 'gitlab-profile')); + } + + return { + name: str(p.name), + bio: str(p.bio), + avatar: str(p.avatar_url), + // GitLab groups are not returned by the user search at all, so anything + // that answers here is a user account. + kind: 'user', + links: links.filter(Boolean), + }; +} + +/** + * Codeberg, which runs Gitea and answers the same shape. + * + * @param {any} p + */ +function fromCodeberg(p) { + const links = []; + if (p.website) links.push(profileLink('website', p.website, 'codeberg-profile')); + if (p.email) links.push(profileLink('email', `mailto:${p.email}`, 'codeberg-profile')); + + return { + name: str(p.full_name), + bio: str(p.description), + avatar: str(p.avatar_url), + // Gitea marks organisations with a user_type of 1. + kind: Number(p.user_type) === 1 ? 'org' : 'user', + links: links.filter(Boolean), + }; +} + +/** + * A fediverse account, and the best source in this module. + * + * Mastodon's profile `fields` are free-form rows the account holder wrote, and + * an entry carries `verified_at` when the instance followed the link and found + * a rel="me" pointing back. That is the IndieWeb handshake **already performed + * by somebody else** — the same proof `enrichFeedAuthors` spends up to three + * fetches on, arriving for free with the profile. + * + * So a verified field is marked `verified` and nothing else here is. + * + * @param {any} p + * @param {{ url?: string }} link + */ +function fromMastodon(p, link) { + const links = []; + + for (const field of Array.isArray(p.fields) ? p.fields : []) { + // The value is HTML — Mastodon renders the link itself — so the href is + // what to read, falling back to the text for an entry typed as bare text. + const href = String(field?.value ?? '').match(/href="([^"]+)"/)?.[1] ?? String(field?.value ?? ''); + const url = href.replace(/<[^>]*>/g, '').trim(); + if (!url) continue; + + links.push({ + url, + source: 'fediverse-profile', + verified: Boolean(field?.verified_at), + // A field the instance proved is as good as evidence gets short of the + // person telling us themselves. + confidence: field?.verified_at ? 0.95 : 0.5, + }); + } + + if (p.url && !links.some((l) => l.url === p.url)) { + links.push({ url: String(p.url), source: 'fediverse-profile', verified: false, confidence: 0.6 }); + } + + return { + name: str(p.display_name), + kind: p.group ? 'org' : 'user', + // The bio is HTML on this platform and prose everywhere else. + bio: plain(p.note), + avatar: str(p.avatar_static || p.avatar), + links, + }; +} + +/** + * @param {string} network + * @param {unknown} url + * @param {string} source + */ +function profileLink(network, url, source) { + const value = str(url); + if (!value) return null; + return { + network, + // A platform field is frequently typed without a scheme ("example.com"), + // which is not a URL until it has one. + url: /^[a-z][a-z0-9+.-]*:/i.test(value) ? value : `https://${value}`, + source, + verified: false, + // Published by the person on an account we already believe is theirs, which + // is above the floor but short of a proved backlink. + confidence: 0.8, + }; +} + +/** + * HTML as the prose it was written as. + * + * Only the fediverse needs this — its bio field is rendered markup rather than + * text. Which tags become a space and which vanish is the whole job: replacing + * *every* tag with a space turns "InfoSec." into "InfoSec ." (the test + * that found this was right to), while replacing every tag with nothing joins + * two paragraphs into one word. So the block-level tags separate and the inline + * ones close up, which is what they mean. + * + * Entities are decoded too, because a bio is displayed to a reader and + * "Tom & Jerry" is not what anyone wrote. + * + * @param {unknown} html + * @returns {string} + */ +function plain(html) { + return String(html ?? '') + .replace(/<\/(?:p|div|li|h[1-6])>|/gi, ' ') + .replace(/<[^>]*>/g, '') + .replace(/&(nbsp|amp|lt|gt|quot|#39|apos);/gi, (_, name) => { + const map = { + nbsp: ' ', + amp: '&', + lt: '<', + gt: '>', + quot: '"', + '#39': "'", + apos: "'", + }; + return map[String(name).toLowerCase()] ?? _; + }) + .replace(/\s+/g, ' ') + .trim(); +} + +/** @param {unknown} v */ +function str(v) { + return v == null ? '' : String(v).trim(); +} + +/** @param {unknown} v handle fields that arrive with an @ or a full URL */ +function strip(v) { + return String(v ?? '') + .trim() + .replace(/^@/, '') + .replace(/^https?:\/\/[^/]+\/(?:in\/)?/i, '') + .replace(/\/+$/, ''); +} diff --git a/packages/feed/test/platforms.test.js b/packages/feed/test/platforms.test.js new file mode 100644 index 0000000..c6f031d --- /dev/null +++ b/packages/feed/test/platforms.test.js @@ -0,0 +1,157 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { hostIdentity, identityFromProfile, profileRequest } from '../src/platforms.js'; + +// What is tested here is the half of author enrichment that reads what nobody +// published: the account named by a hostname, and the profile behind it. The +// expensive mistake in this direction is not missing somebody, it is inventing +// them -- so most of these are about what must *not* be derived. + +test('a blog on a personal platform names its owner in the hostname', () => { + // The case this module exists for. felginep.github.io publishes no rel="me", + // no h-card and one outbound link, to its Jekyll theme -- so every other + // source finds nobody, while the account is sitting in the address. + assert.deepEqual(hostIdentity('https://felginep.github.io/feed.xml'), [ + { + network: 'github', + url: 'https://github.com/felginep', + handle: 'felginep', + source: 'host-derived', + confidence: 0.5, + }, + ]); +}); + +test('the feed URL and the site URL are both read, and agree on one account', () => { + // They disagree often -- a blog on its own domain with a feed proxied through + // a platform names its author in only one of them -- but when both name the + // same account it must not be stored twice. + const found = hostIdentity('https://jane.substack.com/feed', 'https://jane.substack.com/'); + assert.equal(found.length, 1); + assert.equal(found[0].network, 'substack'); + assert.equal(found[0].handle, 'jane'); +}); + +test("a platform's own subdomains are not people", () => { + for (const url of [ + 'https://www.github.io/', + 'https://docs.github.io/', + 'https://blog.substack.com/', + 'https://api.micro.blog/', + ]) { + assert.deepEqual(hostIdentity(url), [], url); + } +}); + +test('a host that is nobody in particular derives nothing', () => { + // The overwhelming majority of the directory. Deriving a "website" link back + // to the feed we are already enriching would be a row that teaches nobody + // anything, so a plain domain yields nothing at all. + assert.deepEqual(hostIdentity('https://kevquirk.com/feed'), []); + assert.deepEqual(hostIdentity('https://example.wordpress.com/feed'), []); + assert.deepEqual(hostIdentity('not a url'), []); + assert.deepEqual(hostIdentity('ftp://example.github.io/'), []); +}); + +test('a derived account stays below the publishing floor until something confirms it', () => { + // 0.6 is what /api/authors publishes at. A hostname is a strong hint and not + // a fact: it is the profile answering that turns it into one. + const [account] = hostIdentity('https://felginep.github.io/'); + assert.ok(account.confidence < 0.6, 'a bare derivation must not be publishable on its own'); +}); + +test('a GitHub profile is read for the fields a person filled in', () => { + const profile = identityFromProfile('github', { + type: 'User', + name: 'Pierre Felgines', + bio: 'iOS developer', + avatar_url: 'https://avatars.example/u/1', + blog: 'felginep.github.io', + email: 'pierre@example.com', + twitter_username: 'pfelgines', + }); + + assert.equal(profile.name, 'Pierre Felgines'); + assert.equal(profile.kind, 'user'); + assert.deepEqual( + profile.links.map((l) => [l.network, l.url]), + [ + // A profile field typed without a scheme is not a URL until it has one. + ['website', 'https://felginep.github.io'], + ['email', 'mailto:pierre@example.com'], + ['twitter', 'https://x.com/pfelgines'], + ], + ); +}); + +test('an organisation is not a person', () => { + // jekyll.github.io resolves to an account whose "name" is a product. Turning + // that into an author row would publish a project as a human being. + const profile = identityFromProfile('github', { type: 'Organization', name: 'Jekyll' }); + assert.equal(profile.kind, 'org'); +}); + +test("a fediverse profile carries the instance's own rel=me verification", () => { + // The best source in the module: the instance already followed the link and + // found a backlink, so the IndieWeb handshake arrives done. + const profile = identityFromProfile('fediverse', { + display_name: 'Kev Quirk', + note: '

I work in InfoSec.

', + url: 'https://fosstodon.org/@kev', + avatar_static: 'https://cdn.example/a.png', + fields: [ + { + name: 'Blog', + value: 'kevquirk.com', + verified_at: '2022-10-22T22:21:08.251+00:00', + }, + { name: 'Unproven', value: 'example.org', verified_at: null }, + ], + }); + + assert.equal(profile.name, 'Kev Quirk'); + // The bio is HTML on this platform and prose everywhere else. + assert.equal(profile.bio, 'I work in InfoSec.'); + + const verified = profile.links.find((l) => l.url === 'https://kevquirk.com'); + assert.equal(verified.verified, true); + assert.ok(verified.confidence > 0.9); + + const unproven = profile.links.find((l) => l.url === 'https://example.org'); + assert.equal(unproven.verified, false, 'an unverified field is stored, but not as proof'); +}); + +test('a request is described for the platforms that answer, and refused for the rest', () => { + assert.equal( + profileRequest({ network: 'github', handle: 'felginep' }).url, + 'https://api.github.com/users/felginep', + ); + // The fediverse has no host list, so the endpoint comes out of the handle. + assert.equal( + profileRequest({ network: 'fediverse', handle: '@kev@fosstodon.org' }).url, + 'https://fosstodon.org/api/v1/accounts/lookup?acct=kev', + ); + assert.equal(profileRequest({ network: 'linkedin', handle: 'someone' }), null); + assert.equal(profileRequest({ network: 'github', handle: '' }), null); + assert.equal(profileRequest({ network: 'fediverse', handle: 'no-host' }), null); +}); + +test('a GitHub token is sent when there is one, because 60 an hour is not a budget', () => { + const anonymous = profileRequest({ network: 'github', handle: 'x' }); + assert.equal(anonymous.headers.authorization, undefined); + + const authed = profileRequest({ network: 'github', handle: 'x' }, { token: 'ghp_test' }); + assert.equal(authed.headers.authorization, 'Bearer ghp_test'); +}); + +test('a malformed profile body is nothing, not a throw', () => { + for (const body of [null, undefined, 'not json', 42]) { + const profile = identityFromProfile('github', body); + assert.deepEqual(profile.links, []); + assert.equal(profile.name, ''); + } + // GitLab answers its user search with an array, and an empty one when the + // handle matched nobody. + assert.equal(identityFromProfile('gitlab', []).name, ''); +}); diff --git a/packages/ingest/src/enrich.js b/packages/ingest/src/enrich.js index 6f31fb3..38fbcf0 100644 --- a/packages/ingest/src/enrich.js +++ b/packages/ingest/src/enrich.js @@ -1,12 +1,18 @@ import { BIO_HOSTS, + classifyLink, + credit, + hostIdentity, identityFromHtml, + identityFromProfile, identityKey, linksBackTo, linksFromBioPage, + looksLikePersonName, mergeCredits, normalizeIdentityUrl, normalizeName, + profileRequest, resolveFeed, safeFetch, uniqueSlug, @@ -46,6 +52,16 @@ const MAX_BIO_PAGES = 1; /** Cap on rel="me" backlinks verified per author, since each is a fetch. */ const MAX_VERIFY = 3; +/** + * Cap on platform profiles resolved per feed. + * + * Each is one JSON request, and the yield falls off a cliff after the first: + * the account named by the feed's own hostname is the one that describes the + * publisher, and the third-best account on the page is usually a colleague, a + * project, or the theme's author. + */ +const MAX_PROFILES = 3; + /** * Networks whose profile pages publish a rel="me" link back to the author's * own site, which is the only cheap proof an account really is theirs. @@ -230,9 +246,11 @@ export async function prepareCredits(db, feed, credits, links = []) { * * @param {Client} db * @param {{ id: string, slug?: string, feed_url: string, site_url?: string|null, title?: string }} feed - * @param {{ verify?: boolean, fetch?: typeof safeFetch, resolve?: typeof resolveFeed }} [opts] + * @param {{ verify?: boolean, fetch?: typeof safeFetch, resolve?: typeof resolveFeed, + * githubToken?: string }} [opts] * `fetch` and `resolve` are injected by the tests so the pass can be - * exercised end to end without a network + * exercised end to end without a network; `githubToken` raises the GitHub + * API's 60-per-hour anonymous ceiling to 5,000 * @returns {Promise<{ people: number, links: number, pages: number, verified: number }>} */ export async function enrichFeedAuthors(db, feed, opts = {}) { @@ -321,9 +339,102 @@ export async function enrichFeedAuthors(db, feed, opts = {}) { collect(linksFromBioPage(page.body, page.url || link.url)); } + // 4. The accounts the addresses themselves name, and the profiles behind + // them. + // + // This is the half that reaches the publishers who marked nothing up, which + // is most of them. The case it was built for: felginep.github.io publishes a + // blog with no rel="me", no h-card and one outbound link -- to the Jekyll + // theme its author used. Everything above finds nobody. But the author's + // GitHub account is named in the hostname the feed is served from, and one + // request to that account returns "Pierre Felgines" and an avatar. + // + // Deriving the account is string arithmetic on URLs already in hand, so a + // feed on a platform we do not recognise costs nothing extra at all. + collect(hostIdentity(feed.feed_url, siteUrl)); + + let profiles = 0; + + for (const account of [...links.values()]) { + if (profiles >= MAX_PROFILES) break; + + const request = profileRequest(account, { token: opts.githubToken }); + if (!request) continue; + profiles += 1; + + // A 404 here is the useful negative: the hostname proposed an account that + // does not exist, so the derivation was wrong and nothing is stored for it. + const page = await fetchPage(request.url, { headers: request.headers }).catch(() => null); + if (!page?.ok) continue; + pages += 1; + + let body; + try { + body = JSON.parse(page.body); + } catch { + continue; + } + + const profile = identityFromProfile(request.network, body, account); + + for (const found of profile.links) { + // A platform's own "website" field is a homepage, which matches none of + // the profile shapes -- classifyLink is right to return null for it, and + // it is still the single most useful link a profile carries. + const classified = classifyLink(found.url) ?? asWebsite(found.url); + if (!classified) continue; + + collect([ + { + ...classified, + source: found.source, + // Only the fediverse hands us a verification, and it hands us a real + // one: the instance already followed the link and found a rel="me" + // pointing back. That is the same handshake the verify pass below + // spends fetches on, arriving for free. + verified: Boolean(found.verified), + }, + ]); + + // The account points back at the site we are enriching. That is the + // IndieWeb handshake in the other direction and is proof the account + // belongs to this publisher, so the account link earns `verified` -- the + // column means "the destination links back", and here it does. + if (classified.network === 'website' && siteUrl && sameSite(classified.url, siteUrl)) { + account.verified = true; + verified += 1; + } + } + + // An organisation is not a person. GitHub and Gitea serve both from one + // endpoint, so a project site on `someproject.github.io` resolves to an + // account whose name is a product -- publishing that as an author is the + // mistake identity.js's role filters exist to prevent, arriving by a + // different door. + if (profile.kind !== 'user') continue; + if (!looksLikePersonName(profile.name)) continue; + + credits.push( + credit({ + name: profile.name, + url: account.url, + avatar: profile.avatar, + bio: profile.bio, + role: 'author', + source: `${request.network}-profile`, + // Above the publishing floor, and deliberately only just. The blog is + // served from this account's own pages and the account is a person + // whose name reads as one -- but nobody has said in so many words that + // they wrote it, which is what the stronger sources have. A confirmed + // backlink lifts it to where a rel="me" pair sits. + confidence: account.verified ? 0.9 : account.source === 'host-derived' ? 0.7 : 0.65, + }), + ); + } + const merged = mergeCredits(credits.filter(Boolean)); - // 4. The IndieWeb handshake, for the accounts that answer it. Only worth + // 5. The IndieWeb handshake, for the accounts that answer it. Only worth // spending requests on when there is a site to link back to, and only when // the caller asked — the pass over 52,000 feeds does not, a re-check of one // feed does. @@ -407,7 +518,10 @@ const PER_HOST = 3; * @param {Client} db * @param {number} [batchSize] * @param {{ verify?: boolean, recheckDays?: number, onEvent?: ((event: object) => void)|null, - * concurrency?: number, fetch?: typeof safeFetch, resolve?: typeof resolveFeed }} [opts] + * concurrency?: number, fetch?: typeof safeFetch, resolve?: typeof resolveFeed, + * githubToken?: string }} [opts] `githubToken` is not optional in practice: + * the unauthenticated GitHub API allows 60 requests an hour per IP, which one + * batch exhausts, so a pass without it resolves almost no profiles * @returns {Promise<{ feeds: number, people: number, links: number, hosts: number }>} */ export async function enrichDue(db, batchSize = 10, opts = {}) { @@ -442,6 +556,7 @@ export async function enrichDue(db, batchSize = 10, opts = {}) { verify: opts.verify, fetch: opts.fetch, resolve: opts.resolve, + githubToken: opts.githubToken, }); feeds += 1; people += result.people; @@ -581,3 +696,48 @@ function report(onEvent, feed, started, outcome) { // A broken listener loses its line and nothing else. } } + +/** + * A bare homepage as a link row. + * + * `classifyLink` returns null for anything that is not a recognised profile + * shape, which is correct for a link found loose in a page -- it would + * otherwise file every outbound link as somebody's website. A URL taken out of + * a *profile field* is different: the person put it there to say "this is my + * site", so it is the one place the fallback is warranted. + * + * @param {unknown} value + * @returns {{ network: string, url: string, handle: string }|null} + */ +function asWebsite(value) { + const raw = normalizeIdentityUrl(value); + if (!raw) return null; + + try { + return { network: 'website', url: raw, handle: new URL(raw).hostname }; + } catch { + return null; + } +} + +/** + * Do two URLs name the same site? + * + * Compared on host alone, with `www.` folded away: a profile that links + * `https://example.com` and a feed that declares `https://www.example.com/blog/` + * are the same publisher, and requiring the paths to match would refuse every + * real backlink. + * + * @param {unknown} a + * @param {unknown} b + * @returns {boolean} + */ +function sameSite(a, b) { + try { + const host = (u) => new URL(String(u)).hostname.toLowerCase().replace(/^www\./, ''); + const left = host(a); + return Boolean(left) && left === host(b); + } catch { + return false; + } +} diff --git a/packages/ingest/test/enrich-platforms.test.js b/packages/ingest/test/enrich-platforms.test.js new file mode 100644 index 0000000..795aae4 --- /dev/null +++ b/packages/ingest/test/enrich-platforms.test.js @@ -0,0 +1,220 @@ +import assert from 'node:assert/strict'; +import { test, before, after } from 'node:test'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { connect, migrate, q, authors as a } from '@rssamplifier/db'; + +import { enrichFeedAuthors } from '../src/enrich.js'; + +// The publisher who marked nothing up. +// +// Everything in enrich.test.js is about reading what a page says. This file is +// about the case where the page says nothing at all -- which is most of the +// directory, and was 100% of what the pass returned empty-handed on. The proof +// case is a real one: felginep.github.io publishes a blog with no rel="me", no +// h-card and exactly one outbound link, to the Jekyll theme its author used. + +let dir; +let db; + +before(async () => { + dir = await mkdtemp(join(tmpdir(), 'rssamp-platforms-')); + db = connect({ url: `file:${join(dir, 'test.db')}` }); + await migrate(db); +}); + +after(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +async function seedFeed(feed) { + const row = await q.insertFeed(db, { + slug: feed.slug, + feed_url: feed.feedUrl, + site_url: feed.siteUrl ?? null, + title: feed.title ?? 'A Blog', + categories: [], + kind: 'blog', + status: 'active', + }); + return { ...row, feed_url: feed.feedUrl, site_url: feed.siteUrl ?? null }; +} + +/** + * A fetcher answering HTML pages and JSON APIs from one map. + * + * The content type follows the URL rather than the caller, because that is what + * distinguishes the two paths under test: a page is parsed as markup and a + * profile is parsed as JSON, and a stub that returned `text/html` for an API + * would let a broken content-type check pass. + */ +function fakeFetch(responses) { + const asked = []; + const fetcher = async (url) => { + asked.push(url); + const body = responses[url]; + if (body == null) return { ok: false, status: 404, contentType: '', body: '', url }; + const json = typeof body !== 'string'; + return { + ok: true, + status: 200, + contentType: json ? 'application/json' : 'text/html', + body: json ? JSON.stringify(body) : body, + url, + }; + }; + fetcher.asked = asked; + return fetcher; +} + +const fakeResolve = (siteUrl) => async () => ({ + ok: true, + feed: { title: 'A Blog', siteUrl, credits: [], items: [] }, +}); + +/** A page with a byline nowhere and one link, to somebody else's theme. */ +const BARE_PAGE = ` +

Posts

+ theme +`; + +test('a blog that names nobody still finds its author, through the hostname', async () => { + const feed = await seedFeed({ + slug: 'felginep-github-io', + feedUrl: 'https://felginep.github.io/feed.xml', + siteUrl: 'https://felginep.github.io/', + }); + + const fetcher = fakeFetch({ + 'https://felginep.github.io/': BARE_PAGE, + 'https://api.github.com/users/felginep': { + type: 'User', + name: 'Pierre Felgines', + avatar_url: 'https://avatars.example/u/1', + }, + }); + + const result = await enrichFeedAuthors(db, feed, { + fetch: fetcher, + resolve: fakeResolve('https://felginep.github.io/'), + }); + + assert.equal(result.people, 1, 'the page named nobody, so this can only have come from the profile'); + + const [author] = await a.authorsForFeed(db, feed.id); + assert.equal(author.name, 'Pierre Felgines'); + assert.ok( + author.confidence >= 0.6, + `must clear the publishing floor, got ${author.confidence}`, + ); + + // And the account itself is stored, which is a contact surface in its own right. + const links = await a.linksForFeed(db, feed.id); + assert.ok( + links.some((l) => l.network === 'github' && l.url === 'https://github.com/felginep'), + 'the derived account is stored against the feed', + ); +}); + +test('the profile is asked about exactly once, and only when the host names an account', async () => { + const feed = await seedFeed({ + slug: 'plain-domain', + feedUrl: 'https://kevquirk.com/feed', + siteUrl: 'https://kevquirk.com/', + }); + + const fetcher = fakeFetch({ 'https://kevquirk.com/': BARE_PAGE }); + await enrichFeedAuthors(db, feed, { + fetch: fetcher, + resolve: fakeResolve('https://kevquirk.com/'), + }); + + assert.equal( + fetcher.asked.filter((u) => u.includes('api.github.com')).length, + 0, + 'a plain domain names no account, so no API request may be spent on it', + ); +}); + +test('an organisation behind a project site does not become a person', async () => { + // jekyll.github.io is a project, and its account has a name that reads like + // one. Publishing it as an author would put a piece of software on a page + // that says these are people. + const feed = await seedFeed({ + slug: 'jekyll-github-io', + feedUrl: 'https://jekyll.github.io/feed.xml', + siteUrl: 'https://jekyll.github.io/', + }); + + const fetcher = fakeFetch({ + 'https://jekyll.github.io/': BARE_PAGE, + 'https://api.github.com/users/jekyll': { type: 'Organization', name: 'Jekyll' }, + }); + + const result = await enrichFeedAuthors(db, feed, { + fetch: fetcher, + resolve: fakeResolve('https://jekyll.github.io/'), + }); + + assert.equal(result.people, 0); + assert.deepEqual(await a.authorsForFeed(db, feed.id), [], 'no fictional person'); + + // The account is still worth storing: it is where the feed comes from, and a + // link is not a claim about who anybody is. + const links = await a.linksForFeed(db, feed.id); + assert.ok(links.some((l) => l.url === 'https://github.com/jekyll')); +}); + +test('a profile that links back to the site proves the account, and says so', async () => { + // The IndieWeb handshake in the other direction: the account points at the + // blog, which is what `verified` means in the schema. + const feed = await seedFeed({ + slug: 'backlinked', + feedUrl: 'https://ann.github.io/feed.xml', + siteUrl: 'https://ann.github.io/', + }); + + const fetcher = fakeFetch({ + 'https://ann.github.io/': BARE_PAGE, + 'https://api.github.com/users/ann': { + type: 'User', + name: 'Ann Example', + blog: 'https://ann.github.io', + }, + }); + + const result = await enrichFeedAuthors(db, feed, { + fetch: fetcher, + resolve: fakeResolve('https://ann.github.io/'), + }); + + assert.equal(result.verified, 1, 'the backlink is counted as the proof it is'); + + const [author] = await a.authorsForFeed(db, feed.id); + assert.ok( + author.confidence >= 0.9, + `a proved account outranks a derived one, got ${author.confidence}`, + ); +}); + +test('an account that does not exist teaches us nothing and costs nothing', async () => { + // The useful negative. A hostname can propose an account that was deleted or + // never existed; a 404 must leave no trace rather than half a person. + const feed = await seedFeed({ + slug: 'ghost-account', + feedUrl: 'https://nobodyhome.github.io/feed.xml', + siteUrl: 'https://nobodyhome.github.io/', + }); + + const fetcher = fakeFetch({ 'https://nobodyhome.github.io/': BARE_PAGE }); + + const result = await enrichFeedAuthors(db, feed, { + fetch: fetcher, + resolve: fakeResolve('https://nobodyhome.github.io/'), + }); + + assert.equal(result.people, 0); + assert.deepEqual(await a.authorsForFeed(db, feed.id), []); +}); From 0665a0f20addeba648cb701b44ea22e8daeb47c8 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Wed, 19 Aug 2026 08:12:35 +0000 Subject: [PATCH 2/5] Read the file whose only job is to say who made this MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every source the enrichment had was markup carrying identity as a side effect: a link with a rel attribute, a microformat class, a byline in a feed document. humans.txt is the one convention written to answer the question directly, and the pass was walking past it. Parsed rather than scraped, because the format inverts everything else: the key is the role and the value is the person ("Chef: Jane Doe"), and the contact lines that follow belong to the name above them. A parser that collected links globally would hand Jane's Mastodon to Bob, which is worse than finding nothing — it is a wrong contact address published as a right one. Two sections of the file are not this blog's authors. /* THANKS */ credits other people's work, so reading it as authorship attributes a blog to whoever its author admires; /* SITE */ describes the build. Both are skipped, and the role filters still apply, so "Developer: the web team" names nobody. Fetched only when the ordinary pages named nobody. Most sites do not publish one, so asking every site costs a request per feed across the directory to help a minority; asking after the pages come back empty spends it exactly where it decides between an author and no author. It is also read as text, not HTML — plenty of servers answer every path with their 404 page, and parsing that would turn a stylesheet reference into somebody's website. /now and /uses join the page list for the same reason humans.txt earns a request: they are conventions of exactly the population this directory indexes, written in the first person, and a blog that has one often has no /about. One fix found while testing: a "Site:" line is the most useful in the file and classifyLink returns null for it, because it rightly refuses to file arbitrary links as somebody's website. Here the key has already said that is what it is, so the fallback is scoped to the keys that said so — Standards: and Language: still produce nothing. --- packages/feed/index.js | 1 + packages/feed/src/identity.js | 217 ++++++++++++++++++++++++++++++ packages/feed/test/humans.test.js | 108 +++++++++++++++ packages/ingest/src/enrich.js | 62 ++++++++- 4 files changed, 386 insertions(+), 2 deletions(-) create mode 100644 packages/feed/test/humans.test.js diff --git a/packages/feed/index.js b/packages/feed/index.js index 2852bc1..419898e 100644 --- a/packages/feed/index.js +++ b/packages/feed/index.js @@ -67,6 +67,7 @@ export { feedContacts, feedCredits, identityFromHtml, + identityFromHumansTxt, identityKey, isRoleEmail, linksBackTo, diff --git a/packages/feed/src/identity.js b/packages/feed/src/identity.js index c8811c7..9b3f8c4 100644 --- a/packages/feed/src/identity.js +++ b/packages/feed/src/identity.js @@ -1035,6 +1035,223 @@ export function identityFromHtml(html, baseUrl = '') { }; } +/** + * `humans.txt`, the file whose entire purpose is to name the people. + * + * Worth a request precisely because of what it is. Every other source here is + * markup that happens to carry identity as a side effect — a link with a `rel` + * attribute, a microformat class, a byline in a feed. `humans.txt` is a + * convention with one job: the author writing it was answering the question + * "who made this", which is the question being asked. + * + * The format is loose and its shape is inverted from everything else. The + * *key* is the role and the *value* is the person: + * + * /* TEAM *\/ + * Chef: Jane Doe + * Site: https://jane.example + * Mastodon: @jane@example.social + * + * So a person is a block: a naming line, then the contact lines that follow it + * until the next person or the next section. Attaching links to the nearest + * preceding name is the whole parsing job, and getting it wrong on a two-person + * file would give one of them the other's accounts. + * + * Only `/* TEAM *\/` and its unlabelled equivalent are read. `/* THANKS *\/` + * exists to credit other people's work — libraries, inspirations, a designer at + * another company — and treating those as this feed's authors would attribute a + * blog to whoever its author admires. + * + * @param {string} text the file, as served + * @param {string} baseUrl for resolving relative links + * @returns {{ credits: Credit[], profiles: Array<{ network: string, url: string, handle: string, source: string }> }} + */ +export function identityFromHumansTxt(text, baseUrl = '') { + const empty = { credits: [], profiles: [] }; + if (typeof text !== 'string' || !text.trim()) return empty; + + // A server that answers every path with its 404 page is common enough that + // this has to be checked: HTML here is not a humans.txt, it is a miss. + if (/^\s*<(?:!doctype|html)/i.test(text)) return empty; + + /** Keys whose value is a person's name. */ + const NAMES = new Set([ + 'name', + 'chef', + 'developer', + 'developers', + 'designer', + 'author', + 'owner', + 'maintainer', + 'engineer', + 'writer', + 'creator', + 'programmer', + ]); + + /** Keys whose value is somewhere to reach that person. */ + const LINKS = new Set([ + 'site', + 'website', + 'url', + 'homepage', + 'blog', + 'twitter', + 'x', + 'mastodon', + 'fediverse', + 'github', + 'gitlab', + 'codeberg', + 'linkedin', + 'instagram', + 'bluesky', + 'contact', + 'email', + 'e-mail', + 'mail', + ]); + + const credits = []; + const profiles = []; + /** @type {Credit|null} the person the following contact lines belong to */ + let current = null; + let inTeam = true; + + for (const rawLine of text.split(/\r?\n/)) { + const line = rawLine.trim(); + + // A section header both switches context and ends the current person. + const section = line.match(/^\/\*+\s*(.+?)\s*\*+\//); + if (section) { + inTeam = /team|staff|people|author|owner/i.test(section[1]); + current = null; + continue; + } + + if (!line) { + current = null; + continue; + } + if (!inTeam) continue; + + const pair = line.match(/^([A-Za-z][\w -]{0,24})\s*:\s*(.+)$/); + if (!pair) continue; + + const key = pair[1].trim().toLowerCase(); + const value = pair[2].trim(); + + if (NAMES.has(key)) { + const name = cleanName(value); + // The role filters apply here exactly as they do to a byline: "Developer: + // the web team" names nobody, and a file that says so must not produce a + // person called "the web team". + current = looksLikePersonName(name) + ? credit({ name, role: 'author', source: 'humans-txt', confidence: 0.7, base: baseUrl }) + : null; + if (current) credits.push(current); + continue; + } + + if (!LINKS.has(key)) continue; + + // A bare handle is not a URL, and which platform it belongs to is exactly + // what the key just said. + const expanded = expandHandle(key, value); + + // `Site: https://jane.example` is the most useful line in the file and + // matches no profile shape, so classifyLink returns null for it — rightly, + // since it refuses to file arbitrary links as somebody's website. Here the + // key has already said that is what this is, so the fallback is safe and is + // limited to the keys that said it. + const link = classifyLink(expanded, baseUrl) ?? homepage(expanded, key); + if (!link) continue; + + profiles.push({ ...link, source: 'humans-txt' }); + + // Attach to the person this block is about, so a two-person file does not + // hand one of them the other's accounts. A link before any name belongs to + // the site, which is what `feed_links` is for, and the caller stores it + // there. + if (!current) continue; + if (link.network === 'email' && !current.email) current.email = link.handle; + if (link.network === 'website' && !current.url) current.url = link.url; + } + + return { credits, profiles }; +} + +/** + * A homepage line from a humans.txt, as a link row. + * + * Only for the keys that name a site. Everything else that fails to classify is + * genuinely unrecognised and is dropped, which is what keeps a `Standards:` or + * `Language:` line from becoming a link. + * + * @param {string} url already expanded to an absolute URL + * @param {string} key the humans.txt key it came from + * @returns {{ network: string, url: string, handle: string }|null} + */ +function homepage(url, key) { + if (!['site', 'website', 'url', 'homepage', 'blog'].includes(key)) return null; + + const normalized = normalizeIdentityUrl(url); + if (!normalized) return null; + + try { + return { network: 'website', url: normalized, handle: new URL(normalized).hostname }; + } catch { + return null; + } +} + +/** + * A handle written the way people write them in a humans.txt, as a URL. + * + * `Twitter: @jane` is the common form and is not a link until it is made one. + * A value that is already a URL is left alone. + * + * @param {string} key the platform, which the key names + * @param {string} value + * @returns {string} + */ +function expandHandle(key, value) { + const raw = value.trim(); + if (/^(?:https?:|mailto:|xmpp:|nostr:)/i.test(raw)) return raw; + if (/^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(raw)) return `mailto:${raw}`; + + const handle = raw.replace(/^@/, ''); + if (!handle) return ''; + + switch (key) { + case 'twitter': + case 'x': + return `https://x.com/${handle}`; + case 'github': + return `https://github.com/${handle}`; + case 'gitlab': + return `https://gitlab.com/${handle}`; + case 'codeberg': + return `https://codeberg.org/${handle}`; + case 'linkedin': + return `https://www.linkedin.com/in/${handle}`; + case 'instagram': + return `https://instagram.com/${handle}`; + case 'bluesky': + return `https://bsky.app/profile/${handle}`; + case 'mastodon': + case 'fediverse': { + // `@jane@example.social` is the only form that names its own host. + const parts = handle.split('@'); + return parts.length === 2 ? `https://${parts[1]}/@${parts[0]}` : ''; + } + default: + // site/url/homepage/blog, written without a scheme. + return /^[\w.-]+\.[a-z]{2,}(?:\/|$)/i.test(handle) ? `https://${handle}` : ''; + } +} + /** * The links on a bio page, which all belong to whoever owns the page. * diff --git a/packages/feed/test/humans.test.js b/packages/feed/test/humans.test.js new file mode 100644 index 0000000..da2dbe5 --- /dev/null +++ b/packages/feed/test/humans.test.js @@ -0,0 +1,108 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { identityFromHumansTxt } from '../src/identity.js'; + +// humans.txt is the one source here written to answer the question we are +// asking, so the parsing has to be right about two things: whose links are +// whose, and which section of the file is talking about this site's people +// rather than about somebody else's work. + +const TEAM = `/* TEAM */ +Chef: Jane Doe +Site: https://jane.example +Mastodon: @jane@example.social +Contact: jane@example.com + +Developer: Bob Smith +Twitter: @bobsmith + +/* THANKS */ +Inspiration: Famous Person +Site: https://famous.example + +/* SITE */ +Standards: HTML5 +Language: English +`; + +test('each person keeps their own accounts', () => { + // The failure this guards against is quiet and wrong rather than empty: a + // parser that collects links globally hands Jane's Mastodon to Bob. + const { credits, profiles } = identityFromHumansTxt(TEAM, 'https://blog.example/'); + + assert.deepEqual( + credits.map((c) => c.name), + ['Jane Doe', 'Bob Smith'], + ); + + const jane = credits[0]; + assert.equal(jane.email, 'jane@example.com'); + assert.equal(jane.url, 'https://jane.example/'); + assert.equal(credits[1].email, '', 'Bob published no address and must not inherit one'); + + assert.deepEqual( + profiles.map((p) => p.network).sort(), + ['email', 'fediverse', 'twitter', 'website'], + ); + assert.ok(profiles.every((p) => p.source === 'humans-txt')); +}); + +test('the THANKS section is not this blog\'s authors', () => { + // It exists to credit other people's work — a library, an inspiration, a + // designer at another company. Reading it as authorship would attribute a + // blog to whoever its author admires. + const { credits } = identityFromHumansTxt(TEAM, 'https://blog.example/'); + assert.ok(!credits.some((c) => c.name === 'Famous Person')); +}); + +test('lines that describe the site are not links', () => { + const { profiles } = identityFromHumansTxt(TEAM, 'https://blog.example/'); + assert.ok(!profiles.some((p) => /HTML5|English/i.test(p.url))); +}); + +test('a role is still not a person here', () => { + // The same rule that governs a byline. A file can say "Developer: the web + // team", and that names nobody. + const { credits } = identityFromHumansTxt( + '/* TEAM */\nDeveloper: the web team\nName: Editor\nContact: info@example.com', + 'https://blog.example/', + ); + assert.deepEqual(credits, []); +}); + +test('a bare handle becomes the account the key says it is', () => { + const { profiles } = identityFromHumansTxt( + '/* TEAM */\nName: Ann Example\nGithub: annex\nBluesky: ann.example\nLinkedin: ann-example', + 'https://blog.example/', + ); + + assert.deepEqual( + profiles.map((p) => p.url), + [ + 'https://github.com/annex', + 'https://bsky.app/profile/ann.example', + 'https://www.linkedin.com/in/ann-example', + ], + ); +}); + +test('a 404 page dressed as a text file is not a humans.txt', () => { + // Plenty of servers answer every path with their HTML 404. Parsing that would + // turn a stylesheet reference into somebody's website. + assert.deepEqual(identityFromHumansTxt('Not found'), { + credits: [], + profiles: [], + }); + assert.deepEqual(identityFromHumansTxt(''), { credits: [], profiles: [] }); + assert.deepEqual(identityFromHumansTxt(null), { credits: [], profiles: [] }); +}); + +test('a file with no sections at all is read as a team', () => { + // The convention is loose and plenty of files skip the header entirely. + const { credits } = identityFromHumansTxt('Name: Ann Example\nSite: https://ann.example'); + assert.deepEqual( + credits.map((c) => c.name), + ['Ann Example'], + ); +}); diff --git a/packages/ingest/src/enrich.js b/packages/ingest/src/enrich.js index 38fbcf0..e9dd4d7 100644 --- a/packages/ingest/src/enrich.js +++ b/packages/ingest/src/enrich.js @@ -4,6 +4,7 @@ import { credit, hostIdentity, identityFromHtml, + identityFromHumansTxt, identityFromProfile, identityKey, linksBackTo, @@ -40,8 +41,36 @@ import { authors as a } from '@rssamplifier/db'; * @typedef {import('@libsql/client').Client} Client */ -/** Pages tried on a site, in order, until one names somebody. */ -const IDENTITY_PATHS = ['/', '/about', '/about/', '/about-me', '/contact', '/colophon']; +/** + * Pages tried on a site, in order, until one names somebody. + * + * `/now` and `/uses` are here because this is the small web: both are + * conventions of exactly the population this directory indexes, both are + * written in the first person, and a blog that has one usually has no `/about` + * — which is why they sit after the pages that are more likely to exist rather + * than instead of them. `MAX_PAGES` still bounds the whole list. + */ +const IDENTITY_PATHS = [ + '/', + '/about', + '/about/', + '/about-me', + '/contact', + '/colophon', + '/now', + '/uses', +]; + +/** + * The file whose only job is to name the people. + * + * Tried last and only when nothing else named anybody, because it is a request + * spent on a file most sites do not have. When a site does have one it is the + * best source on it: every other place identity turns up is markup that carries + * it as a side effect, while somebody writing this was answering the question + * directly. + */ +const HUMANS_TXT = '/humans.txt'; /** Cap on the pages fetched per feed, whatever the list above allows. */ const MAX_PAGES = 3; @@ -320,6 +349,35 @@ export async function enrichFeedAuthors(db, feed, opts = {}) { } } + // 2b. humans.txt, when the pages named nobody. + // + // Conditional on purpose. This is a file most sites do not publish, so asking + // every site for it would spend one request per feed across the directory to + // help the minority that has one. Asking only after the ordinary pages have + // come back empty spends it exactly where it is the difference between an + // author and no author. + // + // Not parsed as HTML: it is a text file, and a server that answers every path + // with its 404 page would otherwise turn that page into a person. + if (siteUrl && credits.length === 0) { + let target; + try { + target = new URL(HUMANS_TXT, siteUrl).toString(); + } catch { + target = ''; + } + + if (target) { + const page = await fetchPage(target).catch(() => null); + if (page?.ok) { + pages += 1; + const found = identityFromHumansTxt(page.body, page.url || target); + collect(found.profiles); + credits.push(...found.credits); + } + } + } + // 3. One hop through a links page, which is the only kind of page where // every outbound link is known to belong to the same person. let hops = 0; From b836e7c386484351860472cbbf5d067a0533a8de Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Wed, 19 Aug 2026 08:18:24 +0000 Subject: [PATCH 3/5] Buy a search for the people who left no trail, on a budget that cannot be reset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The free sources read what a publisher put where we could find it. Some people put nothing anywhere, and for them a search engine is the only remaining route. It is also the one that can spend money and the one that can invent a person, so almost all of this is what it refuses to do. **It cannot run over the directory.** The credits come from CrawlProof's 25,000 a month, already shared with CrawlProof's own outreach runner, against 369,056 feeds — one query each would be fifteen times the monthly allowance. So it takes a key, a non-zero budget and an explicit switch, and is off without all three. **The budget survives a restart, because it is written down.** A counter in the process would be reset by every deploy, which is not a budget but a rate limit with a hole in it. `author_searches` is the ledger and the authority, and it is counted from the provider's own cycle — this account resets on the 13th, not the 1st, so a calendar month would let the allowance be spent twice across a reset. The ledger is also the audit trail: which people we spent money looking for, and whether it found anything, is how the gate gets tuned rather than guessed at. **It cannot invent a person.** Every query is scoped to the author's own domain as well as to one network, because the name is the ambiguous part — "Jane Doe" site:linkedin.com/in returns every Jane Doe, and the blog she writes is what distinguishes her from them. Results are filtered through classifyLink, so a company page, a job posting and an article about somebody are all discarded where a profile is kept. Nothing found this way is ever marked `verified`: that column means the IndieWeb handshake, and a search engine's opinion that two strings co-occur is not it. Every link is stamped `web-search` so a consumer can exclude the class outright. The gate is mean on purpose — confident it is a person, more than one word of name, publishes here, and currently unreachable — and it is stated twice, in the SQL that selects and in `worthSearching` that re-checks, so a drift between them costs nothing rather than money. On LinkedIn, since it is the thing that was actually asked for: a profile URL found this way is stored, because it is a public address the search engine has already indexed. The profile behind it is not fetched — auth-walled, 999 to anything automated, and its terms forbid scraping. We can say where somebody's LinkedIn is; we cannot say what is on it. Two things the tests caught rather than the reader: `authors` has no `site` column (it is `site_url`), and `addAuthorLinks` already existed and batches its inserts into one round trip, so the second one I wrote was deleted rather than kept. --- apps/poller/src/index.js | 53 +++ .../db/migrations/0033_author_searches.sql | 38 ++ packages/db/src/authors.js | 105 ++++++ packages/ingest/index.js | 7 + packages/ingest/src/websearch.js | 338 ++++++++++++++++++ packages/ingest/test/websearch.test.js | 238 ++++++++++++ 6 files changed, 779 insertions(+) create mode 100644 packages/db/migrations/0033_author_searches.sql create mode 100644 packages/ingest/src/websearch.js create mode 100644 packages/ingest/test/websearch.test.js diff --git a/apps/poller/src/index.js b/apps/poller/src/index.js index 88835d0..00613fd 100644 --- a/apps/poller/src/index.js +++ b/apps/poller/src/index.js @@ -2,6 +2,7 @@ import { connect, migrate, q, accounts, alerts } from '@rssamplifier/db'; import { crawlDue, enrichDue, + searchDue, notifyFinishedSubmissions, notifyFinishedDiscoveries, drainDiscoveryQueue, @@ -153,6 +154,24 @@ const authorEnabled = env['AUTHOR_ENRICH'] !== '0' && env['AUTHOR_ENRICH'] !== ' // anything once the hour's 60 are gone, and does so quietly, as a 403. const authorGithubToken = String(env['GITHUB_TOKEN'] ?? '').trim(); +// Buying searches for the people who left no trail. +// +// **Off unless every one of these is set**, and that is the design rather than +// caution. The credits are metered, they come from an account shared with +// another product, and there are 369,056 feeds here -- one query each would be +// fifteen times the monthly allowance. So it takes a key, a non-zero budget, +// and an explicit switch, and the budget is counted from the ledger in the +// database rather than from a variable, because this process restarts on every +// deploy and a budget that resets with it is not a budget. +const searchEnabled = env['AUTHOR_SEARCH'] === '1' || env['AUTHOR_SEARCH'] === 'true'; +const searchApiKey = String(env['VALUESERP_API_KEY'] ?? '').trim(); +const searchMonthlyBudget = Number(env['AUTHOR_SEARCH_BUDGET']) || 0; +const searchPerAuthor = Number(env['AUTHOR_SEARCH_PER_AUTHOR']) || 2; +const searchBatch = Number(env['AUTHOR_SEARCH_BATCH']) || 5; +// Slow on purpose: this is the one pass that costs money per unit of work, so +// its default cadence spends at most a few credits an hour even misconfigured. +const searchIntervalMs = (Number(env['AUTHOR_SEARCH_INTERVAL_SECONDS']) || 900) * 1000; + // Accounts considered per alert pass, and how often a pass runs. Its own timer // for the same reason the card and cluster passes have one: a tick spends // minutes inside the crawl, and work queued behind that only happens if the @@ -611,6 +630,38 @@ async function enrichTick() { } } +/** + * Spend a little of the month's search allowance on the unreachable. + * + * Guarded twice over. It does nothing without a key, a budget and the switch; + * and `searchDue` re-reads the ledger every pass, so two pollers or a restarted + * one cannot between them spend more than the month allows. + */ +let searching = false; + +async function searchTick() { + if (!searchEnabled || !searchApiKey || searchMonthlyBudget <= 0) return; + if (searching) return; + searching = true; + + try { + const result = await searchDue(db, { + apiKey: searchApiKey, + monthlyBudget: searchMonthlyBudget, + perAuthor: searchPerAuthor, + batchSize: searchBatch, + }); + // Logged whenever anything was bought, even when it found nothing -- + // especially then. A pass that spends credits and stores no links is the + // signal that the gate needs tightening, and it is invisible otherwise. + if (result.spent) log('author-search', result); + } catch (err) { + log('author-search-error', { message: String(err?.message ?? err) }); + } finally { + searching = false; + } +} + /** * Write down how deep each queue is, so /crawlstats can draw the slope. * @@ -656,6 +707,7 @@ const cardTimer = setInterval(cardTick, cardIntervalMs); const alertTimer = setInterval(alertTick, alertIntervalMs); const enrichTimer = setInterval(enrichTick, authorIntervalMs); const queueTimer = setInterval(queueTick, queueSampleMs); +const searchTimer = setInterval(searchTick, searchIntervalMs); void tick(); void backfillTick(); void cardTick(); @@ -693,6 +745,7 @@ function shutdown(signal) { clearInterval(enrichTimer); clearInterval(alertTimer); clearInterval(queueTimer); + clearInterval(searchTimer); // Recorded before the buffer is closed, so the live log's last line is the // daemon saying it stopped rather than the log simply going quiet — which is diff --git a/packages/db/migrations/0033_author_searches.sql b/packages/db/migrations/0033_author_searches.sql new file mode 100644 index 0000000..7174f64 --- /dev/null +++ b/packages/db/migrations/0033_author_searches.sql @@ -0,0 +1,38 @@ +-- What we spent looking for people, and on whom. +-- +-- Author enrichment reads what publishers put where we could find it, and that +-- is free. Buying a search is not: the credits come from a metered account +-- shared with another product, and the directory is large enough that an +-- unbounded pass would spend a month's allowance in an afternoon. +-- +-- A counter in the process would not do. The poller restarts on every deploy, +-- and a budget that resets when the process does is not a budget — it is a +-- rate limit with a hole in it. So the spend is written down, and the ledger is +-- the authority. +-- +-- It is also the audit trail, which matters as much. "Which people did we spend +-- money looking for, and did it find anything" is a question worth being able +-- to answer, both to tune the gate that chooses them and to justify the line +-- item. +create table if not exists author_searches ( + id text primary key, + + -- Null once an author is deleted: the spend still happened, and dropping the + -- row would make the month's total disagree with what was billed. + author_id text references authors (id) on delete set null, + + at text not null, + + -- Credits actually spent, which is not the same as results found. A query + -- that returns nothing is still billed, and a budget that counted only hits + -- would overspend precisely on the people who are hardest to find. + queries integer not null default 0, + + -- Links stored as a result, so the gate can be judged on its yield rather + -- than on how it felt. + found integer not null default 0 +); + +-- The only query this table answers in the hot path: how much has been spent +-- since the start of the current billing period. +create index if not exists author_searches_at_idx on author_searches (at); diff --git a/packages/db/src/authors.js b/packages/db/src/authors.js index 64a3516..23dbe63 100644 --- a/packages/db/src/authors.js +++ b/packages/db/src/authors.js @@ -812,3 +812,108 @@ export async function postsByAuthor(db, feedIds, limit = 12) { return rows; } + +/* ------------------------------------------------------------------ * + * Bought searches + * ------------------------------------------------------------------ */ + +/** + * The people it would be worth buying a search for. + * + * The gate is mean because the budget is small. What it selects for is the + * person we are confident *is* a person, who writes here, and whom nobody can + * currently contact — which is the only case where a paid query buys something + * the free sources could not. + * + * Ordered by how much they publish, so a limited budget is spent on the + * publishers a reader is most likely to want to reach. + * + * Anyone searched for already is excluded outright rather than re-searched on a + * schedule: a second query for somebody the web did not know about the first + * time is the easiest way to spend a month's credits on nothing. + * + * @param {Client} db + * @param {number} [limit] + * @param {number} [minConfidence] the floor the caller publishes at + * @returns {Promise} + */ +export async function authorsWithoutContact(db, limit = 10, minConfidence = 0.8) { + const { rows } = await db.execute({ + sql: `select a.id, a.slug, a.name, a.site_url as site, a.confidence, + count(distinct fa.feed_id) as feed_count + from authors a + join feed_authors fa on fa.author_id = a.id + where a.confidence >= ? + and not exists (select 1 from author_links l where l.author_id = a.id) + and not exists (select 1 from author_searches s where s.author_id = a.id) + -- A single word is not a searchable name: it returns the world. + and instr(trim(a.name), ' ') > 0 + group by a.id + order by feed_count desc, a.confidence desc + limit ?`, + args: [minConfidence, limit], + }); + + return rows; +} + +/** + * Write down what a search cost, whether or not it found anything. + * + * @param {Client} db + * @param {{ authorId: string|null, queries: number, found: number }} spend + * @returns {Promise} + */ +export async function recordAuthorSearch(db, spend) { + await db.execute({ + sql: `insert into author_searches (id, author_id, at, queries, found) + values (?, ?, ?, ?, ?)`, + args: [ + newId(), + spend.authorId ?? null, + nowIso(), + Math.max(0, Math.floor(Number(spend.queries) || 0)), + Math.max(0, Math.floor(Number(spend.found) || 0)), + ], + }); +} + +/** + * Credits spent since a moment, which is how much of the budget is gone. + * + * @param {Client} db + * @param {string} since ISO 8601 + * @returns {Promise} + */ +export async function searchSpendSince(db, since) { + const { rows } = await db.execute({ + sql: 'select coalesce(sum(queries), 0) as spent from author_searches where at >= ?', + args: [String(since)], + }); + + return Number(rows[0]?.spent ?? 0); +} + +/** + * The start of the current billing period. + * + * The provider's month does not begin on the first: ValueSERP resets this + * account's allowance on the **13th**, so a budget counted per calendar month + * would let the allowance be spent twice across a reset and refuse spending + * that is actually available just after one. + * + * @param {Date} [now] + * @param {number} [resetDay] + * @returns {string} ISO 8601 + */ +export function billingPeriodStart(now = new Date(), resetDay = 13) { + const at = new Date(now.getTime()); + const start = new Date( + Date.UTC(at.getUTCFullYear(), at.getUTCMonth(), resetDay, 0, 0, 0, 0), + ); + + // Before this month's reset day, the period began last month. + if (at.getTime() < start.getTime()) start.setUTCMonth(start.getUTCMonth() - 1); + + return start.toISOString(); +} diff --git a/packages/ingest/index.js b/packages/ingest/index.js index f55bb48..f6be9cc 100644 --- a/packages/ingest/index.js +++ b/packages/ingest/index.js @@ -32,3 +32,10 @@ export { enrichFeedAuthors, storeCredits, } from './src/enrich.js'; +export { + linksFromSearch, + searchDue, + searchForAuthor, + searchesFor, + worthSearching, +} from './src/websearch.js'; diff --git a/packages/ingest/src/websearch.js b/packages/ingest/src/websearch.js new file mode 100644 index 0000000..5ec69ec --- /dev/null +++ b/packages/ingest/src/websearch.js @@ -0,0 +1,338 @@ +/* + * Looking for a person who did not leave a trail. + * + * Everything else in the enrichment reads something the publisher put where we + * could find it: their feed, their page, their hostname, their profile. This + * asks a search engine instead, which is a different kind of act and carries a + * different kind of risk, so it is fenced in three ways that are not optional. + * + * **It cannot run over the directory.** Search is metered. The account this + * borrows is CrawlProof's 25,000-credit month, already shared with CrawlProof's + * own outreach runner, and the directory is 369,056 feeds — one query each + * would be fifteen times the entire monthly allowance. So this is a targeted + * tool with a hard budget, off unless switched on, and it refuses rather than + * degrades when the budget is gone. + * + * **It cannot invent a person.** Searching a name and attaching what comes back + * is how two different people with one name become one page. `identity.js` has + * refused to key on a name alone since it was written, and this must not become + * the back door: a search result is a *candidate*, and what promotes it is + * corroboration — the page linking back to the site we already know is theirs, + * or a handle we already hold from a source that proved itself. + * + * **It cannot be the reason we hold something.** Every link it produces is + * stamped `web-search`, so any consumer can exclude the whole class. Nothing + * here is ever marked `verified` on the strength of a search result. + * + * On LinkedIn specifically, since it is the thing everybody asks for: a profile + * URL found this way is stored as a link, because it is a public address the + * search engine already indexed. The profile behind it is not fetched — it is + * auth-walled, returns 999 to anything automated, and its terms forbid + * scraping. So we can tell you where somebody's LinkedIn is, and cannot tell + * you what is on it. + */ + +import { classifyLink } from '@rssamplifier/feed'; +import { authors as a } from '@rssamplifier/db'; + +/** Where the searches are bought. */ +const ENDPOINT = 'https://api.valueserp.com/search'; + +/** How long to wait for a result before abandoning it. */ +const TIMEOUT_MS = 8000; + +/** + * The networks worth spending a query on, and how to ask for them. + * + * One query per network rather than one broad query, because a search for a + * name alone returns news articles and namesakes, while a site-scoped one + * returns a profile or nothing — and "nothing" is the answer we want when the + * person genuinely has no account there. + * + * LinkedIn leads because it is the one this exists for. The rest are ordered by + * how often this directory's population actually has them. + */ +const TARGETS = [ + { network: 'linkedin', site: 'linkedin.com/in' }, + { network: 'github', site: 'github.com' }, + { network: 'twitter', site: 'x.com' }, + { network: 'bluesky', site: 'bsky.app/profile' }, +]; + +/** + * Is a search worth buying for this person? + * + * The gate is deliberately mean, because the budget is small and the value of a + * query is wildly uneven. A person who writes one dormant blog and already has + * an email is not worth a credit; a person behind several live feeds with no + * way to reach them at all is the whole point. + * + * @param {{ feedCount?: number, linkCount?: number, confidence?: number, name?: string }} author + * @param {{ minFeeds?: number }} [opts] + * @returns {boolean} + */ +export function worthSearching(author, opts = {}) { + const minFeeds = Number(opts.minFeeds ?? 1); + + // Somebody we are not confident is a person should not be searched for: the + // query would be for a name we half-believe, and the results would be + // attached to it. + if (Number(author?.confidence ?? 0) < 0.8) return false; + + // A name that is one word is not searchable in any useful way -- "Sakrecoer" + // returns the person, but "Jane" returns the world. + const words = String(author?.name ?? '').trim().split(/\s+/).filter(Boolean); + if (words.length < 2) return false; + + if (Number(author?.feedCount ?? 0) < minFeeds) return false; + + // Already reachable. The point of the budget is the people we cannot contact. + if (Number(author?.linkCount ?? 0) > 0) return false; + + return true; +} + +/** + * The queries to buy for one person. + * + * Scoped to their own site as well as to each network, because the name alone + * is the ambiguous part: "Jane Doe" site:linkedin.com/in returns every Jane + * Doe, while the domain of the blog she writes is the thing that distinguishes + * her from them. + * + * @param {{ name: string, site?: string|null }} author + * @returns {Array<{ network: string, q: string }>} + */ +export function searchesFor(author) { + const name = String(author?.name ?? '').trim(); + if (!name) return []; + + const domain = hostOf(author?.site); + + return TARGETS.map((target) => ({ + network: target.network, + q: domain + ? `"${name}" ${domain} site:${target.site}` + : `"${name}" site:${target.site}`, + })); +} + +/** + * Read a ValueSERP response into the links it actually contains. + * + * Only results whose URL classifies as the network we asked for are kept. A + * search for a LinkedIn profile routinely returns a LinkedIn *company* page, a + * job posting or an article about the person, none of which is an account — + * `classifyLink` already knows the difference and is the filter. + * + * @param {string} network the network the query was scoped to + * @param {unknown} body the parsed JSON + * @returns {Array<{ network: string, url: string, handle: string, source: string, verified: boolean }>} + */ +export function linksFromSearch(network, body) { + const results = Array.isArray(/** @type {any} */ (body)?.organic_results) + ? /** @type {any} */ (body).organic_results + : []; + + /** @type {Map} */ + const found = new Map(); + + for (const result of results) { + const link = classifyLink(result?.link); + if (!link || link.network !== network) continue; + if (found.has(link.url)) continue; + + found.set(link.url, { + ...link, + source: 'web-search', + // Never. A search engine's opinion that two strings co-occur is not the + // IndieWeb handshake, and this column means the handshake. + verified: false, + }); + } + + return [...found.values()]; +} + +/** + * Buy the searches for one person, within a budget. + * + * Returns the links and how many credits were actually spent, so the caller can + * keep a running total that survives a restart by being written down rather + * than held in memory. + * + * Every failure is empty and silent, exactly as the ad fetch is: enrichment is + * a bonus on top of a directory, and a search provider having a bad afternoon + * must not stop the pass that does not need it. + * + * @param {{ name: string, site?: string|null }} author + * @param {{ apiKey: string, budget: number, fetch?: typeof globalThis.fetch }} opts + * `budget` is the number of queries this call may spend, already decided by + * the caller against the month's remaining allowance + * @returns {Promise<{ links: Array, spent: number, exhausted?: boolean }>} + * `exhausted` says the account is empty, which will not change before the + * provider's reset -- the caller stops rather than asking again + */ +export async function searchForAuthor(author, opts) { + const apiKey = String(opts?.apiKey ?? ''); + const budget = Math.max(0, Math.floor(Number(opts?.budget ?? 0))); + if (!apiKey || budget === 0) return { links: [], spent: 0 }; + + const doFetch = opts.fetch ?? globalThis.fetch; + const queries = searchesFor(author).slice(0, budget); + + /** @type {Map} */ + const links = new Map(); + let spent = 0; + + for (const query of queries) { + const url = + `${ENDPOINT}?api_key=${encodeURIComponent(apiKey)}` + + `&q=${encodeURIComponent(query.q)}&num=10&output=json`; + + let body; + try { + const res = await doFetch(url, { + signal: AbortSignal.timeout(TIMEOUT_MS), + headers: { accept: 'application/json' }, + }); + + // 402 is the documented answer for an exhausted account, and it is not a + // transient error -- there is no point asking again this month, so the + // caller is told to stop rather than left to burn the rest of the batch + // on the same refusal. + if (res.status === 402) return { links: [...links.values()], spent, exhausted: true }; + if (!res.ok) continue; + + body = await res.json(); + } catch { + continue; + } + + // Counted whether or not anything useful came back: the credit is spent at + // the provider either way, and a budget that only counts hits is not a + // budget. + spent += 1; + + for (const link of linksFromSearch(query.network, body)) { + if (!links.has(link.url)) links.set(link.url, link); + } + } + + return { links: [...links.values()], spent }; +} + +/** + * The bare host of a URL, for scoping a query to somebody's own domain. + * + * @param {unknown} value + * @returns {string} + */ +function hostOf(value) { + try { + return new URL(String(value ?? '')).hostname.toLowerCase().replace(/^www\./, ''); + } catch { + return ''; + } +} + +/** + * Spend part of the month's allowance on the people nobody can reach. + * + * The budget is read from the ledger rather than from a counter, because the + * poller restarts on every deploy and a budget that resets with the process is + * not a budget. `billingPeriodStart` matches the provider's own cycle -- this + * account resets on the 13th, not the 1st -- so the total this compares against + * is the total the invoice will show. + * + * Stops at the first sign the account is empty. A 402 is the documented answer + * for an exhausted allowance and it will not change before the reset, so + * carrying on would spend the rest of the batch's wall-clock re-reading the + * same refusal. + * + * @param {import('@libsql/client').Client} db + * @param {{ + * apiKey: string, + * monthlyBudget: number, + * perAuthor?: number, + * batchSize?: number, + * minConfidence?: number, + * fetch?: typeof globalThis.fetch, + * }} opts + * @returns {Promise<{ people: number, links: number, spent: number, remaining: number }>} + */ +export async function searchDue(db, opts) { + const apiKey = String(opts?.apiKey ?? ''); + const monthly = Math.max(0, Math.floor(Number(opts?.monthlyBudget ?? 0))); + if (!apiKey || monthly === 0) return { people: 0, links: 0, spent: 0, remaining: 0 }; + + const since = a.billingPeriodStart(); + const already = await a.searchSpendSince(db, since); + let remaining = monthly - already; + if (remaining <= 0) return { people: 0, links: 0, spent: 0, remaining: 0 }; + + const perAuthor = Math.max(1, Math.floor(Number(opts.perAuthor ?? 2))); + const batchSize = Math.max(1, Math.floor(Number(opts.batchSize ?? 5))); + + const candidates = await a.authorsWithoutContact( + db, + batchSize, + Number(opts.minConfidence ?? 0.8), + ); + + let people = 0; + let links = 0; + let spent = 0; + + for (const author of candidates) { + if (remaining <= 0) break; + + // The SQL above already selects for this, and the check is repeated here on + // purpose: the query and the rule are two statements of one policy, and the + // cheap one is the one that must not be the only one. If they ever drift, + // this refuses to spend money on the difference. + if ( + !worthSearching({ + name: String(author.name ?? ''), + confidence: Number(author.confidence ?? 0), + feedCount: Number(author.feed_count ?? 0), + linkCount: 0, + }) + ) { + continue; + } + + const result = await searchForAuthor( + { name: String(author.name), site: author.site }, + { + apiKey, + budget: Math.min(perAuthor, remaining), + fetch: opts.fetch, + }, + ); + + // Written down before anything else, because the credits are gone whether + // or not the rest of this succeeds, and a ledger that only records + // successful passes will drift under the real spend. + if (result.spent > 0) { + await a.recordAuthorSearch(db, { + authorId: String(author.id), + queries: result.spent, + found: result.links.length, + }); + } + + spent += result.spent; + remaining -= result.spent; + + if (result.links.length > 0) { + links += await a.addAuthorLinks(db, String(author.id), result.links); + people += 1; + } + + // The account is empty. Nothing further this period will succeed. + if (result.exhausted) break; + } + + return { people, links, spent, remaining: Math.max(0, remaining) }; +} diff --git a/packages/ingest/test/websearch.test.js b/packages/ingest/test/websearch.test.js new file mode 100644 index 0000000..3d61dad --- /dev/null +++ b/packages/ingest/test/websearch.test.js @@ -0,0 +1,238 @@ +import assert from 'node:assert/strict'; +import { test, before, after } from 'node:test'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { connect, migrate, q, authors as a } from '@rssamplifier/db'; + +import { storeCredits } from '../src/enrich.js'; +import { linksFromSearch, searchDue, searchesFor, worthSearching } from '../src/websearch.js'; + +// This is the only pass that spends money, so what is tested is mostly what it +// refuses to do: search for somebody it is not sure is a person, search for +// somebody already reachable, spend past the budget, or believe a result. + +let dir; +let db; + +before(async () => { + dir = await mkdtemp(join(tmpdir(), 'rssamp-search-')); + db = connect({ url: `file:${join(dir, 'test.db')}` }); + await migrate(db); +}); + +after(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +test('the gate refuses everyone a credit would be wasted on', () => { + const ok = { name: 'Jane Doe', confidence: 0.85, feedCount: 1, linkCount: 0 }; + assert.equal(worthSearching(ok), true); + + // A name we only half-believe would put the results on a person who may not + // exist. + assert.equal(worthSearching({ ...ok, confidence: 0.6 }), false); + // One word returns the world. + assert.equal(worthSearching({ ...ok, name: 'Jane' }), false); + // Already reachable: the budget exists for the people who are not. + assert.equal(worthSearching({ ...ok, linkCount: 2 }), false); + assert.equal(worthSearching({}), false); +}); + +test('a query is scoped by the domain, which is what tells two namesakes apart', () => { + const queries = searchesFor({ name: 'Jane Doe', site: 'https://www.jane.example/blog' }); + assert.ok(queries.every((s) => s.q.includes('jane.example'))); + assert.ok(queries.some((s) => s.q === '"Jane Doe" jane.example site:linkedin.com/in')); + + // Without a site there is nothing to scope by, and the name has to stand alone. + const bare = searchesFor({ name: 'Jane Doe' }); + assert.equal(bare[0].q, '"Jane Doe" site:linkedin.com/in'); + assert.deepEqual(searchesFor({ name: '' }), []); +}); + +test('only real profiles survive, and none of them is evidence', () => { + const links = linksFromSearch('linkedin', { + organic_results: [ + { link: 'https://www.linkedin.com/in/janedoe' }, + // A company page is not a person, and a post is not an account. + { link: 'https://www.linkedin.com/company/acme' }, + { link: 'https://www.linkedin.com/posts/janedoe_hiring-activity-123' }, + // The right shape on the wrong network for this query. + { link: 'https://github.com/janedoe' }, + ], + }); + + assert.deepEqual( + links.map((l) => l.url), + ['https://www.linkedin.com/in/janedoe'], + ); + assert.equal(links[0].source, 'web-search'); + assert.equal(links[0].verified, false, 'a search engine cannot verify anything'); +}); + +test('a malformed or empty response yields nothing rather than throwing', () => { + assert.deepEqual(linksFromSearch('github', null), []); + assert.deepEqual(linksFromSearch('github', {}), []); + assert.deepEqual(linksFromSearch('github', { organic_results: 'no' }), []); +}); + +/** Seed one author with no way to reach them. */ +async function seedUnreachable(slug, name) { + const feed = await q.insertFeed(db, { + slug, + feed_url: `https://${slug}.example/feed.xml`, + site_url: `https://${slug}.example/`, + title: name, + categories: [], + kind: 'blog', + status: 'active', + }); + + await storeCredits( + db, + { id: feed.id, feed_url: `https://${slug}.example/feed.xml` }, + [ + { + name, + email: '', + url: '', + avatar: '', + role: 'author', + source: 'atom-feed-author', + confidence: 0.85, + }, + ], + ); + + return feed; +} + +test('the budget is read from the ledger, so a restart cannot reset it', async () => { + await seedUnreachable('ledger-blog', 'Jane Ledger'); + + // Everything already spent this period. + await a.recordAuthorSearch(db, { authorId: null, queries: 25, found: 0 }); + + let called = 0; + const result = await searchDue(db, { + apiKey: 'test-key', + monthlyBudget: 25, + fetch: async () => { + called += 1; + return { ok: true, status: 200, json: async () => ({}) }; + }, + }); + + assert.equal(called, 0, 'the allowance is gone, so nothing may be bought'); + assert.equal(result.spent, 0); +}); + +test('spending stops at the budget even when more people qualify', async () => { + const fresh = await mkdtemp(join(tmpdir(), 'rssamp-budget-')); + const db2 = connect({ url: `file:${join(fresh, 'b.db')}` }); + await migrate(db2); + + for (const [slug, name] of [ + ['aa-blog', 'Anna Aardvark'], + ['bb-blog', 'Bob Bison'], + ['cc-blog', 'Cara Cat'], + ]) { + const feed = await q.insertFeed(db2, { + slug, + feed_url: `https://${slug}.example/feed.xml`, + site_url: `https://${slug}.example/`, + title: name, + categories: [], + kind: 'blog', + status: 'active', + }); + await storeCredits( + db2, + { id: feed.id, feed_url: `https://${slug}.example/feed.xml` }, + [{ name, email: '', url: '', avatar: '', role: 'author', source: 'atom-feed-author', confidence: 0.85 }], + ); + } + + let called = 0; + const result = await searchDue(db2, { + apiKey: 'test-key', + monthlyBudget: 3, + perAuthor: 2, + batchSize: 10, + fetch: async () => { + called += 1; + return { ok: true, status: 200, json: async () => ({ organic_results: [] }) }; + }, + }); + + assert.equal(called, 3, `bought ${called} queries against a budget of 3`); + assert.equal(result.spent, 3); + + // And the ledger agrees with what was bought, which is the number that has to + // match the invoice. + assert.equal(await a.searchSpendSince(db2, a.billingPeriodStart()), 3); + + await rm(fresh, { recursive: true, force: true }); +}); + +test('an exhausted account stops the batch instead of being asked again', async () => { + const fresh = await mkdtemp(join(tmpdir(), 'rssamp-402-')); + const db2 = connect({ url: `file:${join(fresh, 'c.db')}` }); + await migrate(db2); + + const feed = await q.insertFeed(db2, { + slug: 'dd-blog', + feed_url: 'https://dd-blog.example/feed.xml', + site_url: 'https://dd-blog.example/', + title: 'Dee Blogger', + categories: [], + kind: 'blog', + status: 'active', + }); + await storeCredits( + db2, + { id: feed.id, feed_url: 'https://dd-blog.example/feed.xml' }, + [{ name: 'Dee Blogger', email: '', url: '', avatar: '', role: 'author', source: 'atom-feed-author', confidence: 0.85 }], + ); + + let called = 0; + await searchDue(db2, { + apiKey: 'test-key', + monthlyBudget: 50, + perAuthor: 4, + fetch: async () => { + called += 1; + // The provider's documented answer for an empty account. + return { ok: false, status: 402, json: async () => ({}) }; + }, + }); + + assert.equal(called, 1, '402 will not change before the reset, so asking twice is waste'); + + await rm(fresh, { recursive: true, force: true }); +}); + +test('nothing is bought without a key, a budget and the switch', async () => { + let called = 0; + const fetcher = async () => { + called += 1; + return { ok: true, status: 200, json: async () => ({}) }; + }; + + assert.deepEqual(await searchDue(db, { apiKey: '', monthlyBudget: 100, fetch: fetcher }), { + people: 0, + links: 0, + spent: 0, + remaining: 0, + }); + await searchDue(db, { apiKey: 'k', monthlyBudget: 0, fetch: fetcher }); + assert.equal(called, 0); +}); + +test('the billing period follows the provider, which resets on the 13th', () => { + // A calendar month would let the allowance be spent twice across a reset. + assert.equal(a.billingPeriodStart(new Date('2026-08-19T00:00:00Z')), '2026-08-13T00:00:00.000Z'); + // Before the 13th, the period began the month before. + assert.equal(a.billingPeriodStart(new Date('2026-08-02T00:00:00Z')), '2026-07-13T00:00:00.000Z'); +}); From cc6bd3eadb0fefc10d398ed5eeec225a2f456fd8 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Wed, 19 Aug 2026 08:22:36 +0000 Subject: [PATCH 4/5] Stop losing a publisher for ninety days because their site was down once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two things in the same area, both found while looking for a queue that turned out not to be needed. **A failure was recorded as a miss.** Every fetch in the enrichment fails softly — a dead host, a timeout and a 503 all come back as "no page" rather than as an exception — so a site that was simply down looked identical to a site that names nobody, and both were stamped as checked. That cost the publisher their enrichment for the whole ninety-day recheck cycle on the strength of one bad afternoon, and on a pass that has so far reached 3,275 of 369,056 feeds it quietly loses everyone on a flaky host. The pass now tracks whether it got an answer out of the publisher at all. Reached and found nobody: stamped, left alone until the recheck. Never reached: stamped to come back in a few days, because nothing was learned and the next attempt may well work. Still stamped either way, which is what keeps a permanently broken feed off the head of the queue. Done by back-dating the stamp rather than adding an attempts column, and that is a trade rather than a shortcut: writes here serialize and the crawl is already write-bound, so the fix that costs one UPDATE beats the tidier one that costs a migration and a second column on every read. **The pass had no row on /crawlstats.** It is the only job on the board that was invisible, which is a large part of why it looked like it had never started. It now shows its backlog, how many publishers it looked at in the last hour, and how far through the directory it is — read off the partial index 0024 already built, and counted as the stamped set rather than its complement, because 3,275 index entries is a cheap question and 369,056 rows is not. The queue rebuild this started as is deliberately not here. dueForAuthors already scopes to active feeds, so the pass has never been walking the 280,360 pending ones, and at the rate now observed the 84,398 active feeds are done in under a fortnight. A claim/lease table would have been machinery for a problem the numbers say does not exist. An existing test caught the new row before it shipped: every job on the board must report when it last ran, and one that emits an event nobody records is a row that reads as permanently stalled. --- apps/web/src/lib/jobs.js | 14 ++++ apps/web/test/jobs.test.js | 4 + packages/db/src/authors.js | 40 ++++++++++ packages/db/src/queries.js | 19 ++++- packages/ingest/src/enrich.js | 35 ++++++++- packages/ingest/test/enrich-retry.test.js | 89 +++++++++++++++++++++++ 6 files changed, 197 insertions(+), 4 deletions(-) create mode 100644 packages/ingest/test/enrich-retry.test.js diff --git a/apps/web/src/lib/jobs.js b/apps/web/src/lib/jobs.js index d41c979..ca6820c 100644 --- a/apps/web/src/lib/jobs.js +++ b/apps/web/src/lib/jobs.js @@ -102,6 +102,20 @@ export function jobRows({ // The one row where a backlog is a problem rather than a workload. alarmAbove: 500, }, + { + key: 'authors', + label: 'Author enrichment', + what: 'Finds who writes each feed, and how to reach them', + backlog: backlogs.authorsPending ?? 0, + // Genuinely a queue rather than a fire: it walks the directory once and + // then rechecks on a 90-day cycle, so a large number here is the work + // remaining and not a stall. + expectFull: true, + rate: backlogs.authorsLastHour ?? 0, + rateNote: 'publishers looked at', + done: `${backlogs.authorsDone ?? 0} looked at so far`, + events: ['author', 'authors-error', 'author-search', 'author-search-error'], + }, { key: 'cards', label: 'Feed pictures', diff --git a/apps/web/test/jobs.test.js b/apps/web/test/jobs.test.js index c3b686f..3231d37 100644 --- a/apps/web/test/jobs.test.js +++ b/apps/web/test/jobs.test.js @@ -21,6 +21,9 @@ function busy(overrides = {}) { cardsNone: 283, cardsError: 31, cardsLastHour: 1_320, + authorsPending: 81_123, + authorsDone: 3_275, + authorsLastHour: 283, ...overrides.backlogs, }, activity: { @@ -32,6 +35,7 @@ function busy(overrides = {}) { 'cluster-backfill': { lines: 360, errors: 0, amount: 12_000, lastAt: at, ms: 90 }, topics: { lines: 4, errors: 0, amount: 0, lastAt: at, ms: 300 }, alerts: { lines: 30, errors: 0, amount: 48, lastAt: at, ms: 600 }, + author: { lines: 283, errors: 4, amount: 60, lastAt: at, ms: 2_400 }, ...overrides.activity, }, alertAccounts: overrides.alertAccounts ?? 4, diff --git a/packages/db/src/authors.js b/packages/db/src/authors.js index 23dbe63..b33aa81 100644 --- a/packages/db/src/authors.js +++ b/packages/db/src/authors.js @@ -43,6 +43,46 @@ export async function dueForAuthors(db, limit = 25, recheckBefore = '') { return /** @type {any} */ (rows); } +/** + * Record that a feed's enrichment *failed*, so it is tried again sooner. + * + * `markAuthorsChecked` is right for a miss — a site that genuinely names nobody + * should not be re-read tomorrow — and wrong for a failure. They were the same + * call, which meant a DNS hiccup, a timeout or a 503 cost that publisher its + * enrichment for the **full recheck cycle**, ninety days, on the strength of one + * bad afternoon. On a pass that has so far reached 3,275 of 369,056 feeds, that + * is a quiet way to lose the ones on flaky hosts permanently. + * + * Done by back-dating the stamp rather than by adding an attempts column, and + * that is a deliberate trade. Writes on this database serialize and the crawl + * is already write-bound (see the notes in `crawl.js`), so the cheap fix that + * costs one UPDATE beats the tidy one that costs a migration and a second + * column to read. The feed still counts as "looked at" for the backlog, and + * still comes due again in `retryDays`. + * + * It must never back-date past *now*: a stamp in the future would hide the feed + * from a pass whose recheck window is shorter than this one's. + * + * @param {Client} db + * @param {string} feedId + * @param {{ retryDays?: number, recheckDays?: number }} [opts] + * @returns {Promise} + */ +export async function markAuthorsFailed(db, feedId, opts = {}) { + const retryDays = Math.max(0, Number(opts.retryDays ?? 3)); + const recheckDays = Math.max(retryDays, Number(opts.recheckDays ?? 90)); + + // Stamped as though it were checked (recheckDays - retryDays) ago, so the + // ordinary due test brings it back in retryDays without knowing why. + const backdated = (recheckDays - retryDays) * 86_400_000; + const at = new Date(Date.now() - backdated).toISOString(); + + await db.execute({ + sql: 'update feeds set authors_checked_at = ? where id = ?', + args: [at, feedId], + }); +} + /** * Record that a feed has been looked at, whether or not anyone was found. * diff --git a/packages/db/src/queries.js b/packages/db/src/queries.js index 53ecf01..7b8948f 100644 --- a/packages/db/src/queries.js +++ b/packages/db/src/queries.js @@ -1382,7 +1382,7 @@ export async function jobBacklogs(db) { // The fix is not "stop using conditional aggregates" — two of them survive // below. It is to make sure the scan they force is over a *covering index* // rather than over the table, at which point the same CASE is cheap. - const [byStatus, backlog, submitted, cards] = await Promise.all([ + const [byStatus, backlog, submitted, cards, enriched] = await Promise.all([ // One covering scan of feeds_status_success_idx (0028) answering three // questions at once: the status breakdown, and how many feeds in each state // have never once been read successfully. @@ -1410,6 +1410,20 @@ export async function jobBacklogs(db) { from feeds group by card_state`, args: [hourAgo], }), + + // How far the author enrichment has walked, read off the partial index + // 0024 already built for it (`feeds (authors_checked_at) where status = + // 'active'`). Counted as the *stamped* set rather than the unstamped one + // for the reason this whole function exists: 3,275 of 369,056 feeds carry a + // stamp, so this touches a few thousand index entries, while asking for the + // complement would visit every row. The backlog is arithmetic afterwards. + db.execute({ + sql: `select count(*) as n, + sum(case when authors_checked_at >= ? then 1 else 0 end) as hour + from feeds + where status = 'active' and authors_checked_at is not null`, + args: [hourAgo], + }), ]); // Deliberately no "first crawls completed this hour". Nothing records when a @@ -1443,6 +1457,9 @@ export async function jobBacklogs(db) { cardsNone: card('none'), cardsError: card('error'), cardsLastHour: cards.rows.reduce((a, r) => a + Number(r.hour ?? 0), 0), + authorsDone: Number(enriched.rows[0]?.n ?? 0), + authorsPending: Math.max(0, n('active') - Number(enriched.rows[0]?.n ?? 0)), + authorsLastHour: Number(enriched.rows[0]?.hour ?? 0), }; } diff --git a/packages/ingest/src/enrich.js b/packages/ingest/src/enrich.js index e9dd4d7..b7aaa02 100644 --- a/packages/ingest/src/enrich.js +++ b/packages/ingest/src/enrich.js @@ -291,6 +291,16 @@ export async function enrichFeedAuthors(db, feed, opts = {}) { let pages = 0; let verified = 0; + // Did we ever get an answer out of this publisher at all? + // + // The distinction this records is the one the stamp used to lose. Every fetch + // in here fails softly -- a dead host, a timeout and a 503 all come back as + // "no page" rather than as an exception -- so a site that was simply down + // looked exactly like a site that named nobody, and both were stamped as + // checked. That cost the publisher their enrichment for the full ninety-day + // recheck cycle on the strength of one bad afternoon. + let reached = false; + const collect = (found) => { for (const link of found) { const existing = links.get(link.url); @@ -305,6 +315,7 @@ export async function enrichFeedAuthors(db, feed, opts = {}) { // the page fetches below. const resolved = await resolve(String(feed.feed_url)).catch(() => null); if (resolved?.ok) { + reached = true; credits.push(...(resolved.feed.credits ?? [])); // A feed that declares its site is a better base for relative links than // whatever the row was stored with. @@ -326,6 +337,7 @@ export async function enrichFeedAuthors(db, feed, opts = {}) { } const page = await fetchPage(target).catch(() => null); + if (page?.ok) reached = true; if (!page?.ok || !/html/i.test(page.contentType)) continue; pages += 1; @@ -514,9 +526,17 @@ export async function enrichFeedAuthors(db, feed, opts = {}) { } const stored = await storeCredits(db, feed, merged, [...links.values()]); - await a.markAuthorsChecked(db, String(feed.id)); - return { ...stored, pages, verified }; + // Looked and found nobody: stamped, and left alone until the recheck. Could + // not look at all: stamped so it comes back in a few days, because nothing + // was learned about this publisher and the next attempt may well succeed. + if (reached) { + await a.markAuthorsChecked(db, String(feed.id)); + } else { + await a.markAuthorsFailed(db, String(feed.id), { recheckDays: opts.recheckDays }); + } + + return { ...stored, pages, verified, reached }; } /** @@ -612,6 +632,7 @@ export async function enrichDue(db, batchSize = 10, opts = {}) { try { const result = await enrichFeedAuthors(db, feed, { verify: opts.verify, + recheckDays, fetch: opts.fetch, resolve: opts.resolve, githubToken: opts.githubToken, @@ -622,7 +643,15 @@ export async function enrichDue(db, batchSize = 10, opts = {}) { feedLinks += result.feedLinks ?? 0; report(opts.onEvent, feed, started, { ok: true, amount: result.people, detail: null }); } catch (err) { - await a.markAuthorsChecked(db, String(feed.id)).catch(() => {}); + // A failure is not a miss. Stamping this the same way as "looked, and + // there was nobody" would cost a publisher on a flaky host their + // enrichment for the whole recheck cycle over one timeout, so it is + // stamped to come back in a few days instead. The feed is still + // stamped, which is what keeps a permanently broken one from sitting + // at the head of the queue forever. + await a + .markAuthorsFailed(db, String(feed.id), { recheckDays }) + .catch(() => {}); report(opts.onEvent, feed, started, { ok: false, amount: null, diff --git a/packages/ingest/test/enrich-retry.test.js b/packages/ingest/test/enrich-retry.test.js new file mode 100644 index 0000000..41b4cf5 --- /dev/null +++ b/packages/ingest/test/enrich-retry.test.js @@ -0,0 +1,89 @@ +import assert from 'node:assert/strict'; +import { test, before, after } from 'node:test'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { connect, migrate, q, authors as a } from '@rssamplifier/db'; + +import { enrichDue } from '../src/enrich.js'; + +// A failure is not a miss, and treating them the same is how a publisher on a +// flaky host loses their enrichment for three months over one timeout. + +let dir; +let db; + +before(async () => { + dir = await mkdtemp(join(tmpdir(), 'rssamp-retry-')); + db = connect({ url: `file:${join(dir, 'test.db')}` }); + await migrate(db); +}); + +after(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +test('a feed whose site fell over is tried again in days, not in a season', async () => { + await q.insertFeed(db, { + slug: 'flaky', + feed_url: 'https://flaky.example/feed.xml', + site_url: 'https://flaky.example/', + title: 'Flaky Blog', + categories: [], + kind: 'blog', + status: 'active', + }); + + const recheckDays = 90; + const recheckBefore = new Date(Date.now() - recheckDays * 86_400_000).toISOString(); + + await enrichDue(db, 5, { + recheckDays, + // The shape of a bad afternoon: the host simply does not answer. + resolve: async () => { + throw new Error('ETIMEDOUT'); + }, + fetch: async () => { + throw new Error('ETIMEDOUT'); + }, + }); + + // Still stamped, which is what keeps a permanently broken feed from sitting + // at the head of the queue forever. + const [row] = (await db.execute('select authors_checked_at from feeds where slug = \'flaky\'')).rows; + assert.ok(row.authors_checked_at, 'a failure is still recorded'); + + // But not due 90 days from now. It comes back within a few days. + assert.deepEqual( + await a.dueForAuthors(db, 5, recheckBefore), + [], + 'not due again immediately, or it would block the queue', + ); + + const inFourDays = new Date(Date.now() + 4 * 86_400_000 - recheckDays * 86_400_000).toISOString(); + const soon = await a.dueForAuthors(db, 5, inFourDays); + assert.equal(soon.length, 1, 'and it is due again within days rather than in three months'); +}); + +test('the retry stamp is never in the future', async () => { + // A stamp ahead of now would hide the feed from any pass whose recheck window + // is shorter than this one's. + await q.insertFeed(db, { + slug: 'future', + feed_url: 'https://future.example/feed.xml', + title: 'Future Blog', + categories: [], + kind: 'blog', + status: 'active', + }); + + const [feed] = (await db.execute("select id from feeds where slug = 'future'")).rows; + await a.markAuthorsFailed(db, String(feed.id), { retryDays: 200, recheckDays: 90 }); + + const [row] = (await db.execute("select authors_checked_at from feeds where slug = 'future'")).rows; + assert.ok( + String(row.authors_checked_at) <= new Date().toISOString(), + `stamp must not be in the future, got ${row.authors_checked_at}`, + ); +}); From 5f696f817893c1c8d40b7f6fa33602886028b67f Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Wed, 19 Aug 2026 09:29:02 +0000 Subject: [PATCH 5/5] Name the author-search migration for when it was written MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `0033_author_searches.sql` took the next sequential number, and there is no next sequential number any more: #131 froze the scheme at 0032 and added a test that says so, because 0006, 0018 and 0032 were each claimed by two or more branches before anybody noticed. Git never flags it — the files have different names — so the guard is the only thing that would. Renamed to the moment the file was actually written, 2026-08-19 08:18:24 UTC, per `packages/db/migrations/README.md`. Nothing referenced it by name, and it has never been applied anywhere, so this is a rename rather than a second migration. The failure was also hiding the rest of the suite: `pnpm -r` stops at the first package that fails, so packages/db going red meant ingest, auth, notify, translate, discover, web and poller never ran in CI at all. On Node 22, which is what CI uses: 1,067 pass, 0 fail, 0 cancelled, and `pnpm build` clean. Co-Authored-By: Claude Opus 5 (1M context) --- ...033_author_searches.sql => 20260819081824_author_searches.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/db/migrations/{0033_author_searches.sql => 20260819081824_author_searches.sql} (100%) diff --git a/packages/db/migrations/0033_author_searches.sql b/packages/db/migrations/20260819081824_author_searches.sql similarity index 100% rename from packages/db/migrations/0033_author_searches.sql rename to packages/db/migrations/20260819081824_author_searches.sql