diff --git a/apps/poller/src/index.js b/apps/poller/src/index.js index 0e12f75..00613fd 100644 --- a/apps/poller/src/index.js +++ b/apps/poller/src/index.js @@ -2,6 +2,7 @@ import { connect, migrate, q, accounts, alerts } from '@rssamplifier/db'; import { crawlDue, enrichDue, + searchDue, notifyFinishedSubmissions, notifyFinishedDiscoveries, drainDiscoveryQueue, @@ -145,6 +146,32 @@ const authorVerify = env['AUTHOR_VERIFY'] === '1' || env['AUTHOR_VERIFY'] === 't // is the switch to reach for if a site ever objects to the extra fetches. const authorEnabled = env['AUTHOR_ENRICH'] !== '0' && env['AUTHOR_ENRICH'] !== 'false'; +// A GitHub credential, and the difference between the profile lookups working +// and not working at all: unauthenticated, the API allows **60 requests an hour +// per IP**, which a single batch spends. With a token it is 5,000, which is +// more than this pass can use. Optional only in the sense that the rest of the +// enrichment carries on without it -- profile resolution simply stops finding +// anything once the hour's 60 are gone, and does so quietly, as a 403. +const authorGithubToken = String(env['GITHUB_TOKEN'] ?? '').trim(); + +// Buying searches for the people who left no trail. +// +// **Off unless every one of these is set**, and that is the design rather than +// caution. The credits are metered, they come from an account shared with +// another product, and there are 369,056 feeds here -- one query each would be +// fifteen times the monthly allowance. So it takes a key, a non-zero budget, +// and an explicit switch, and the budget is counted from the ledger in the +// database rather than from a variable, because this process restarts on every +// deploy and a budget that resets with it is not a budget. +const searchEnabled = env['AUTHOR_SEARCH'] === '1' || env['AUTHOR_SEARCH'] === 'true'; +const searchApiKey = String(env['VALUESERP_API_KEY'] ?? '').trim(); +const searchMonthlyBudget = Number(env['AUTHOR_SEARCH_BUDGET']) || 0; +const searchPerAuthor = Number(env['AUTHOR_SEARCH_PER_AUTHOR']) || 2; +const searchBatch = Number(env['AUTHOR_SEARCH_BATCH']) || 5; +// Slow on purpose: this is the one pass that costs money per unit of work, so +// its default cadence spends at most a few credits an hour even misconfigured. +const searchIntervalMs = (Number(env['AUTHOR_SEARCH_INTERVAL_SECONDS']) || 900) * 1000; + // Accounts considered per alert pass, and how often a pass runs. Its own timer // for the same reason the card and cluster passes have one: a tick spends // minutes inside the crawl, and work queued behind that only happens if the @@ -590,6 +617,7 @@ async function enrichTick() { verify: authorVerify, recheckDays: authorRecheckDays, concurrency: authorConcurrency, + githubToken: authorGithubToken, onEvent: publishLog ? recorder.record : null, }); // Silent when nothing was due, which is the steady state once the @@ -602,6 +630,38 @@ async function enrichTick() { } } +/** + * Spend a little of the month's search allowance on the unreachable. + * + * Guarded twice over. It does nothing without a key, a budget and the switch; + * and `searchDue` re-reads the ledger every pass, so two pollers or a restarted + * one cannot between them spend more than the month allows. + */ +let searching = false; + +async function searchTick() { + if (!searchEnabled || !searchApiKey || searchMonthlyBudget <= 0) return; + if (searching) return; + searching = true; + + try { + const result = await searchDue(db, { + apiKey: searchApiKey, + monthlyBudget: searchMonthlyBudget, + perAuthor: searchPerAuthor, + batchSize: searchBatch, + }); + // Logged whenever anything was bought, even when it found nothing -- + // especially then. A pass that spends credits and stores no links is the + // signal that the gate needs tightening, and it is invisible otherwise. + if (result.spent) log('author-search', result); + } catch (err) { + log('author-search-error', { message: String(err?.message ?? err) }); + } finally { + searching = false; + } +} + /** * Write down how deep each queue is, so /crawlstats can draw the slope. * @@ -647,6 +707,7 @@ const cardTimer = setInterval(cardTick, cardIntervalMs); const alertTimer = setInterval(alertTick, alertIntervalMs); const enrichTimer = setInterval(enrichTick, authorIntervalMs); const queueTimer = setInterval(queueTick, queueSampleMs); +const searchTimer = setInterval(searchTick, searchIntervalMs); void tick(); void backfillTick(); void cardTick(); @@ -684,6 +745,7 @@ function shutdown(signal) { clearInterval(enrichTimer); clearInterval(alertTimer); clearInterval(queueTimer); + clearInterval(searchTimer); // Recorded before the buffer is closed, so the live log's last line is the // daemon saying it stopped rather than the log simply going quiet — which is diff --git a/apps/web/src/lib/jobs.js b/apps/web/src/lib/jobs.js index d41c979..ca6820c 100644 --- a/apps/web/src/lib/jobs.js +++ b/apps/web/src/lib/jobs.js @@ -102,6 +102,20 @@ export function jobRows({ // The one row where a backlog is a problem rather than a workload. alarmAbove: 500, }, + { + key: 'authors', + label: 'Author enrichment', + what: 'Finds who writes each feed, and how to reach them', + backlog: backlogs.authorsPending ?? 0, + // Genuinely a queue rather than a fire: it walks the directory once and + // then rechecks on a 90-day cycle, so a large number here is the work + // remaining and not a stall. + expectFull: true, + rate: backlogs.authorsLastHour ?? 0, + rateNote: 'publishers looked at', + done: `${backlogs.authorsDone ?? 0} looked at so far`, + events: ['author', 'authors-error', 'author-search', 'author-search-error'], + }, { key: 'cards', label: 'Feed pictures', diff --git a/apps/web/test/jobs.test.js b/apps/web/test/jobs.test.js index c3b686f..3231d37 100644 --- a/apps/web/test/jobs.test.js +++ b/apps/web/test/jobs.test.js @@ -21,6 +21,9 @@ function busy(overrides = {}) { cardsNone: 283, cardsError: 31, cardsLastHour: 1_320, + authorsPending: 81_123, + authorsDone: 3_275, + authorsLastHour: 283, ...overrides.backlogs, }, activity: { @@ -32,6 +35,7 @@ function busy(overrides = {}) { 'cluster-backfill': { lines: 360, errors: 0, amount: 12_000, lastAt: at, ms: 90 }, topics: { lines: 4, errors: 0, amount: 0, lastAt: at, ms: 300 }, alerts: { lines: 30, errors: 0, amount: 48, lastAt: at, ms: 600 }, + author: { lines: 283, errors: 4, amount: 60, lastAt: at, ms: 2_400 }, ...overrides.activity, }, alertAccounts: overrides.alertAccounts ?? 4, diff --git a/packages/db/migrations/20260819081824_author_searches.sql b/packages/db/migrations/20260819081824_author_searches.sql new file mode 100644 index 0000000..7174f64 --- /dev/null +++ b/packages/db/migrations/20260819081824_author_searches.sql @@ -0,0 +1,38 @@ +-- What we spent looking for people, and on whom. +-- +-- Author enrichment reads what publishers put where we could find it, and that +-- is free. Buying a search is not: the credits come from a metered account +-- shared with another product, and the directory is large enough that an +-- unbounded pass would spend a month's allowance in an afternoon. +-- +-- A counter in the process would not do. The poller restarts on every deploy, +-- and a budget that resets when the process does is not a budget — it is a +-- rate limit with a hole in it. So the spend is written down, and the ledger is +-- the authority. +-- +-- It is also the audit trail, which matters as much. "Which people did we spend +-- money looking for, and did it find anything" is a question worth being able +-- to answer, both to tune the gate that chooses them and to justify the line +-- item. +create table if not exists author_searches ( + id text primary key, + + -- Null once an author is deleted: the spend still happened, and dropping the + -- row would make the month's total disagree with what was billed. + author_id text references authors (id) on delete set null, + + at text not null, + + -- Credits actually spent, which is not the same as results found. A query + -- that returns nothing is still billed, and a budget that counted only hits + -- would overspend precisely on the people who are hardest to find. + queries integer not null default 0, + + -- Links stored as a result, so the gate can be judged on its yield rather + -- than on how it felt. + found integer not null default 0 +); + +-- The only query this table answers in the hot path: how much has been spent +-- since the start of the current billing period. +create index if not exists author_searches_at_idx on author_searches (at); diff --git a/packages/db/src/authors.js b/packages/db/src/authors.js index 64a3516..b33aa81 100644 --- a/packages/db/src/authors.js +++ b/packages/db/src/authors.js @@ -43,6 +43,46 @@ export async function dueForAuthors(db, limit = 25, recheckBefore = '') { return /** @type {any} */ (rows); } +/** + * Record that a feed's enrichment *failed*, so it is tried again sooner. + * + * `markAuthorsChecked` is right for a miss — a site that genuinely names nobody + * should not be re-read tomorrow — and wrong for a failure. They were the same + * call, which meant a DNS hiccup, a timeout or a 503 cost that publisher its + * enrichment for the **full recheck cycle**, ninety days, on the strength of one + * bad afternoon. On a pass that has so far reached 3,275 of 369,056 feeds, that + * is a quiet way to lose the ones on flaky hosts permanently. + * + * Done by back-dating the stamp rather than by adding an attempts column, and + * that is a deliberate trade. Writes on this database serialize and the crawl + * is already write-bound (see the notes in `crawl.js`), so the cheap fix that + * costs one UPDATE beats the tidy one that costs a migration and a second + * column to read. The feed still counts as "looked at" for the backlog, and + * still comes due again in `retryDays`. + * + * It must never back-date past *now*: a stamp in the future would hide the feed + * from a pass whose recheck window is shorter than this one's. + * + * @param {Client} db + * @param {string} feedId + * @param {{ retryDays?: number, recheckDays?: number }} [opts] + * @returns {Promise} + */ +export async function markAuthorsFailed(db, feedId, opts = {}) { + const retryDays = Math.max(0, Number(opts.retryDays ?? 3)); + const recheckDays = Math.max(retryDays, Number(opts.recheckDays ?? 90)); + + // Stamped as though it were checked (recheckDays - retryDays) ago, so the + // ordinary due test brings it back in retryDays without knowing why. + const backdated = (recheckDays - retryDays) * 86_400_000; + const at = new Date(Date.now() - backdated).toISOString(); + + await db.execute({ + sql: 'update feeds set authors_checked_at = ? where id = ?', + args: [at, feedId], + }); +} + /** * Record that a feed has been looked at, whether or not anyone was found. * @@ -812,3 +852,108 @@ export async function postsByAuthor(db, feedIds, limit = 12) { return rows; } + +/* ------------------------------------------------------------------ * + * Bought searches + * ------------------------------------------------------------------ */ + +/** + * The people it would be worth buying a search for. + * + * The gate is mean because the budget is small. What it selects for is the + * person we are confident *is* a person, who writes here, and whom nobody can + * currently contact — which is the only case where a paid query buys something + * the free sources could not. + * + * Ordered by how much they publish, so a limited budget is spent on the + * publishers a reader is most likely to want to reach. + * + * Anyone searched for already is excluded outright rather than re-searched on a + * schedule: a second query for somebody the web did not know about the first + * time is the easiest way to spend a month's credits on nothing. + * + * @param {Client} db + * @param {number} [limit] + * @param {number} [minConfidence] the floor the caller publishes at + * @returns {Promise} + */ +export async function authorsWithoutContact(db, limit = 10, minConfidence = 0.8) { + const { rows } = await db.execute({ + sql: `select a.id, a.slug, a.name, a.site_url as site, a.confidence, + count(distinct fa.feed_id) as feed_count + from authors a + join feed_authors fa on fa.author_id = a.id + where a.confidence >= ? + and not exists (select 1 from author_links l where l.author_id = a.id) + and not exists (select 1 from author_searches s where s.author_id = a.id) + -- A single word is not a searchable name: it returns the world. + and instr(trim(a.name), ' ') > 0 + group by a.id + order by feed_count desc, a.confidence desc + limit ?`, + args: [minConfidence, limit], + }); + + return rows; +} + +/** + * Write down what a search cost, whether or not it found anything. + * + * @param {Client} db + * @param {{ authorId: string|null, queries: number, found: number }} spend + * @returns {Promise} + */ +export async function recordAuthorSearch(db, spend) { + await db.execute({ + sql: `insert into author_searches (id, author_id, at, queries, found) + values (?, ?, ?, ?, ?)`, + args: [ + newId(), + spend.authorId ?? null, + nowIso(), + Math.max(0, Math.floor(Number(spend.queries) || 0)), + Math.max(0, Math.floor(Number(spend.found) || 0)), + ], + }); +} + +/** + * Credits spent since a moment, which is how much of the budget is gone. + * + * @param {Client} db + * @param {string} since ISO 8601 + * @returns {Promise} + */ +export async function searchSpendSince(db, since) { + const { rows } = await db.execute({ + sql: 'select coalesce(sum(queries), 0) as spent from author_searches where at >= ?', + args: [String(since)], + }); + + return Number(rows[0]?.spent ?? 0); +} + +/** + * The start of the current billing period. + * + * The provider's month does not begin on the first: ValueSERP resets this + * account's allowance on the **13th**, so a budget counted per calendar month + * would let the allowance be spent twice across a reset and refuse spending + * that is actually available just after one. + * + * @param {Date} [now] + * @param {number} [resetDay] + * @returns {string} ISO 8601 + */ +export function billingPeriodStart(now = new Date(), resetDay = 13) { + const at = new Date(now.getTime()); + const start = new Date( + Date.UTC(at.getUTCFullYear(), at.getUTCMonth(), resetDay, 0, 0, 0, 0), + ); + + // Before this month's reset day, the period began last month. + if (at.getTime() < start.getTime()) start.setUTCMonth(start.getUTCMonth() - 1); + + return start.toISOString(); +} diff --git a/packages/db/src/queries.js b/packages/db/src/queries.js index 53ecf01..7b8948f 100644 --- a/packages/db/src/queries.js +++ b/packages/db/src/queries.js @@ -1382,7 +1382,7 @@ export async function jobBacklogs(db) { // The fix is not "stop using conditional aggregates" — two of them survive // below. It is to make sure the scan they force is over a *covering index* // rather than over the table, at which point the same CASE is cheap. - const [byStatus, backlog, submitted, cards] = await Promise.all([ + const [byStatus, backlog, submitted, cards, enriched] = await Promise.all([ // One covering scan of feeds_status_success_idx (0028) answering three // questions at once: the status breakdown, and how many feeds in each state // have never once been read successfully. @@ -1410,6 +1410,20 @@ export async function jobBacklogs(db) { from feeds group by card_state`, args: [hourAgo], }), + + // How far the author enrichment has walked, read off the partial index + // 0024 already built for it (`feeds (authors_checked_at) where status = + // 'active'`). Counted as the *stamped* set rather than the unstamped one + // for the reason this whole function exists: 3,275 of 369,056 feeds carry a + // stamp, so this touches a few thousand index entries, while asking for the + // complement would visit every row. The backlog is arithmetic afterwards. + db.execute({ + sql: `select count(*) as n, + sum(case when authors_checked_at >= ? then 1 else 0 end) as hour + from feeds + where status = 'active' and authors_checked_at is not null`, + args: [hourAgo], + }), ]); // Deliberately no "first crawls completed this hour". Nothing records when a @@ -1443,6 +1457,9 @@ export async function jobBacklogs(db) { cardsNone: card('none'), cardsError: card('error'), cardsLastHour: cards.rows.reduce((a, r) => a + Number(r.hour ?? 0), 0), + authorsDone: Number(enriched.rows[0]?.n ?? 0), + authorsPending: Math.max(0, n('active') - Number(enriched.rows[0]?.n ?? 0)), + authorsLastHour: Number(enriched.rows[0]?.hour ?? 0), }; } diff --git a/packages/feed/index.js b/packages/feed/index.js index 96ceaca..419898e 100644 --- a/packages/feed/index.js +++ b/packages/feed/index.js @@ -67,6 +67,7 @@ export { feedContacts, feedCredits, identityFromHtml, + identityFromHumansTxt, identityKey, isRoleEmail, linksBackTo, @@ -78,3 +79,4 @@ export { personalEmail, splitBylines, } from './src/identity.js'; +export { hostIdentity, identityFromProfile, profileRequest } from './src/platforms.js'; diff --git a/packages/feed/src/fetch.js b/packages/feed/src/fetch.js index f2cf3ca..29fc5fb 100644 --- a/packages/feed/src/fetch.js +++ b/packages/feed/src/fetch.js @@ -102,7 +102,8 @@ export async function isPublicHost(hostname) { * keep revalidating against a stale one for ever. * * @param {string} url - * @param {{ etag?: string|null, lastModified?: string|null }} [conditional] + * @param {{ etag?: string|null, lastModified?: string|null, + * headers?: Record }} [conditional] * @returns {Promise<{ ok: boolean, status: number, contentType: string, body: string, url: string, notModified?: boolean, etag?: string|null, lastModified?: string|null, error?: string }>} */ export async function safeFetch(url, conditional = {}) { @@ -132,6 +133,18 @@ export async function safeFetch(url, conditional = {}) { if (conditional?.etag) headers['if-none-match'] = String(conditional.etag); if (conditional?.lastModified) headers['if-modified-since'] = String(conditional.lastModified); + // Caller-supplied headers, for the JSON APIs the author enrichment asks + // about a person. They go through this function rather than around it so + // an API host is still checked against the private-address guard and still + // bounded by the same timeout -- a profile URL is built from a hostname we + // read out of somebody else's feed, so it is exactly as untrusted as a + // page URL. The user-agent stays ours and cannot be overridden. + for (const [key, value] of Object.entries(conditional?.headers ?? {})) { + const name = String(key).toLowerCase(); + if (name === 'user-agent') continue; + headers[name] = String(value); + } + const res = await fetch(normalized, { headers, redirect: 'follow', diff --git a/packages/feed/src/identity.js b/packages/feed/src/identity.js index c8811c7..9b3f8c4 100644 --- a/packages/feed/src/identity.js +++ b/packages/feed/src/identity.js @@ -1035,6 +1035,223 @@ export function identityFromHtml(html, baseUrl = '') { }; } +/** + * `humans.txt`, the file whose entire purpose is to name the people. + * + * Worth a request precisely because of what it is. Every other source here is + * markup that happens to carry identity as a side effect — a link with a `rel` + * attribute, a microformat class, a byline in a feed. `humans.txt` is a + * convention with one job: the author writing it was answering the question + * "who made this", which is the question being asked. + * + * The format is loose and its shape is inverted from everything else. The + * *key* is the role and the *value* is the person: + * + * /* TEAM *\/ + * Chef: Jane Doe + * Site: https://jane.example + * Mastodon: @jane@example.social + * + * So a person is a block: a naming line, then the contact lines that follow it + * until the next person or the next section. Attaching links to the nearest + * preceding name is the whole parsing job, and getting it wrong on a two-person + * file would give one of them the other's accounts. + * + * Only `/* TEAM *\/` and its unlabelled equivalent are read. `/* THANKS *\/` + * exists to credit other people's work — libraries, inspirations, a designer at + * another company — and treating those as this feed's authors would attribute a + * blog to whoever its author admires. + * + * @param {string} text the file, as served + * @param {string} baseUrl for resolving relative links + * @returns {{ credits: Credit[], profiles: Array<{ network: string, url: string, handle: string, source: string }> }} + */ +export function identityFromHumansTxt(text, baseUrl = '') { + const empty = { credits: [], profiles: [] }; + if (typeof text !== 'string' || !text.trim()) return empty; + + // A server that answers every path with its 404 page is common enough that + // this has to be checked: HTML here is not a humans.txt, it is a miss. + if (/^\s*<(?:!doctype|html)/i.test(text)) return empty; + + /** Keys whose value is a person's name. */ + const NAMES = new Set([ + 'name', + 'chef', + 'developer', + 'developers', + 'designer', + 'author', + 'owner', + 'maintainer', + 'engineer', + 'writer', + 'creator', + 'programmer', + ]); + + /** Keys whose value is somewhere to reach that person. */ + const LINKS = new Set([ + 'site', + 'website', + 'url', + 'homepage', + 'blog', + 'twitter', + 'x', + 'mastodon', + 'fediverse', + 'github', + 'gitlab', + 'codeberg', + 'linkedin', + 'instagram', + 'bluesky', + 'contact', + 'email', + 'e-mail', + 'mail', + ]); + + const credits = []; + const profiles = []; + /** @type {Credit|null} the person the following contact lines belong to */ + let current = null; + let inTeam = true; + + for (const rawLine of text.split(/\r?\n/)) { + const line = rawLine.trim(); + + // A section header both switches context and ends the current person. + const section = line.match(/^\/\*+\s*(.+?)\s*\*+\//); + if (section) { + inTeam = /team|staff|people|author|owner/i.test(section[1]); + current = null; + continue; + } + + if (!line) { + current = null; + continue; + } + if (!inTeam) continue; + + const pair = line.match(/^([A-Za-z][\w -]{0,24})\s*:\s*(.+)$/); + if (!pair) continue; + + const key = pair[1].trim().toLowerCase(); + const value = pair[2].trim(); + + if (NAMES.has(key)) { + const name = cleanName(value); + // The role filters apply here exactly as they do to a byline: "Developer: + // the web team" names nobody, and a file that says so must not produce a + // person called "the web team". + current = looksLikePersonName(name) + ? credit({ name, role: 'author', source: 'humans-txt', confidence: 0.7, base: baseUrl }) + : null; + if (current) credits.push(current); + continue; + } + + if (!LINKS.has(key)) continue; + + // A bare handle is not a URL, and which platform it belongs to is exactly + // what the key just said. + const expanded = expandHandle(key, value); + + // `Site: https://jane.example` is the most useful line in the file and + // matches no profile shape, so classifyLink returns null for it — rightly, + // since it refuses to file arbitrary links as somebody's website. Here the + // key has already said that is what this is, so the fallback is safe and is + // limited to the keys that said it. + const link = classifyLink(expanded, baseUrl) ?? homepage(expanded, key); + if (!link) continue; + + profiles.push({ ...link, source: 'humans-txt' }); + + // Attach to the person this block is about, so a two-person file does not + // hand one of them the other's accounts. A link before any name belongs to + // the site, which is what `feed_links` is for, and the caller stores it + // there. + if (!current) continue; + if (link.network === 'email' && !current.email) current.email = link.handle; + if (link.network === 'website' && !current.url) current.url = link.url; + } + + return { credits, profiles }; +} + +/** + * A homepage line from a humans.txt, as a link row. + * + * Only for the keys that name a site. Everything else that fails to classify is + * genuinely unrecognised and is dropped, which is what keeps a `Standards:` or + * `Language:` line from becoming a link. + * + * @param {string} url already expanded to an absolute URL + * @param {string} key the humans.txt key it came from + * @returns {{ network: string, url: string, handle: string }|null} + */ +function homepage(url, key) { + if (!['site', 'website', 'url', 'homepage', 'blog'].includes(key)) return null; + + const normalized = normalizeIdentityUrl(url); + if (!normalized) return null; + + try { + return { network: 'website', url: normalized, handle: new URL(normalized).hostname }; + } catch { + return null; + } +} + +/** + * A handle written the way people write them in a humans.txt, as a URL. + * + * `Twitter: @jane` is the common form and is not a link until it is made one. + * A value that is already a URL is left alone. + * + * @param {string} key the platform, which the key names + * @param {string} value + * @returns {string} + */ +function expandHandle(key, value) { + const raw = value.trim(); + if (/^(?:https?:|mailto:|xmpp:|nostr:)/i.test(raw)) return raw; + if (/^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(raw)) return `mailto:${raw}`; + + const handle = raw.replace(/^@/, ''); + if (!handle) return ''; + + switch (key) { + case 'twitter': + case 'x': + return `https://x.com/${handle}`; + case 'github': + return `https://github.com/${handle}`; + case 'gitlab': + return `https://gitlab.com/${handle}`; + case 'codeberg': + return `https://codeberg.org/${handle}`; + case 'linkedin': + return `https://www.linkedin.com/in/${handle}`; + case 'instagram': + return `https://instagram.com/${handle}`; + case 'bluesky': + return `https://bsky.app/profile/${handle}`; + case 'mastodon': + case 'fediverse': { + // `@jane@example.social` is the only form that names its own host. + const parts = handle.split('@'); + return parts.length === 2 ? `https://${parts[1]}/@${parts[0]}` : ''; + } + default: + // site/url/homepage/blog, written without a scheme. + return /^[\w.-]+\.[a-z]{2,}(?:\/|$)/i.test(handle) ? `https://${handle}` : ''; + } +} + /** * The links on a bio page, which all belong to whoever owns the page. * diff --git a/packages/feed/src/platforms.js b/packages/feed/src/platforms.js new file mode 100644 index 0000000..863f23a --- /dev/null +++ b/packages/feed/src/platforms.js @@ -0,0 +1,522 @@ +/* + * The identity a publisher never wrote down. + * + * `identity.js` reads what a page *says*: rel="me", an h-card, a JSON-LD + * `sameAs`, a footer link. That works beautifully on the part of the small web + * that marks itself up, and it finds exactly nothing on the part that does not. + * + * The case that made this necessary is the ordinary one. `felginep.github.io` + * publishes a blog with no rel="me", no h-card, and one outbound link — to the + * Jekyll theme its author happened to use. The old pass fetched the page, + * correctly found nobody, and stamped the feed as checked. Meanwhile the + * author's GitHub account was named *in the hostname the feed was served from*, + * and the profile behind it confirms his real name. + * + * So this module reads two things nobody had to publish: + * + * - **The host.** A blog on `.github.io`, `.substack.com` or + * `medium.com/@` names its owner's account in the address. That costs + * no request at all — it is a fact about the URL we already have. + * - **The profile behind it.** Given the account, its platform will describe + * the person: GitHub returns a real name, a homepage and sometimes a public + * email; GitLab returns a name and a public email without a token, and its + * contact fields with one; a Mastodon account returns its profile fields + * *with the instance's own rel="me" verification already done for us*. + * + * **Pure by construction.** Nothing here fetches. The functions describe a + * request (`profileRequest`) and read a response (`identityFromProfile`), which + * is the same split `identity.js` keeps and the reason both can be tested + * without a network. `enrich.js` owns the sockets. + * + * **The confidence rule.** A host-derived account is evidence, not proof: + * `blog.example.com` on a shared host is not a person, and a project site can + * live under an organisation's `github.io`. So a derivation is only made where + * the platform gives each *user* their own subdomain, and the account it + * proposes is confirmed the moment the profile answers with a matching name. + * Anything unconfirmed stays below the publishing floor. + */ + +/** + * Platforms that put a user's handle in the hostname. + * + * Only the ones where a subdomain is *a person's account* rather than a site + * somebody happens to host. `.wordpress.com` and `.blogspot.com` + * are deliberately absent: they name a blog, and we already have the blog — + * deriving "a website link to the feed we are enriching" is a row that teaches + * nobody anything. + * + * `reserved` are the subdomains the platform itself uses, which are never + * people. Without it, `www.github.io` and `pages.github.com` become authors. + */ +const HOST_ACCOUNTS = [ + { + network: 'github', + host: /^([a-z0-9](?:[a-z0-9-]{0,38}))\.github\.io$/i, + profile: (handle) => `https://github.com/${handle}`, + }, + { + network: 'gitlab', + host: /^([a-z0-9][\w.-]{1,60})\.gitlab\.io$/i, + profile: (handle) => `https://gitlab.com/${handle}`, + }, + { + network: 'codeberg', + host: /^([a-z0-9][\w-]{0,38})\.codeberg\.page$/i, + profile: (handle) => `https://codeberg.org/${handle}`, + }, + { + network: 'sourcehut', + host: /^([\w.-]{1,64})\.srht\.site$/i, + profile: (handle) => `https://git.sr.ht/~${handle}`, + }, + { + network: 'substack', + host: /^([\w-]{1,60})\.substack\.com$/i, + profile: (handle) => `https://${handle}.substack.com`, + }, + { + network: 'medium', + host: /^([\w-]{1,60})\.medium\.com$/i, + profile: (handle) => `https://medium.com/@${handle}`, + }, + { + network: 'tumblr', + host: /^([\w-]{1,32})\.tumblr\.com$/i, + profile: (handle) => `https://${handle}.tumblr.com`, + }, + { + network: 'microblog', + host: /^([\w.-]{1,60})\.micro\.blog$/i, + profile: (handle) => `https://micro.blog/${handle}`, + }, + { + network: 'bearblog', + host: /^([\w-]{1,60})\.bearblog\.dev$/i, + profile: (handle) => `https://bearblog.dev/${handle}`, + }, +]; + +/** + * Path-shaped platforms, where the handle is the first segment. + * + * Same idea as above for hosts that give everybody one domain. Only applied to + * the *feed's own* address, never to an arbitrary link — `classifyLink` already + * reads links, and this is about the address the feed itself lives at. + */ +const PATH_ACCOUNTS = [ + { network: 'medium', host: /^(?:www\.)?medium\.com$/i, path: /^\/@([\w.-]{1,60})(?:\/|$)/ }, + { network: 'devto', host: /^dev\.to$/i, path: /^\/([\w-]{1,60})(?:\/|$)/ }, + { network: 'hashnode', host: /^hashnode\.com$/i, path: /^\/@([\w-]{1,60})(?:\/|$)/ }, + { network: 'microblog', host: /^(?:www\.)?micro\.blog$/i, path: /^\/([\w.-]{1,60})(?:\/|$)/ }, + { network: 'substack', host: /^(?:www\.)?substack\.com$/i, path: /^\/@([\w-]{1,60})(?:\/|$)/ }, +]; + +/** + * Subdomains that belong to the platform rather than to a person. + * + * Kept deliberately short. A false *positive* here costs an author page for a + * person who does not exist, which is the expensive mistake; a false negative + * costs one blogger a link we would have found on their page anyway. + */ +const RESERVED_HANDLES = new Set([ + 'www', + 'blog', + 'docs', + 'help', + 'api', + 'status', + 'about', + 'support', + 'mail', + 'admin', + 'pages', + 'app', + 'static', + 'assets', + 'cdn', + 'media', + 'news', + 'test', + 'demo', + 'example', +]); + +/** + * Accounts named by the addresses a feed already has. + * + * Costs no request: this is arithmetic on strings we were given. Both the feed + * URL and the site URL are read, because they disagree often — a blog on its + * own domain whose feed is proxied through Substack names its author in the + * feed URL and nowhere else. + * + * @param {...unknown} urls the feed URL, the site URL, in any order + * @returns {Array<{ network: string, url: string, handle: string, source: string, confidence: number }>} + */ +export function hostIdentity(...urls) { + /** @type {Map} */ + const found = new Map(); + + for (const raw of urls) { + let parsed; + try { + parsed = new URL(String(raw ?? '')); + } catch { + continue; + } + if (!/^https?:$/.test(parsed.protocol)) continue; + + const host = parsed.hostname.toLowerCase(); + + for (const rule of HOST_ACCOUNTS) { + const match = host.match(rule.host); + if (!match) continue; + add(found, rule.network, match[1], rule.profile); + break; + } + + for (const rule of PATH_ACCOUNTS) { + if (!rule.host.test(host)) continue; + const match = parsed.pathname.match(rule.path); + if (!match) continue; + add(found, rule.network, match[1], (handle) => + rule.network === 'medium' + ? `https://medium.com/@${handle}` + : `https://${host}/${rule.network === 'hashnode' || rule.network === 'substack' ? '@' : ''}${handle}`, + ); + break; + } + } + + return [...found.values()]; +} + +/** + * @param {Map} found + * @param {string} network + * @param {string} rawHandle + * @param {(handle: string) => string} profile + */ +function add(found, network, rawHandle, profile) { + const handle = String(rawHandle ?? '').toLowerCase(); + if (!handle || RESERVED_HANDLES.has(handle)) return; + + const url = profile(handle); + const key = `${network}:${handle}`; + if (found.has(key)) return; + + found.set(key, { + network, + url, + handle, + source: 'host-derived', + // Below the 0.6 publishing floor on purpose. The hostname is a strong hint + // and not a fact: it becomes one when `identityFromProfile` finds a profile + // that agrees, and that is what raises it. + confidence: 0.5, + }); +} + +/** + * The platforms that will describe a person if asked, and how to ask. + * + * Only APIs that answer unauthenticated, return JSON, and publish a *person* + * rather than a repository. Bluesky and LinkedIn are absent for the reason they + * are absent from the rel="me" verification list — one renders client-side and + * the other refuses robots outright. + */ +const PROFILE_APIS = { + github: (handle) => `https://api.github.com/users/${encodeURIComponent(handle)}`, + gitlab: (handle) => `https://gitlab.com/api/v4/users?username=${encodeURIComponent(handle)}`, + codeberg: (handle) => `https://codeberg.org/api/v1/users/${encodeURIComponent(handle)}`, +}; + +/** + * How to ask a platform about one of its people. + * + * Returns null for a link no API here can resolve, which is most of them — the + * caller treats that as "nothing more to learn from this one" rather than as an + * error. + * + * `token` is the caller's GitHub credential, and it matters more than it looks: + * **the unauthenticated GitHub API allows 60 requests an hour per IP**, which is + * roughly one enrichment batch. With a token it is 5,000. A pass over this + * directory without one is not slow, it is stopped. + * + * @param {{ network: string, handle?: string, url?: string }} link + * @param {{ token?: string }} [opts] + * @returns {{ url: string, network: string, headers: Record }|null} + */ +export function profileRequest(link, opts = {}) { + const network = String(link?.network ?? ''); + const handle = String(link?.handle ?? '').replace(/^[@~]/, ''); + if (!handle) return null; + + // The fediverse has no host list, so its endpoint is derived from the account + // rather than looked up: `@user@host` is served by `host`. + if (network === 'fediverse') { + const parts = handle.replace(/^@/, '').split('@'); + if (parts.length !== 2 || !parts[0] || !parts[1]) return null; + return { + network, + url: `https://${parts[1]}/api/v1/accounts/lookup?acct=${encodeURIComponent(parts[0])}`, + headers: { accept: 'application/json' }, + }; + } + + const build = PROFILE_APIS[network]; + if (!build) return null; + + /** @type {Record} */ + const headers = { accept: 'application/json' }; + + if (network === 'github') { + headers.accept = 'application/vnd.github+json'; + if (opts.token) headers.authorization = `Bearer ${opts.token}`; + } + + return { network, url: build(handle), headers }; +} + +/** + * Read a profile document into the credits and links it implies. + * + * Each platform is mapped explicitly rather than by a shared field list, + * because the one field that matters is different on each: GitHub calls the + * homepage `blog`, GitLab calls it `website_url` and publishes the email as + * `public_email`, and Mastodon does not have named fields at all — it has an + * array the user filled in themselves, of which the *verified* entries are + * worth more than anything else this module can find. + * + * `kind` is the one field the caller must not ignore. GitHub and Gitea both + * serve organisations from the same endpoint as people, so a project site on + * `someproject.github.io` resolves to an account whose "name" is a product. + * Turning that into an author row would publish a company as a person, which is + * the one mistake this directory has been careful not to make since the role + * filters went in. + * + * @param {string} network + * @param {unknown} body the parsed JSON + * @param {{ handle?: string, url?: string }} [link] the account this describes + * @returns {{ name: string, bio: string, avatar: string, kind: string, links: Array }} + */ +export function identityFromProfile(network, body, link = {}) { + const empty = { name: '', bio: '', avatar: '', kind: 'unknown', links: [] }; + if (!body || typeof body !== 'object') return empty; + + switch (network) { + case 'github': + return fromGithub(/** @type {any} */ (body)); + case 'gitlab': + // The users endpoint answers with an array, because it is a search. + return fromGitlab(Array.isArray(body) ? body[0] : body); + case 'codeberg': + return fromCodeberg(/** @type {any} */ (body)); + case 'fediverse': + return fromMastodon(/** @type {any} */ (body), link); + default: + return empty; + } +} + +/** + * GitHub. + * + * `email` is only ever populated when the account holder ticked the box that + * publishes it, so it is a contact address they chose to make public rather + * than one scraped out of commit metadata — which is a different thing with + * different rules, and is not collected here. + * + * @param {any} p + */ +function fromGithub(p) { + const links = []; + if (p.blog) links.push(profileLink('website', p.blog, 'github-profile')); + if (p.email) links.push(profileLink('email', `mailto:${p.email}`, 'github-profile')); + if (p.twitter_username) { + links.push(profileLink('twitter', `https://x.com/${p.twitter_username}`, 'github-profile')); + } + + return { + name: str(p.name), + bio: str(p.bio), + avatar: str(p.avatar_url), + // "User" or "Organization"; anything else is a shape we do not know and is + // treated as unknown rather than assumed to be a person. + kind: p.type === 'Organization' ? 'org' : p.type === 'User' ? 'user' : 'unknown', + links: links.filter(Boolean), + }; +} + +/** + * GitLab. + * + * Measured rather than assumed: the *unauthenticated* `/users?username=` search + * answers a reduced object — `name`, `public_email`, `avatar_url`, `web_url` + * and nothing else. The richer profile (`website_url`, `bio`, `twitter`, + * `linkedin`) needs a token, and `/users/:id` refuses outright without one. The + * mappings below cover both shapes, so a deployment that sets a GitLab token + * gets the contact fields and one that does not still gets a name and a public + * email, which is the part that matters. + * + * @param {any} p + */ +function fromGitlab(p) { + if (!p || typeof p !== 'object') return { name: '', bio: '', avatar: '', kind: 'unknown', links: [] }; + + const links = []; + if (p.website_url) links.push(profileLink('website', p.website_url, 'gitlab-profile')); + if (p.public_email) links.push(profileLink('email', `mailto:${p.public_email}`, 'gitlab-profile')); + if (p.twitter) links.push(profileLink('twitter', `https://x.com/${strip(p.twitter)}`, 'gitlab-profile')); + if (p.linkedin) { + links.push(profileLink('linkedin', `https://www.linkedin.com/in/${strip(p.linkedin)}`, 'gitlab-profile')); + } + + return { + name: str(p.name), + bio: str(p.bio), + avatar: str(p.avatar_url), + // GitLab groups are not returned by the user search at all, so anything + // that answers here is a user account. + kind: 'user', + links: links.filter(Boolean), + }; +} + +/** + * Codeberg, which runs Gitea and answers the same shape. + * + * @param {any} p + */ +function fromCodeberg(p) { + const links = []; + if (p.website) links.push(profileLink('website', p.website, 'codeberg-profile')); + if (p.email) links.push(profileLink('email', `mailto:${p.email}`, 'codeberg-profile')); + + return { + name: str(p.full_name), + bio: str(p.description), + avatar: str(p.avatar_url), + // Gitea marks organisations with a user_type of 1. + kind: Number(p.user_type) === 1 ? 'org' : 'user', + links: links.filter(Boolean), + }; +} + +/** + * A fediverse account, and the best source in this module. + * + * Mastodon's profile `fields` are free-form rows the account holder wrote, and + * an entry carries `verified_at` when the instance followed the link and found + * a rel="me" pointing back. That is the IndieWeb handshake **already performed + * by somebody else** — the same proof `enrichFeedAuthors` spends up to three + * fetches on, arriving for free with the profile. + * + * So a verified field is marked `verified` and nothing else here is. + * + * @param {any} p + * @param {{ url?: string }} link + */ +function fromMastodon(p, link) { + const links = []; + + for (const field of Array.isArray(p.fields) ? p.fields : []) { + // The value is HTML — Mastodon renders the link itself — so the href is + // what to read, falling back to the text for an entry typed as bare text. + const href = String(field?.value ?? '').match(/href="([^"]+)"/)?.[1] ?? String(field?.value ?? ''); + const url = href.replace(/<[^>]*>/g, '').trim(); + if (!url) continue; + + links.push({ + url, + source: 'fediverse-profile', + verified: Boolean(field?.verified_at), + // A field the instance proved is as good as evidence gets short of the + // person telling us themselves. + confidence: field?.verified_at ? 0.95 : 0.5, + }); + } + + if (p.url && !links.some((l) => l.url === p.url)) { + links.push({ url: String(p.url), source: 'fediverse-profile', verified: false, confidence: 0.6 }); + } + + return { + name: str(p.display_name), + kind: p.group ? 'org' : 'user', + // The bio is HTML on this platform and prose everywhere else. + bio: plain(p.note), + avatar: str(p.avatar_static || p.avatar), + links, + }; +} + +/** + * @param {string} network + * @param {unknown} url + * @param {string} source + */ +function profileLink(network, url, source) { + const value = str(url); + if (!value) return null; + return { + network, + // A platform field is frequently typed without a scheme ("example.com"), + // which is not a URL until it has one. + url: /^[a-z][a-z0-9+.-]*:/i.test(value) ? value : `https://${value}`, + source, + verified: false, + // Published by the person on an account we already believe is theirs, which + // is above the floor but short of a proved backlink. + confidence: 0.8, + }; +} + +/** + * HTML as the prose it was written as. + * + * Only the fediverse needs this — its bio field is rendered markup rather than + * text. Which tags become a space and which vanish is the whole job: replacing + * *every* tag with a space turns "InfoSec." into "InfoSec ." (the test + * that found this was right to), while replacing every tag with nothing joins + * two paragraphs into one word. So the block-level tags separate and the inline + * ones close up, which is what they mean. + * + * Entities are decoded too, because a bio is displayed to a reader and + * "Tom & Jerry" is not what anyone wrote. + * + * @param {unknown} html + * @returns {string} + */ +function plain(html) { + return String(html ?? '') + .replace(/<\/(?:p|div|li|h[1-6])>|/gi, ' ') + .replace(/<[^>]*>/g, '') + .replace(/&(nbsp|amp|lt|gt|quot|#39|apos);/gi, (_, name) => { + const map = { + nbsp: ' ', + amp: '&', + lt: '<', + gt: '>', + quot: '"', + '#39': "'", + apos: "'", + }; + return map[String(name).toLowerCase()] ?? _; + }) + .replace(/\s+/g, ' ') + .trim(); +} + +/** @param {unknown} v */ +function str(v) { + return v == null ? '' : String(v).trim(); +} + +/** @param {unknown} v handle fields that arrive with an @ or a full URL */ +function strip(v) { + return String(v ?? '') + .trim() + .replace(/^@/, '') + .replace(/^https?:\/\/[^/]+\/(?:in\/)?/i, '') + .replace(/\/+$/, ''); +} diff --git a/packages/feed/test/humans.test.js b/packages/feed/test/humans.test.js new file mode 100644 index 0000000..da2dbe5 --- /dev/null +++ b/packages/feed/test/humans.test.js @@ -0,0 +1,108 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { identityFromHumansTxt } from '../src/identity.js'; + +// humans.txt is the one source here written to answer the question we are +// asking, so the parsing has to be right about two things: whose links are +// whose, and which section of the file is talking about this site's people +// rather than about somebody else's work. + +const TEAM = `/* TEAM */ +Chef: Jane Doe +Site: https://jane.example +Mastodon: @jane@example.social +Contact: jane@example.com + +Developer: Bob Smith +Twitter: @bobsmith + +/* THANKS */ +Inspiration: Famous Person +Site: https://famous.example + +/* SITE */ +Standards: HTML5 +Language: English +`; + +test('each person keeps their own accounts', () => { + // The failure this guards against is quiet and wrong rather than empty: a + // parser that collects links globally hands Jane's Mastodon to Bob. + const { credits, profiles } = identityFromHumansTxt(TEAM, 'https://blog.example/'); + + assert.deepEqual( + credits.map((c) => c.name), + ['Jane Doe', 'Bob Smith'], + ); + + const jane = credits[0]; + assert.equal(jane.email, 'jane@example.com'); + assert.equal(jane.url, 'https://jane.example/'); + assert.equal(credits[1].email, '', 'Bob published no address and must not inherit one'); + + assert.deepEqual( + profiles.map((p) => p.network).sort(), + ['email', 'fediverse', 'twitter', 'website'], + ); + assert.ok(profiles.every((p) => p.source === 'humans-txt')); +}); + +test('the THANKS section is not this blog\'s authors', () => { + // It exists to credit other people's work — a library, an inspiration, a + // designer at another company. Reading it as authorship would attribute a + // blog to whoever its author admires. + const { credits } = identityFromHumansTxt(TEAM, 'https://blog.example/'); + assert.ok(!credits.some((c) => c.name === 'Famous Person')); +}); + +test('lines that describe the site are not links', () => { + const { profiles } = identityFromHumansTxt(TEAM, 'https://blog.example/'); + assert.ok(!profiles.some((p) => /HTML5|English/i.test(p.url))); +}); + +test('a role is still not a person here', () => { + // The same rule that governs a byline. A file can say "Developer: the web + // team", and that names nobody. + const { credits } = identityFromHumansTxt( + '/* TEAM */\nDeveloper: the web team\nName: Editor\nContact: info@example.com', + 'https://blog.example/', + ); + assert.deepEqual(credits, []); +}); + +test('a bare handle becomes the account the key says it is', () => { + const { profiles } = identityFromHumansTxt( + '/* TEAM */\nName: Ann Example\nGithub: annex\nBluesky: ann.example\nLinkedin: ann-example', + 'https://blog.example/', + ); + + assert.deepEqual( + profiles.map((p) => p.url), + [ + 'https://github.com/annex', + 'https://bsky.app/profile/ann.example', + 'https://www.linkedin.com/in/ann-example', + ], + ); +}); + +test('a 404 page dressed as a text file is not a humans.txt', () => { + // Plenty of servers answer every path with their HTML 404. Parsing that would + // turn a stylesheet reference into somebody's website. + assert.deepEqual(identityFromHumansTxt('Not found'), { + credits: [], + profiles: [], + }); + assert.deepEqual(identityFromHumansTxt(''), { credits: [], profiles: [] }); + assert.deepEqual(identityFromHumansTxt(null), { credits: [], profiles: [] }); +}); + +test('a file with no sections at all is read as a team', () => { + // The convention is loose and plenty of files skip the header entirely. + const { credits } = identityFromHumansTxt('Name: Ann Example\nSite: https://ann.example'); + assert.deepEqual( + credits.map((c) => c.name), + ['Ann Example'], + ); +}); diff --git a/packages/feed/test/platforms.test.js b/packages/feed/test/platforms.test.js new file mode 100644 index 0000000..c6f031d --- /dev/null +++ b/packages/feed/test/platforms.test.js @@ -0,0 +1,157 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { hostIdentity, identityFromProfile, profileRequest } from '../src/platforms.js'; + +// What is tested here is the half of author enrichment that reads what nobody +// published: the account named by a hostname, and the profile behind it. The +// expensive mistake in this direction is not missing somebody, it is inventing +// them -- so most of these are about what must *not* be derived. + +test('a blog on a personal platform names its owner in the hostname', () => { + // The case this module exists for. felginep.github.io publishes no rel="me", + // no h-card and one outbound link, to its Jekyll theme -- so every other + // source finds nobody, while the account is sitting in the address. + assert.deepEqual(hostIdentity('https://felginep.github.io/feed.xml'), [ + { + network: 'github', + url: 'https://github.com/felginep', + handle: 'felginep', + source: 'host-derived', + confidence: 0.5, + }, + ]); +}); + +test('the feed URL and the site URL are both read, and agree on one account', () => { + // They disagree often -- a blog on its own domain with a feed proxied through + // a platform names its author in only one of them -- but when both name the + // same account it must not be stored twice. + const found = hostIdentity('https://jane.substack.com/feed', 'https://jane.substack.com/'); + assert.equal(found.length, 1); + assert.equal(found[0].network, 'substack'); + assert.equal(found[0].handle, 'jane'); +}); + +test("a platform's own subdomains are not people", () => { + for (const url of [ + 'https://www.github.io/', + 'https://docs.github.io/', + 'https://blog.substack.com/', + 'https://api.micro.blog/', + ]) { + assert.deepEqual(hostIdentity(url), [], url); + } +}); + +test('a host that is nobody in particular derives nothing', () => { + // The overwhelming majority of the directory. Deriving a "website" link back + // to the feed we are already enriching would be a row that teaches nobody + // anything, so a plain domain yields nothing at all. + assert.deepEqual(hostIdentity('https://kevquirk.com/feed'), []); + assert.deepEqual(hostIdentity('https://example.wordpress.com/feed'), []); + assert.deepEqual(hostIdentity('not a url'), []); + assert.deepEqual(hostIdentity('ftp://example.github.io/'), []); +}); + +test('a derived account stays below the publishing floor until something confirms it', () => { + // 0.6 is what /api/authors publishes at. A hostname is a strong hint and not + // a fact: it is the profile answering that turns it into one. + const [account] = hostIdentity('https://felginep.github.io/'); + assert.ok(account.confidence < 0.6, 'a bare derivation must not be publishable on its own'); +}); + +test('a GitHub profile is read for the fields a person filled in', () => { + const profile = identityFromProfile('github', { + type: 'User', + name: 'Pierre Felgines', + bio: 'iOS developer', + avatar_url: 'https://avatars.example/u/1', + blog: 'felginep.github.io', + email: 'pierre@example.com', + twitter_username: 'pfelgines', + }); + + assert.equal(profile.name, 'Pierre Felgines'); + assert.equal(profile.kind, 'user'); + assert.deepEqual( + profile.links.map((l) => [l.network, l.url]), + [ + // A profile field typed without a scheme is not a URL until it has one. + ['website', 'https://felginep.github.io'], + ['email', 'mailto:pierre@example.com'], + ['twitter', 'https://x.com/pfelgines'], + ], + ); +}); + +test('an organisation is not a person', () => { + // jekyll.github.io resolves to an account whose "name" is a product. Turning + // that into an author row would publish a project as a human being. + const profile = identityFromProfile('github', { type: 'Organization', name: 'Jekyll' }); + assert.equal(profile.kind, 'org'); +}); + +test("a fediverse profile carries the instance's own rel=me verification", () => { + // The best source in the module: the instance already followed the link and + // found a backlink, so the IndieWeb handshake arrives done. + const profile = identityFromProfile('fediverse', { + display_name: 'Kev Quirk', + note: '

I work in InfoSec.

', + url: 'https://fosstodon.org/@kev', + avatar_static: 'https://cdn.example/a.png', + fields: [ + { + name: 'Blog', + value: 'kevquirk.com', + verified_at: '2022-10-22T22:21:08.251+00:00', + }, + { name: 'Unproven', value: 'example.org', verified_at: null }, + ], + }); + + assert.equal(profile.name, 'Kev Quirk'); + // The bio is HTML on this platform and prose everywhere else. + assert.equal(profile.bio, 'I work in InfoSec.'); + + const verified = profile.links.find((l) => l.url === 'https://kevquirk.com'); + assert.equal(verified.verified, true); + assert.ok(verified.confidence > 0.9); + + const unproven = profile.links.find((l) => l.url === 'https://example.org'); + assert.equal(unproven.verified, false, 'an unverified field is stored, but not as proof'); +}); + +test('a request is described for the platforms that answer, and refused for the rest', () => { + assert.equal( + profileRequest({ network: 'github', handle: 'felginep' }).url, + 'https://api.github.com/users/felginep', + ); + // The fediverse has no host list, so the endpoint comes out of the handle. + assert.equal( + profileRequest({ network: 'fediverse', handle: '@kev@fosstodon.org' }).url, + 'https://fosstodon.org/api/v1/accounts/lookup?acct=kev', + ); + assert.equal(profileRequest({ network: 'linkedin', handle: 'someone' }), null); + assert.equal(profileRequest({ network: 'github', handle: '' }), null); + assert.equal(profileRequest({ network: 'fediverse', handle: 'no-host' }), null); +}); + +test('a GitHub token is sent when there is one, because 60 an hour is not a budget', () => { + const anonymous = profileRequest({ network: 'github', handle: 'x' }); + assert.equal(anonymous.headers.authorization, undefined); + + const authed = profileRequest({ network: 'github', handle: 'x' }, { token: 'ghp_test' }); + assert.equal(authed.headers.authorization, 'Bearer ghp_test'); +}); + +test('a malformed profile body is nothing, not a throw', () => { + for (const body of [null, undefined, 'not json', 42]) { + const profile = identityFromProfile('github', body); + assert.deepEqual(profile.links, []); + assert.equal(profile.name, ''); + } + // GitLab answers its user search with an array, and an empty one when the + // handle matched nobody. + assert.equal(identityFromProfile('gitlab', []).name, ''); +}); diff --git a/packages/ingest/index.js b/packages/ingest/index.js index f55bb48..f6be9cc 100644 --- a/packages/ingest/index.js +++ b/packages/ingest/index.js @@ -32,3 +32,10 @@ export { enrichFeedAuthors, storeCredits, } from './src/enrich.js'; +export { + linksFromSearch, + searchDue, + searchForAuthor, + searchesFor, + worthSearching, +} from './src/websearch.js'; diff --git a/packages/ingest/src/enrich.js b/packages/ingest/src/enrich.js index 6f31fb3..b7aaa02 100644 --- a/packages/ingest/src/enrich.js +++ b/packages/ingest/src/enrich.js @@ -1,12 +1,19 @@ import { BIO_HOSTS, + classifyLink, + credit, + hostIdentity, identityFromHtml, + identityFromHumansTxt, + identityFromProfile, identityKey, linksBackTo, linksFromBioPage, + looksLikePersonName, mergeCredits, normalizeIdentityUrl, normalizeName, + profileRequest, resolveFeed, safeFetch, uniqueSlug, @@ -34,8 +41,36 @@ import { authors as a } from '@rssamplifier/db'; * @typedef {import('@libsql/client').Client} Client */ -/** Pages tried on a site, in order, until one names somebody. */ -const IDENTITY_PATHS = ['/', '/about', '/about/', '/about-me', '/contact', '/colophon']; +/** + * Pages tried on a site, in order, until one names somebody. + * + * `/now` and `/uses` are here because this is the small web: both are + * conventions of exactly the population this directory indexes, both are + * written in the first person, and a blog that has one usually has no `/about` + * — which is why they sit after the pages that are more likely to exist rather + * than instead of them. `MAX_PAGES` still bounds the whole list. + */ +const IDENTITY_PATHS = [ + '/', + '/about', + '/about/', + '/about-me', + '/contact', + '/colophon', + '/now', + '/uses', +]; + +/** + * The file whose only job is to name the people. + * + * Tried last and only when nothing else named anybody, because it is a request + * spent on a file most sites do not have. When a site does have one it is the + * best source on it: every other place identity turns up is markup that carries + * it as a side effect, while somebody writing this was answering the question + * directly. + */ +const HUMANS_TXT = '/humans.txt'; /** Cap on the pages fetched per feed, whatever the list above allows. */ const MAX_PAGES = 3; @@ -46,6 +81,16 @@ const MAX_BIO_PAGES = 1; /** Cap on rel="me" backlinks verified per author, since each is a fetch. */ const MAX_VERIFY = 3; +/** + * Cap on platform profiles resolved per feed. + * + * Each is one JSON request, and the yield falls off a cliff after the first: + * the account named by the feed's own hostname is the one that describes the + * publisher, and the third-best account on the page is usually a colleague, a + * project, or the theme's author. + */ +const MAX_PROFILES = 3; + /** * Networks whose profile pages publish a rel="me" link back to the author's * own site, which is the only cheap proof an account really is theirs. @@ -230,9 +275,11 @@ export async function prepareCredits(db, feed, credits, links = []) { * * @param {Client} db * @param {{ id: string, slug?: string, feed_url: string, site_url?: string|null, title?: string }} feed - * @param {{ verify?: boolean, fetch?: typeof safeFetch, resolve?: typeof resolveFeed }} [opts] + * @param {{ verify?: boolean, fetch?: typeof safeFetch, resolve?: typeof resolveFeed, + * githubToken?: string }} [opts] * `fetch` and `resolve` are injected by the tests so the pass can be - * exercised end to end without a network + * exercised end to end without a network; `githubToken` raises the GitHub + * API's 60-per-hour anonymous ceiling to 5,000 * @returns {Promise<{ people: number, links: number, pages: number, verified: number }>} */ export async function enrichFeedAuthors(db, feed, opts = {}) { @@ -244,6 +291,16 @@ export async function enrichFeedAuthors(db, feed, opts = {}) { let pages = 0; let verified = 0; + // Did we ever get an answer out of this publisher at all? + // + // The distinction this records is the one the stamp used to lose. Every fetch + // in here fails softly -- a dead host, a timeout and a 503 all come back as + // "no page" rather than as an exception -- so a site that was simply down + // looked exactly like a site that named nobody, and both were stamped as + // checked. That cost the publisher their enrichment for the full ninety-day + // recheck cycle on the strength of one bad afternoon. + let reached = false; + const collect = (found) => { for (const link of found) { const existing = links.get(link.url); @@ -258,6 +315,7 @@ export async function enrichFeedAuthors(db, feed, opts = {}) { // the page fetches below. const resolved = await resolve(String(feed.feed_url)).catch(() => null); if (resolved?.ok) { + reached = true; credits.push(...(resolved.feed.credits ?? [])); // A feed that declares its site is a better base for relative links than // whatever the row was stored with. @@ -279,6 +337,7 @@ export async function enrichFeedAuthors(db, feed, opts = {}) { } const page = await fetchPage(target).catch(() => null); + if (page?.ok) reached = true; if (!page?.ok || !/html/i.test(page.contentType)) continue; pages += 1; @@ -302,6 +361,35 @@ export async function enrichFeedAuthors(db, feed, opts = {}) { } } + // 2b. humans.txt, when the pages named nobody. + // + // Conditional on purpose. This is a file most sites do not publish, so asking + // every site for it would spend one request per feed across the directory to + // help the minority that has one. Asking only after the ordinary pages have + // come back empty spends it exactly where it is the difference between an + // author and no author. + // + // Not parsed as HTML: it is a text file, and a server that answers every path + // with its 404 page would otherwise turn that page into a person. + if (siteUrl && credits.length === 0) { + let target; + try { + target = new URL(HUMANS_TXT, siteUrl).toString(); + } catch { + target = ''; + } + + if (target) { + const page = await fetchPage(target).catch(() => null); + if (page?.ok) { + pages += 1; + const found = identityFromHumansTxt(page.body, page.url || target); + collect(found.profiles); + credits.push(...found.credits); + } + } + } + // 3. One hop through a links page, which is the only kind of page where // every outbound link is known to belong to the same person. let hops = 0; @@ -321,9 +409,102 @@ export async function enrichFeedAuthors(db, feed, opts = {}) { collect(linksFromBioPage(page.body, page.url || link.url)); } + // 4. The accounts the addresses themselves name, and the profiles behind + // them. + // + // This is the half that reaches the publishers who marked nothing up, which + // is most of them. The case it was built for: felginep.github.io publishes a + // blog with no rel="me", no h-card and one outbound link -- to the Jekyll + // theme its author used. Everything above finds nobody. But the author's + // GitHub account is named in the hostname the feed is served from, and one + // request to that account returns "Pierre Felgines" and an avatar. + // + // Deriving the account is string arithmetic on URLs already in hand, so a + // feed on a platform we do not recognise costs nothing extra at all. + collect(hostIdentity(feed.feed_url, siteUrl)); + + let profiles = 0; + + for (const account of [...links.values()]) { + if (profiles >= MAX_PROFILES) break; + + const request = profileRequest(account, { token: opts.githubToken }); + if (!request) continue; + profiles += 1; + + // A 404 here is the useful negative: the hostname proposed an account that + // does not exist, so the derivation was wrong and nothing is stored for it. + const page = await fetchPage(request.url, { headers: request.headers }).catch(() => null); + if (!page?.ok) continue; + pages += 1; + + let body; + try { + body = JSON.parse(page.body); + } catch { + continue; + } + + const profile = identityFromProfile(request.network, body, account); + + for (const found of profile.links) { + // A platform's own "website" field is a homepage, which matches none of + // the profile shapes -- classifyLink is right to return null for it, and + // it is still the single most useful link a profile carries. + const classified = classifyLink(found.url) ?? asWebsite(found.url); + if (!classified) continue; + + collect([ + { + ...classified, + source: found.source, + // Only the fediverse hands us a verification, and it hands us a real + // one: the instance already followed the link and found a rel="me" + // pointing back. That is the same handshake the verify pass below + // spends fetches on, arriving for free. + verified: Boolean(found.verified), + }, + ]); + + // The account points back at the site we are enriching. That is the + // IndieWeb handshake in the other direction and is proof the account + // belongs to this publisher, so the account link earns `verified` -- the + // column means "the destination links back", and here it does. + if (classified.network === 'website' && siteUrl && sameSite(classified.url, siteUrl)) { + account.verified = true; + verified += 1; + } + } + + // An organisation is not a person. GitHub and Gitea serve both from one + // endpoint, so a project site on `someproject.github.io` resolves to an + // account whose name is a product -- publishing that as an author is the + // mistake identity.js's role filters exist to prevent, arriving by a + // different door. + if (profile.kind !== 'user') continue; + if (!looksLikePersonName(profile.name)) continue; + + credits.push( + credit({ + name: profile.name, + url: account.url, + avatar: profile.avatar, + bio: profile.bio, + role: 'author', + source: `${request.network}-profile`, + // Above the publishing floor, and deliberately only just. The blog is + // served from this account's own pages and the account is a person + // whose name reads as one -- but nobody has said in so many words that + // they wrote it, which is what the stronger sources have. A confirmed + // backlink lifts it to where a rel="me" pair sits. + confidence: account.verified ? 0.9 : account.source === 'host-derived' ? 0.7 : 0.65, + }), + ); + } + const merged = mergeCredits(credits.filter(Boolean)); - // 4. The IndieWeb handshake, for the accounts that answer it. Only worth + // 5. The IndieWeb handshake, for the accounts that answer it. Only worth // spending requests on when there is a site to link back to, and only when // the caller asked — the pass over 52,000 feeds does not, a re-check of one // feed does. @@ -345,9 +526,17 @@ export async function enrichFeedAuthors(db, feed, opts = {}) { } const stored = await storeCredits(db, feed, merged, [...links.values()]); - await a.markAuthorsChecked(db, String(feed.id)); - return { ...stored, pages, verified }; + // Looked and found nobody: stamped, and left alone until the recheck. Could + // not look at all: stamped so it comes back in a few days, because nothing + // was learned about this publisher and the next attempt may well succeed. + if (reached) { + await a.markAuthorsChecked(db, String(feed.id)); + } else { + await a.markAuthorsFailed(db, String(feed.id), { recheckDays: opts.recheckDays }); + } + + return { ...stored, pages, verified, reached }; } /** @@ -407,7 +596,10 @@ const PER_HOST = 3; * @param {Client} db * @param {number} [batchSize] * @param {{ verify?: boolean, recheckDays?: number, onEvent?: ((event: object) => void)|null, - * concurrency?: number, fetch?: typeof safeFetch, resolve?: typeof resolveFeed }} [opts] + * concurrency?: number, fetch?: typeof safeFetch, resolve?: typeof resolveFeed, + * githubToken?: string }} [opts] `githubToken` is not optional in practice: + * the unauthenticated GitHub API allows 60 requests an hour per IP, which one + * batch exhausts, so a pass without it resolves almost no profiles * @returns {Promise<{ feeds: number, people: number, links: number, hosts: number }>} */ export async function enrichDue(db, batchSize = 10, opts = {}) { @@ -440,8 +632,10 @@ export async function enrichDue(db, batchSize = 10, opts = {}) { try { const result = await enrichFeedAuthors(db, feed, { verify: opts.verify, + recheckDays, fetch: opts.fetch, resolve: opts.resolve, + githubToken: opts.githubToken, }); feeds += 1; people += result.people; @@ -449,7 +643,15 @@ export async function enrichDue(db, batchSize = 10, opts = {}) { feedLinks += result.feedLinks ?? 0; report(opts.onEvent, feed, started, { ok: true, amount: result.people, detail: null }); } catch (err) { - await a.markAuthorsChecked(db, String(feed.id)).catch(() => {}); + // A failure is not a miss. Stamping this the same way as "looked, and + // there was nobody" would cost a publisher on a flaky host their + // enrichment for the whole recheck cycle over one timeout, so it is + // stamped to come back in a few days instead. The feed is still + // stamped, which is what keeps a permanently broken one from sitting + // at the head of the queue forever. + await a + .markAuthorsFailed(db, String(feed.id), { recheckDays }) + .catch(() => {}); report(opts.onEvent, feed, started, { ok: false, amount: null, @@ -581,3 +783,48 @@ function report(onEvent, feed, started, outcome) { // A broken listener loses its line and nothing else. } } + +/** + * A bare homepage as a link row. + * + * `classifyLink` returns null for anything that is not a recognised profile + * shape, which is correct for a link found loose in a page -- it would + * otherwise file every outbound link as somebody's website. A URL taken out of + * a *profile field* is different: the person put it there to say "this is my + * site", so it is the one place the fallback is warranted. + * + * @param {unknown} value + * @returns {{ network: string, url: string, handle: string }|null} + */ +function asWebsite(value) { + const raw = normalizeIdentityUrl(value); + if (!raw) return null; + + try { + return { network: 'website', url: raw, handle: new URL(raw).hostname }; + } catch { + return null; + } +} + +/** + * Do two URLs name the same site? + * + * Compared on host alone, with `www.` folded away: a profile that links + * `https://example.com` and a feed that declares `https://www.example.com/blog/` + * are the same publisher, and requiring the paths to match would refuse every + * real backlink. + * + * @param {unknown} a + * @param {unknown} b + * @returns {boolean} + */ +function sameSite(a, b) { + try { + const host = (u) => new URL(String(u)).hostname.toLowerCase().replace(/^www\./, ''); + const left = host(a); + return Boolean(left) && left === host(b); + } catch { + return false; + } +} diff --git a/packages/ingest/src/websearch.js b/packages/ingest/src/websearch.js new file mode 100644 index 0000000..5ec69ec --- /dev/null +++ b/packages/ingest/src/websearch.js @@ -0,0 +1,338 @@ +/* + * Looking for a person who did not leave a trail. + * + * Everything else in the enrichment reads something the publisher put where we + * could find it: their feed, their page, their hostname, their profile. This + * asks a search engine instead, which is a different kind of act and carries a + * different kind of risk, so it is fenced in three ways that are not optional. + * + * **It cannot run over the directory.** Search is metered. The account this + * borrows is CrawlProof's 25,000-credit month, already shared with CrawlProof's + * own outreach runner, and the directory is 369,056 feeds — one query each + * would be fifteen times the entire monthly allowance. So this is a targeted + * tool with a hard budget, off unless switched on, and it refuses rather than + * degrades when the budget is gone. + * + * **It cannot invent a person.** Searching a name and attaching what comes back + * is how two different people with one name become one page. `identity.js` has + * refused to key on a name alone since it was written, and this must not become + * the back door: a search result is a *candidate*, and what promotes it is + * corroboration — the page linking back to the site we already know is theirs, + * or a handle we already hold from a source that proved itself. + * + * **It cannot be the reason we hold something.** Every link it produces is + * stamped `web-search`, so any consumer can exclude the whole class. Nothing + * here is ever marked `verified` on the strength of a search result. + * + * On LinkedIn specifically, since it is the thing everybody asks for: a profile + * URL found this way is stored as a link, because it is a public address the + * search engine already indexed. The profile behind it is not fetched — it is + * auth-walled, returns 999 to anything automated, and its terms forbid + * scraping. So we can tell you where somebody's LinkedIn is, and cannot tell + * you what is on it. + */ + +import { classifyLink } from '@rssamplifier/feed'; +import { authors as a } from '@rssamplifier/db'; + +/** Where the searches are bought. */ +const ENDPOINT = 'https://api.valueserp.com/search'; + +/** How long to wait for a result before abandoning it. */ +const TIMEOUT_MS = 8000; + +/** + * The networks worth spending a query on, and how to ask for them. + * + * One query per network rather than one broad query, because a search for a + * name alone returns news articles and namesakes, while a site-scoped one + * returns a profile or nothing — and "nothing" is the answer we want when the + * person genuinely has no account there. + * + * LinkedIn leads because it is the one this exists for. The rest are ordered by + * how often this directory's population actually has them. + */ +const TARGETS = [ + { network: 'linkedin', site: 'linkedin.com/in' }, + { network: 'github', site: 'github.com' }, + { network: 'twitter', site: 'x.com' }, + { network: 'bluesky', site: 'bsky.app/profile' }, +]; + +/** + * Is a search worth buying for this person? + * + * The gate is deliberately mean, because the budget is small and the value of a + * query is wildly uneven. A person who writes one dormant blog and already has + * an email is not worth a credit; a person behind several live feeds with no + * way to reach them at all is the whole point. + * + * @param {{ feedCount?: number, linkCount?: number, confidence?: number, name?: string }} author + * @param {{ minFeeds?: number }} [opts] + * @returns {boolean} + */ +export function worthSearching(author, opts = {}) { + const minFeeds = Number(opts.minFeeds ?? 1); + + // Somebody we are not confident is a person should not be searched for: the + // query would be for a name we half-believe, and the results would be + // attached to it. + if (Number(author?.confidence ?? 0) < 0.8) return false; + + // A name that is one word is not searchable in any useful way -- "Sakrecoer" + // returns the person, but "Jane" returns the world. + const words = String(author?.name ?? '').trim().split(/\s+/).filter(Boolean); + if (words.length < 2) return false; + + if (Number(author?.feedCount ?? 0) < minFeeds) return false; + + // Already reachable. The point of the budget is the people we cannot contact. + if (Number(author?.linkCount ?? 0) > 0) return false; + + return true; +} + +/** + * The queries to buy for one person. + * + * Scoped to their own site as well as to each network, because the name alone + * is the ambiguous part: "Jane Doe" site:linkedin.com/in returns every Jane + * Doe, while the domain of the blog she writes is the thing that distinguishes + * her from them. + * + * @param {{ name: string, site?: string|null }} author + * @returns {Array<{ network: string, q: string }>} + */ +export function searchesFor(author) { + const name = String(author?.name ?? '').trim(); + if (!name) return []; + + const domain = hostOf(author?.site); + + return TARGETS.map((target) => ({ + network: target.network, + q: domain + ? `"${name}" ${domain} site:${target.site}` + : `"${name}" site:${target.site}`, + })); +} + +/** + * Read a ValueSERP response into the links it actually contains. + * + * Only results whose URL classifies as the network we asked for are kept. A + * search for a LinkedIn profile routinely returns a LinkedIn *company* page, a + * job posting or an article about the person, none of which is an account — + * `classifyLink` already knows the difference and is the filter. + * + * @param {string} network the network the query was scoped to + * @param {unknown} body the parsed JSON + * @returns {Array<{ network: string, url: string, handle: string, source: string, verified: boolean }>} + */ +export function linksFromSearch(network, body) { + const results = Array.isArray(/** @type {any} */ (body)?.organic_results) + ? /** @type {any} */ (body).organic_results + : []; + + /** @type {Map} */ + const found = new Map(); + + for (const result of results) { + const link = classifyLink(result?.link); + if (!link || link.network !== network) continue; + if (found.has(link.url)) continue; + + found.set(link.url, { + ...link, + source: 'web-search', + // Never. A search engine's opinion that two strings co-occur is not the + // IndieWeb handshake, and this column means the handshake. + verified: false, + }); + } + + return [...found.values()]; +} + +/** + * Buy the searches for one person, within a budget. + * + * Returns the links and how many credits were actually spent, so the caller can + * keep a running total that survives a restart by being written down rather + * than held in memory. + * + * Every failure is empty and silent, exactly as the ad fetch is: enrichment is + * a bonus on top of a directory, and a search provider having a bad afternoon + * must not stop the pass that does not need it. + * + * @param {{ name: string, site?: string|null }} author + * @param {{ apiKey: string, budget: number, fetch?: typeof globalThis.fetch }} opts + * `budget` is the number of queries this call may spend, already decided by + * the caller against the month's remaining allowance + * @returns {Promise<{ links: Array, spent: number, exhausted?: boolean }>} + * `exhausted` says the account is empty, which will not change before the + * provider's reset -- the caller stops rather than asking again + */ +export async function searchForAuthor(author, opts) { + const apiKey = String(opts?.apiKey ?? ''); + const budget = Math.max(0, Math.floor(Number(opts?.budget ?? 0))); + if (!apiKey || budget === 0) return { links: [], spent: 0 }; + + const doFetch = opts.fetch ?? globalThis.fetch; + const queries = searchesFor(author).slice(0, budget); + + /** @type {Map} */ + const links = new Map(); + let spent = 0; + + for (const query of queries) { + const url = + `${ENDPOINT}?api_key=${encodeURIComponent(apiKey)}` + + `&q=${encodeURIComponent(query.q)}&num=10&output=json`; + + let body; + try { + const res = await doFetch(url, { + signal: AbortSignal.timeout(TIMEOUT_MS), + headers: { accept: 'application/json' }, + }); + + // 402 is the documented answer for an exhausted account, and it is not a + // transient error -- there is no point asking again this month, so the + // caller is told to stop rather than left to burn the rest of the batch + // on the same refusal. + if (res.status === 402) return { links: [...links.values()], spent, exhausted: true }; + if (!res.ok) continue; + + body = await res.json(); + } catch { + continue; + } + + // Counted whether or not anything useful came back: the credit is spent at + // the provider either way, and a budget that only counts hits is not a + // budget. + spent += 1; + + for (const link of linksFromSearch(query.network, body)) { + if (!links.has(link.url)) links.set(link.url, link); + } + } + + return { links: [...links.values()], spent }; +} + +/** + * The bare host of a URL, for scoping a query to somebody's own domain. + * + * @param {unknown} value + * @returns {string} + */ +function hostOf(value) { + try { + return new URL(String(value ?? '')).hostname.toLowerCase().replace(/^www\./, ''); + } catch { + return ''; + } +} + +/** + * Spend part of the month's allowance on the people nobody can reach. + * + * The budget is read from the ledger rather than from a counter, because the + * poller restarts on every deploy and a budget that resets with the process is + * not a budget. `billingPeriodStart` matches the provider's own cycle -- this + * account resets on the 13th, not the 1st -- so the total this compares against + * is the total the invoice will show. + * + * Stops at the first sign the account is empty. A 402 is the documented answer + * for an exhausted allowance and it will not change before the reset, so + * carrying on would spend the rest of the batch's wall-clock re-reading the + * same refusal. + * + * @param {import('@libsql/client').Client} db + * @param {{ + * apiKey: string, + * monthlyBudget: number, + * perAuthor?: number, + * batchSize?: number, + * minConfidence?: number, + * fetch?: typeof globalThis.fetch, + * }} opts + * @returns {Promise<{ people: number, links: number, spent: number, remaining: number }>} + */ +export async function searchDue(db, opts) { + const apiKey = String(opts?.apiKey ?? ''); + const monthly = Math.max(0, Math.floor(Number(opts?.monthlyBudget ?? 0))); + if (!apiKey || monthly === 0) return { people: 0, links: 0, spent: 0, remaining: 0 }; + + const since = a.billingPeriodStart(); + const already = await a.searchSpendSince(db, since); + let remaining = monthly - already; + if (remaining <= 0) return { people: 0, links: 0, spent: 0, remaining: 0 }; + + const perAuthor = Math.max(1, Math.floor(Number(opts.perAuthor ?? 2))); + const batchSize = Math.max(1, Math.floor(Number(opts.batchSize ?? 5))); + + const candidates = await a.authorsWithoutContact( + db, + batchSize, + Number(opts.minConfidence ?? 0.8), + ); + + let people = 0; + let links = 0; + let spent = 0; + + for (const author of candidates) { + if (remaining <= 0) break; + + // The SQL above already selects for this, and the check is repeated here on + // purpose: the query and the rule are two statements of one policy, and the + // cheap one is the one that must not be the only one. If they ever drift, + // this refuses to spend money on the difference. + if ( + !worthSearching({ + name: String(author.name ?? ''), + confidence: Number(author.confidence ?? 0), + feedCount: Number(author.feed_count ?? 0), + linkCount: 0, + }) + ) { + continue; + } + + const result = await searchForAuthor( + { name: String(author.name), site: author.site }, + { + apiKey, + budget: Math.min(perAuthor, remaining), + fetch: opts.fetch, + }, + ); + + // Written down before anything else, because the credits are gone whether + // or not the rest of this succeeds, and a ledger that only records + // successful passes will drift under the real spend. + if (result.spent > 0) { + await a.recordAuthorSearch(db, { + authorId: String(author.id), + queries: result.spent, + found: result.links.length, + }); + } + + spent += result.spent; + remaining -= result.spent; + + if (result.links.length > 0) { + links += await a.addAuthorLinks(db, String(author.id), result.links); + people += 1; + } + + // The account is empty. Nothing further this period will succeed. + if (result.exhausted) break; + } + + return { people, links, spent, remaining: Math.max(0, remaining) }; +} diff --git a/packages/ingest/test/enrich-platforms.test.js b/packages/ingest/test/enrich-platforms.test.js new file mode 100644 index 0000000..795aae4 --- /dev/null +++ b/packages/ingest/test/enrich-platforms.test.js @@ -0,0 +1,220 @@ +import assert from 'node:assert/strict'; +import { test, before, after } from 'node:test'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { connect, migrate, q, authors as a } from '@rssamplifier/db'; + +import { enrichFeedAuthors } from '../src/enrich.js'; + +// The publisher who marked nothing up. +// +// Everything in enrich.test.js is about reading what a page says. This file is +// about the case where the page says nothing at all -- which is most of the +// directory, and was 100% of what the pass returned empty-handed on. The proof +// case is a real one: felginep.github.io publishes a blog with no rel="me", no +// h-card and exactly one outbound link, to the Jekyll theme its author used. + +let dir; +let db; + +before(async () => { + dir = await mkdtemp(join(tmpdir(), 'rssamp-platforms-')); + db = connect({ url: `file:${join(dir, 'test.db')}` }); + await migrate(db); +}); + +after(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +async function seedFeed(feed) { + const row = await q.insertFeed(db, { + slug: feed.slug, + feed_url: feed.feedUrl, + site_url: feed.siteUrl ?? null, + title: feed.title ?? 'A Blog', + categories: [], + kind: 'blog', + status: 'active', + }); + return { ...row, feed_url: feed.feedUrl, site_url: feed.siteUrl ?? null }; +} + +/** + * A fetcher answering HTML pages and JSON APIs from one map. + * + * The content type follows the URL rather than the caller, because that is what + * distinguishes the two paths under test: a page is parsed as markup and a + * profile is parsed as JSON, and a stub that returned `text/html` for an API + * would let a broken content-type check pass. + */ +function fakeFetch(responses) { + const asked = []; + const fetcher = async (url) => { + asked.push(url); + const body = responses[url]; + if (body == null) return { ok: false, status: 404, contentType: '', body: '', url }; + const json = typeof body !== 'string'; + return { + ok: true, + status: 200, + contentType: json ? 'application/json' : 'text/html', + body: json ? JSON.stringify(body) : body, + url, + }; + }; + fetcher.asked = asked; + return fetcher; +} + +const fakeResolve = (siteUrl) => async () => ({ + ok: true, + feed: { title: 'A Blog', siteUrl, credits: [], items: [] }, +}); + +/** A page with a byline nowhere and one link, to somebody else's theme. */ +const BARE_PAGE = ` +

Posts

+ theme +`; + +test('a blog that names nobody still finds its author, through the hostname', async () => { + const feed = await seedFeed({ + slug: 'felginep-github-io', + feedUrl: 'https://felginep.github.io/feed.xml', + siteUrl: 'https://felginep.github.io/', + }); + + const fetcher = fakeFetch({ + 'https://felginep.github.io/': BARE_PAGE, + 'https://api.github.com/users/felginep': { + type: 'User', + name: 'Pierre Felgines', + avatar_url: 'https://avatars.example/u/1', + }, + }); + + const result = await enrichFeedAuthors(db, feed, { + fetch: fetcher, + resolve: fakeResolve('https://felginep.github.io/'), + }); + + assert.equal(result.people, 1, 'the page named nobody, so this can only have come from the profile'); + + const [author] = await a.authorsForFeed(db, feed.id); + assert.equal(author.name, 'Pierre Felgines'); + assert.ok( + author.confidence >= 0.6, + `must clear the publishing floor, got ${author.confidence}`, + ); + + // And the account itself is stored, which is a contact surface in its own right. + const links = await a.linksForFeed(db, feed.id); + assert.ok( + links.some((l) => l.network === 'github' && l.url === 'https://github.com/felginep'), + 'the derived account is stored against the feed', + ); +}); + +test('the profile is asked about exactly once, and only when the host names an account', async () => { + const feed = await seedFeed({ + slug: 'plain-domain', + feedUrl: 'https://kevquirk.com/feed', + siteUrl: 'https://kevquirk.com/', + }); + + const fetcher = fakeFetch({ 'https://kevquirk.com/': BARE_PAGE }); + await enrichFeedAuthors(db, feed, { + fetch: fetcher, + resolve: fakeResolve('https://kevquirk.com/'), + }); + + assert.equal( + fetcher.asked.filter((u) => u.includes('api.github.com')).length, + 0, + 'a plain domain names no account, so no API request may be spent on it', + ); +}); + +test('an organisation behind a project site does not become a person', async () => { + // jekyll.github.io is a project, and its account has a name that reads like + // one. Publishing it as an author would put a piece of software on a page + // that says these are people. + const feed = await seedFeed({ + slug: 'jekyll-github-io', + feedUrl: 'https://jekyll.github.io/feed.xml', + siteUrl: 'https://jekyll.github.io/', + }); + + const fetcher = fakeFetch({ + 'https://jekyll.github.io/': BARE_PAGE, + 'https://api.github.com/users/jekyll': { type: 'Organization', name: 'Jekyll' }, + }); + + const result = await enrichFeedAuthors(db, feed, { + fetch: fetcher, + resolve: fakeResolve('https://jekyll.github.io/'), + }); + + assert.equal(result.people, 0); + assert.deepEqual(await a.authorsForFeed(db, feed.id), [], 'no fictional person'); + + // The account is still worth storing: it is where the feed comes from, and a + // link is not a claim about who anybody is. + const links = await a.linksForFeed(db, feed.id); + assert.ok(links.some((l) => l.url === 'https://github.com/jekyll')); +}); + +test('a profile that links back to the site proves the account, and says so', async () => { + // The IndieWeb handshake in the other direction: the account points at the + // blog, which is what `verified` means in the schema. + const feed = await seedFeed({ + slug: 'backlinked', + feedUrl: 'https://ann.github.io/feed.xml', + siteUrl: 'https://ann.github.io/', + }); + + const fetcher = fakeFetch({ + 'https://ann.github.io/': BARE_PAGE, + 'https://api.github.com/users/ann': { + type: 'User', + name: 'Ann Example', + blog: 'https://ann.github.io', + }, + }); + + const result = await enrichFeedAuthors(db, feed, { + fetch: fetcher, + resolve: fakeResolve('https://ann.github.io/'), + }); + + assert.equal(result.verified, 1, 'the backlink is counted as the proof it is'); + + const [author] = await a.authorsForFeed(db, feed.id); + assert.ok( + author.confidence >= 0.9, + `a proved account outranks a derived one, got ${author.confidence}`, + ); +}); + +test('an account that does not exist teaches us nothing and costs nothing', async () => { + // The useful negative. A hostname can propose an account that was deleted or + // never existed; a 404 must leave no trace rather than half a person. + const feed = await seedFeed({ + slug: 'ghost-account', + feedUrl: 'https://nobodyhome.github.io/feed.xml', + siteUrl: 'https://nobodyhome.github.io/', + }); + + const fetcher = fakeFetch({ 'https://nobodyhome.github.io/': BARE_PAGE }); + + const result = await enrichFeedAuthors(db, feed, { + fetch: fetcher, + resolve: fakeResolve('https://nobodyhome.github.io/'), + }); + + assert.equal(result.people, 0); + assert.deepEqual(await a.authorsForFeed(db, feed.id), []); +}); diff --git a/packages/ingest/test/enrich-retry.test.js b/packages/ingest/test/enrich-retry.test.js new file mode 100644 index 0000000..41b4cf5 --- /dev/null +++ b/packages/ingest/test/enrich-retry.test.js @@ -0,0 +1,89 @@ +import assert from 'node:assert/strict'; +import { test, before, after } from 'node:test'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { connect, migrate, q, authors as a } from '@rssamplifier/db'; + +import { enrichDue } from '../src/enrich.js'; + +// A failure is not a miss, and treating them the same is how a publisher on a +// flaky host loses their enrichment for three months over one timeout. + +let dir; +let db; + +before(async () => { + dir = await mkdtemp(join(tmpdir(), 'rssamp-retry-')); + db = connect({ url: `file:${join(dir, 'test.db')}` }); + await migrate(db); +}); + +after(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +test('a feed whose site fell over is tried again in days, not in a season', async () => { + await q.insertFeed(db, { + slug: 'flaky', + feed_url: 'https://flaky.example/feed.xml', + site_url: 'https://flaky.example/', + title: 'Flaky Blog', + categories: [], + kind: 'blog', + status: 'active', + }); + + const recheckDays = 90; + const recheckBefore = new Date(Date.now() - recheckDays * 86_400_000).toISOString(); + + await enrichDue(db, 5, { + recheckDays, + // The shape of a bad afternoon: the host simply does not answer. + resolve: async () => { + throw new Error('ETIMEDOUT'); + }, + fetch: async () => { + throw new Error('ETIMEDOUT'); + }, + }); + + // Still stamped, which is what keeps a permanently broken feed from sitting + // at the head of the queue forever. + const [row] = (await db.execute('select authors_checked_at from feeds where slug = \'flaky\'')).rows; + assert.ok(row.authors_checked_at, 'a failure is still recorded'); + + // But not due 90 days from now. It comes back within a few days. + assert.deepEqual( + await a.dueForAuthors(db, 5, recheckBefore), + [], + 'not due again immediately, or it would block the queue', + ); + + const inFourDays = new Date(Date.now() + 4 * 86_400_000 - recheckDays * 86_400_000).toISOString(); + const soon = await a.dueForAuthors(db, 5, inFourDays); + assert.equal(soon.length, 1, 'and it is due again within days rather than in three months'); +}); + +test('the retry stamp is never in the future', async () => { + // A stamp ahead of now would hide the feed from any pass whose recheck window + // is shorter than this one's. + await q.insertFeed(db, { + slug: 'future', + feed_url: 'https://future.example/feed.xml', + title: 'Future Blog', + categories: [], + kind: 'blog', + status: 'active', + }); + + const [feed] = (await db.execute("select id from feeds where slug = 'future'")).rows; + await a.markAuthorsFailed(db, String(feed.id), { retryDays: 200, recheckDays: 90 }); + + const [row] = (await db.execute("select authors_checked_at from feeds where slug = 'future'")).rows; + assert.ok( + String(row.authors_checked_at) <= new Date().toISOString(), + `stamp must not be in the future, got ${row.authors_checked_at}`, + ); +}); diff --git a/packages/ingest/test/websearch.test.js b/packages/ingest/test/websearch.test.js new file mode 100644 index 0000000..3d61dad --- /dev/null +++ b/packages/ingest/test/websearch.test.js @@ -0,0 +1,238 @@ +import assert from 'node:assert/strict'; +import { test, before, after } from 'node:test'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { connect, migrate, q, authors as a } from '@rssamplifier/db'; + +import { storeCredits } from '../src/enrich.js'; +import { linksFromSearch, searchDue, searchesFor, worthSearching } from '../src/websearch.js'; + +// This is the only pass that spends money, so what is tested is mostly what it +// refuses to do: search for somebody it is not sure is a person, search for +// somebody already reachable, spend past the budget, or believe a result. + +let dir; +let db; + +before(async () => { + dir = await mkdtemp(join(tmpdir(), 'rssamp-search-')); + db = connect({ url: `file:${join(dir, 'test.db')}` }); + await migrate(db); +}); + +after(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +test('the gate refuses everyone a credit would be wasted on', () => { + const ok = { name: 'Jane Doe', confidence: 0.85, feedCount: 1, linkCount: 0 }; + assert.equal(worthSearching(ok), true); + + // A name we only half-believe would put the results on a person who may not + // exist. + assert.equal(worthSearching({ ...ok, confidence: 0.6 }), false); + // One word returns the world. + assert.equal(worthSearching({ ...ok, name: 'Jane' }), false); + // Already reachable: the budget exists for the people who are not. + assert.equal(worthSearching({ ...ok, linkCount: 2 }), false); + assert.equal(worthSearching({}), false); +}); + +test('a query is scoped by the domain, which is what tells two namesakes apart', () => { + const queries = searchesFor({ name: 'Jane Doe', site: 'https://www.jane.example/blog' }); + assert.ok(queries.every((s) => s.q.includes('jane.example'))); + assert.ok(queries.some((s) => s.q === '"Jane Doe" jane.example site:linkedin.com/in')); + + // Without a site there is nothing to scope by, and the name has to stand alone. + const bare = searchesFor({ name: 'Jane Doe' }); + assert.equal(bare[0].q, '"Jane Doe" site:linkedin.com/in'); + assert.deepEqual(searchesFor({ name: '' }), []); +}); + +test('only real profiles survive, and none of them is evidence', () => { + const links = linksFromSearch('linkedin', { + organic_results: [ + { link: 'https://www.linkedin.com/in/janedoe' }, + // A company page is not a person, and a post is not an account. + { link: 'https://www.linkedin.com/company/acme' }, + { link: 'https://www.linkedin.com/posts/janedoe_hiring-activity-123' }, + // The right shape on the wrong network for this query. + { link: 'https://github.com/janedoe' }, + ], + }); + + assert.deepEqual( + links.map((l) => l.url), + ['https://www.linkedin.com/in/janedoe'], + ); + assert.equal(links[0].source, 'web-search'); + assert.equal(links[0].verified, false, 'a search engine cannot verify anything'); +}); + +test('a malformed or empty response yields nothing rather than throwing', () => { + assert.deepEqual(linksFromSearch('github', null), []); + assert.deepEqual(linksFromSearch('github', {}), []); + assert.deepEqual(linksFromSearch('github', { organic_results: 'no' }), []); +}); + +/** Seed one author with no way to reach them. */ +async function seedUnreachable(slug, name) { + const feed = await q.insertFeed(db, { + slug, + feed_url: `https://${slug}.example/feed.xml`, + site_url: `https://${slug}.example/`, + title: name, + categories: [], + kind: 'blog', + status: 'active', + }); + + await storeCredits( + db, + { id: feed.id, feed_url: `https://${slug}.example/feed.xml` }, + [ + { + name, + email: '', + url: '', + avatar: '', + role: 'author', + source: 'atom-feed-author', + confidence: 0.85, + }, + ], + ); + + return feed; +} + +test('the budget is read from the ledger, so a restart cannot reset it', async () => { + await seedUnreachable('ledger-blog', 'Jane Ledger'); + + // Everything already spent this period. + await a.recordAuthorSearch(db, { authorId: null, queries: 25, found: 0 }); + + let called = 0; + const result = await searchDue(db, { + apiKey: 'test-key', + monthlyBudget: 25, + fetch: async () => { + called += 1; + return { ok: true, status: 200, json: async () => ({}) }; + }, + }); + + assert.equal(called, 0, 'the allowance is gone, so nothing may be bought'); + assert.equal(result.spent, 0); +}); + +test('spending stops at the budget even when more people qualify', async () => { + const fresh = await mkdtemp(join(tmpdir(), 'rssamp-budget-')); + const db2 = connect({ url: `file:${join(fresh, 'b.db')}` }); + await migrate(db2); + + for (const [slug, name] of [ + ['aa-blog', 'Anna Aardvark'], + ['bb-blog', 'Bob Bison'], + ['cc-blog', 'Cara Cat'], + ]) { + const feed = await q.insertFeed(db2, { + slug, + feed_url: `https://${slug}.example/feed.xml`, + site_url: `https://${slug}.example/`, + title: name, + categories: [], + kind: 'blog', + status: 'active', + }); + await storeCredits( + db2, + { id: feed.id, feed_url: `https://${slug}.example/feed.xml` }, + [{ name, email: '', url: '', avatar: '', role: 'author', source: 'atom-feed-author', confidence: 0.85 }], + ); + } + + let called = 0; + const result = await searchDue(db2, { + apiKey: 'test-key', + monthlyBudget: 3, + perAuthor: 2, + batchSize: 10, + fetch: async () => { + called += 1; + return { ok: true, status: 200, json: async () => ({ organic_results: [] }) }; + }, + }); + + assert.equal(called, 3, `bought ${called} queries against a budget of 3`); + assert.equal(result.spent, 3); + + // And the ledger agrees with what was bought, which is the number that has to + // match the invoice. + assert.equal(await a.searchSpendSince(db2, a.billingPeriodStart()), 3); + + await rm(fresh, { recursive: true, force: true }); +}); + +test('an exhausted account stops the batch instead of being asked again', async () => { + const fresh = await mkdtemp(join(tmpdir(), 'rssamp-402-')); + const db2 = connect({ url: `file:${join(fresh, 'c.db')}` }); + await migrate(db2); + + const feed = await q.insertFeed(db2, { + slug: 'dd-blog', + feed_url: 'https://dd-blog.example/feed.xml', + site_url: 'https://dd-blog.example/', + title: 'Dee Blogger', + categories: [], + kind: 'blog', + status: 'active', + }); + await storeCredits( + db2, + { id: feed.id, feed_url: 'https://dd-blog.example/feed.xml' }, + [{ name: 'Dee Blogger', email: '', url: '', avatar: '', role: 'author', source: 'atom-feed-author', confidence: 0.85 }], + ); + + let called = 0; + await searchDue(db2, { + apiKey: 'test-key', + monthlyBudget: 50, + perAuthor: 4, + fetch: async () => { + called += 1; + // The provider's documented answer for an empty account. + return { ok: false, status: 402, json: async () => ({}) }; + }, + }); + + assert.equal(called, 1, '402 will not change before the reset, so asking twice is waste'); + + await rm(fresh, { recursive: true, force: true }); +}); + +test('nothing is bought without a key, a budget and the switch', async () => { + let called = 0; + const fetcher = async () => { + called += 1; + return { ok: true, status: 200, json: async () => ({}) }; + }; + + assert.deepEqual(await searchDue(db, { apiKey: '', monthlyBudget: 100, fetch: fetcher }), { + people: 0, + links: 0, + spent: 0, + remaining: 0, + }); + await searchDue(db, { apiKey: 'k', monthlyBudget: 0, fetch: fetcher }); + assert.equal(called, 0); +}); + +test('the billing period follows the provider, which resets on the 13th', () => { + // A calendar month would let the allowance be spent twice across a reset. + assert.equal(a.billingPeriodStart(new Date('2026-08-19T00:00:00Z')), '2026-08-13T00:00:00.000Z'); + // Before the 13th, the period began the month before. + assert.equal(a.billingPeriodStart(new Date('2026-08-02T00:00:00Z')), '2026-07-13T00:00:00.000Z'); +});