Carry a contact's provenance, which the crawler was failing without

packages/db/src/authors.js

@@ -240,7 +240,7 @@ export async function addAuthorLinks(db, authorId, links) {
         link.network,
         link.url,
         link.handle || null,
-        link.source,
+        linkSource(link),
         link.verified ? 1 : 0,
         nowIso(),
       ],
@@ -301,7 +301,7 @@ export async function addFeedLinks(db, feedId, links) {
         link.network,
         link.url,
         link.handle || null,
-        link.source,
+        linkSource(link),
         link.verified ? 1 : 0,
         nowIso(),
       ],
@@ -685,6 +685,31 @@ export async function feedHasAuthors(db, feedId) {
  * @param {Array<object>} [input.feedLinks] accounts to file under the feed
  * @returns {Array<{ sql: string, args: unknown[] }>} in dependency order
  */
+/**
+ * Where a link was found, as a string the database will accept.
+ *
+ * `author_links.source` and `feed_links.source` are both `not null`, and this is
+ * the last place before the wire that can say so. It matters more than a
+ * defensive default usually does, because of *how* the remote client fails:
+ * `undefined` is not a bindable libSQL value, so hrana throws `Unsupported type
+ * of value` while the statement is being serialized -- before any SQL runs, with
+ * no column named and no row to point at. That error surfaced as
+ * `could not be crawled`, which reads like a publisher who is down.
+ *
+ * It cost the directory a day. Every Substack newsletter emits `<itunes:owner>`
+ * and no other byline, so `feedContacts` harvested the mailbox, dropped the
+ * provenance, and every one of those feeds failed its crawl at the write --
+ * 985 of 1,385 crawls in the hour this was found. Local SQLite binds `undefined`
+ * as null without complaint, so the tests and every local run passed.
+ *
+ * @param {{ source?: unknown }} link
+ * @returns {string}
+ */
+function linkSource(link) {
+  const source = link?.source;
+  return typeof source === 'string' && source !== '' ? source : 'feed-document';
+}
+
 export function creditStatements({ feedId, identityKey, slug, person, authorLinks = [], feedLinks = [] }) {
   const now = nowIso();
   const statements = [];
@@ -765,7 +790,7 @@ export function creditStatements({ feedId, identityKey, slug, person, authorLink
         link.network,
         link.url,
         link.handle || null,
-        link.source,
+        linkSource(link),
         link.verified ? 1 : 0,
         now,
         identityKey,
@@ -802,7 +827,7 @@ export function feedLinkStatements(feedId, links) {
         link.network,
         link.url,
         link.handle || null,
-        link.source,
+        linkSource(link),
         link.verified ? 1 : 0,
         nowIso(),
       ],

packages/db/test/link-binds.test.js

@@ -0,0 +1,98 @@
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+
+import { creditStatements, feedLinkStatements } from '../src/authors.js';
+
+/**
+ * Everything the remote libSQL client will accept as a bound parameter.
+ *
+ * Mirrors `valueToProto` in @libsql/hrana-client: null, string, finite number,
+ * bigint, boolean, ArrayBuffer, Uint8Array, Date and any other object (which is
+ * stringified). What is left -- `undefined`, symbols and functions -- throws
+ * `TypeError: Unsupported type of value` during serialization.
+ *
+ * This is asserted here rather than left to an integration test because the
+ * local SQLite driver used by every test in this repo does *not* share the
+ * restriction: it binds `undefined` as null and passes. The only place the
+ * difference shows up is production.
+ *
+ * @param {unknown} value
+ * @returns {boolean}
+ */
+function bindable(value) {
+  if (value === null) return true;
+  const type = typeof value;
+  if (type === 'undefined' || type === 'symbol' || type === 'function') return false;
+  if (type === 'number') return Number.isFinite(value);
+  return true;
+}
+
+/**
+ * @param {Array<{ sql: string, args: unknown[] }>} statements
+ */
+function assertBindable(statements) {
+  for (const statement of statements) {
+    for (const [index, arg] of statement.args.entries()) {
+      assert.ok(
+        bindable(arg),
+        `arg ${index} (${String(arg)}) cannot be bound: ${statement.sql.slice(0, 80)}`,
+      );
+    }
+  }
+}
+
+const person = {
+  name: 'Marta Nowak',
+  normName: 'marta nowak',
+  bio: '',
+  avatarUrl: '',
+  siteUrl: '',
+  email: 'marta@example.com',
+  confidence: 0.85,
+  role: 'owner',
+  evidence: 'itunes-owner',
+};
+
+test('a link that names no source is still storable', () => {
+  // The crawler-stopping bug, at the layer that has to be right whatever the
+  // callers do. `feed_links.source` and `author_links.source` are `not null`,
+  // and a link arriving without one used to bind `undefined` -- which the
+  // remote client refuses outright, failing the entire crawl transaction that
+  // carried the feed row and its posts. Every Substack newsletter in the
+  // directory produced exactly this shape.
+  const link = { network: 'email', url: 'mailto:marta@example.com' };
+
+  const statements = creditStatements({
+    feedId: 'feed-1',
+    identityKey: 'marta@example.com',
+    slug: 'marta-nowak',
+    person,
+    authorLinks: [link],
+    feedLinks: [link],
+  });
+
+  assertBindable(statements);
+  assertBindable(feedLinkStatements('feed-1', [link]));
+});
+
+test('a link that does name its source keeps it', () => {
+  const link = { network: 'email', url: 'mailto:marta@example.com', source: 'rel-me' };
+  const [statement] = feedLinkStatements('feed-1', [link]);
+
+  assert.ok(statement.args.includes('rel-me'));
+  assertBindable([statement]);
+});
+
+test('a whole credit binds cleanly when the person is bare', () => {
+  // A credit carrying nothing but a name -- no bio, no avatar, no site, no
+  // email -- is the common case on the small web, and every one of those holes
+  // is a bound parameter.
+  const statements = creditStatements({
+    feedId: 'feed-1',
+    identityKey: 'someone@example.com',
+    slug: 'someone',
+    person: { name: 'Someone', confidence: 0.4, role: 'author' },
+  });
+
+  assertBindable(statements);
+});

packages/feed/src/identity.js

@@ -749,10 +749,16 @@ export function channelCreditInputs(channel, format, base = '') {
  * routed it there -- collecting it again here would file one address in two
  * places and count it twice.
  *
+ * Each contact carries the channel element it was read from, the same way a
+ * credit does. `feed_links.source` is `not null` and is what the page means by
+ * "where we found this" -- a mailbox harvested from `webMaster` is a weaker
+ * claim than one from `itunes:owner`, and dropping the provenance here lost
+ * that distinction *and* left the column with nothing to store.
+ *
  * @param {any} channel the raw parsed channel/feed object
  * @param {'rss'|'atom'|'rdf'|'json'} format
  * @param {string} [base]
- * @returns {Array<{ url: string, network: string }>}
+ * @returns {Array<{ url: string, network: string, source: string }>}
  */
 export function feedContacts(channel, format, base = '') {
   const out = [];
@@ -776,7 +782,7 @@ export function feedContacts(channel, format, base = '') {
       const link = candidate ? classifyLink(candidate, base) : null;
       if (!link || seen.has(link.url)) continue;
       seen.add(link.url);
-      out.push({ url: link.url, network: link.network });
+      out.push({ url: link.url, network: link.network, source: input.source });
     }
   }
 

packages/feed/src/parse.js

@@ -118,6 +118,31 @@ function hasVideoEnclosure(item) {
   );
 }
 
+/**
+ * Does this element carry audio?
+ *
+ * @param {any} item
+ * @returns {boolean}
+ */
+function hasAudioEnclosure(item) {
+  if (arr(item?.enclosure).some((e) => AUDIO_TYPE.test(String(e?.['@type'] ?? '')))) return true;
+  return arr(item?.link).some(
+    (l) => l?.['@rel'] === 'enclosure' && AUDIO_TYPE.test(String(l?.['@type'] ?? '')),
+  );
+}
+
+/**
+ * Tags no platform sets unless it is publishing a show.
+ *
+ * The rest of `PODCAST_CHANNEL_TAGS` is weaker than it looks. Substack emits
+ * `<itunes:owner>` on every publication it hosts, podcast or not, so on its own
+ * that tag files a text newsletter under podcasts -- and Substack is thousands
+ * of feeds here. `itunes:type` and the `podcast:` namespace are different: they
+ * are written by podcast hosting, for podcast directories, and nothing else has
+ * a reason to emit them.
+ */
+const PODCAST_DECLARED_TAGS = ['itunes:type', 'podcast:guid', 'podcast:medium'];
+
 /**
  * How much prose an item carries of its own, in characters of text.
  *
@@ -406,8 +431,9 @@ function isNewsroom(channel, items) {
  * guessing. Then YouTube, because a channel feed says so in its own namespace
  * and nothing else needs weighing. Then video, which is an enclosure *and*
  * corroboration that the enclosure is the point. Then podcast, which is a
- * publisher who filled in the podcast namespaces. Everything else is a blog,
- * which is what the overwhelming majority of the directory is.
+ * publisher who filled in the podcast namespaces *and* ships audio, or who
+ * declared a show outright. Everything else is a blog, which is what the
+ * overwhelming majority of the directory is.
  *
  * One correction, arrived at twice from opposite ends of the directory: an
  * attachment is not a genre. A post with a file on it is still a post, and
@@ -455,7 +481,23 @@ function kindOfChannel(channel, items) {
   const withVideo = sample.filter(hasVideoEnclosure);
   if (withVideo.length > 0 && (podcastTags || isShowShaped(sample, withVideo))) return KIND_VIDEO;
 
-  if (podcastTags) return KIND_PODCAST;
+  // A podcast publishes audio, and the same correction applies here as to video
+  // above: the tag has to be corroborated. A show attaches an episode to every
+  // entry, so one audio enclosure anywhere in the sample is enough -- and a feed
+  // with podcast tags and not a single audio file in five items is a newsletter
+  // whose host fills in the iTunes block. Substack does that for every
+  // publication it serves, which put thousands of text newsletters under
+  // /podcasts.
+  //
+  // A declaration still stands on its own: `itunes:type` and the `podcast:`
+  // namespace are the publisher stating what they made, and a show that has not
+  // released its first episode yet is still a podcast. So is a feed we have no
+  // items for -- with nothing sampled there is no evidence to corroborate, and
+  // the tags are the only thing said.
+  const declaredPodcast = PODCAST_DECLARED_TAGS.some((tag) => channel?.[tag] !== undefined);
+  if (podcastTags && (declaredPodcast || sample.length === 0 || sample.some(hasAudioEnclosure))) {
+    return KIND_PODCAST;
+  }
 
   // No audio branch, deliberately. Audio without a declared medium is a blog
   // that narrated itself, and `declaredMedium` above is the only way to music.

packages/feed/test/contacts.test.js

@@ -15,7 +15,7 @@ test('a feed whose only byline is a role keeps the mailbox that role published',
 
   assert.deepEqual(feedCredits(channel, [], 'rss'), []);
   assert.deepEqual(feedContacts(channel, 'rss'), [
-    { url: 'mailto:marta@example.com', network: 'email' },
+    { url: 'mailto:marta@example.com', network: 'email', source: 'itunes-owner' },
   ]);
 });
 
@@ -54,7 +54,7 @@ test('a profile published beside a rejected name is kept as the feed’s', () =>
 
   assert.deepEqual(feedCredits(channel, [], 'atom'), []);
   assert.deepEqual(feedContacts(channel, 'atom'), [
-    { url: 'https://github.com/wirecutter', network: 'github' },
+    { url: 'https://github.com/wirecutter', network: 'github', source: 'atom-feed-author' },
   ]);
 });
 
@@ -77,12 +77,49 @@ test('the same address published twice is one contact', () => {
     'itunes:owner': { 'itunes:name': 'Editorial Team', 'itunes:email': 'marta@example.com' },
   };
 
+  // The first element to publish it is the one credited with finding it, so a
+  // deduplicated address keeps the stronger provenance rather than the last.
   assert.deepEqual(feedContacts(channel, 'rss'), [
-    { url: 'mailto:marta@example.com', network: 'email' },
+    { url: 'mailto:marta@example.com', network: 'email', source: 'managing-editor' },
   ]);
 });
 
 test('a feed that credits nobody at all offers no contacts', () => {
   assert.deepEqual(feedContacts({ title: 'A Blog' }, 'rss'), []);
   assert.deepEqual(feedContacts(null, 'rss'), []);
 });
+
+test('every contact says where it was found, because the column demands it', () => {
+  // The bug this pins, and it stopped the crawler for a day.
+  //
+  // `feed_links.source` and `author_links.source` are both `not null`, and a
+  // contact used to be built as `{ url, network }` with the provenance dropped.
+  // The remote libSQL client cannot bind `undefined` at all -- it throws
+  // `Unsupported type of value` while serializing the statement, before any SQL
+  // runs -- so the whole crawl failed at the write and the feed was recorded as
+  // uncrawlable. Local SQLite binds it as null without complaining, which is
+  // why every test and every local run passed.
+  //
+  // Substack is the population that found it: it emits `<itunes:owner>` on
+  // every publication it hosts and no other byline, so every Substack
+  // newsletter in the directory took this path.
+  // Copied from https://nemtodamulher.substack.com/feed, which is the shape
+  // every publication on that platform ships: a webMaster address, and an
+  // iTunes block naming the publication rather than a person.
+  const channel = {
+    title: 'Newsletter Nem Toda Mulher',
+    webMaster: 'nemtodamulher@substack.com',
+    'itunes:author': 'Newsletter Nem Toda Mulher',
+    'itunes:owner': {
+      'itunes:name': 'Newsletter Nem Toda Mulher',
+      'itunes:email': 'nemtodamulher@substack.com',
+    },
+  };
+
+  const contacts = feedContacts(channel, 'rss');
+  assert.equal(contacts.length, 1);
+  for (const contact of contacts) {
+    assert.equal(typeof contact.source, 'string');
+    assert.notEqual(contact.source, '');
+  }
+});

packages/feed/test/parse.test.js

@@ -217,6 +217,62 @@ test('a feed carrying the podcast namespaces is a podcast', () => {
   assert.equal(feed.imageUrl, 'https://linuxmatters.sh/cover.png');
 });
 
+const SUBSTACK_RSS = `<?xml version="1.0"?>
+<rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd"
+     xmlns:content="http://purl.org/rss/1.0/modules/content/">
+  <channel>
+    <title>Newsletter Nem Toda Mulher</title>
+    <link>https://nemtodamulher.substack.com</link>
+    <description>Por Vera Iaconelli e Carol Pires</description>
+    <generator>Substack</generator>
+    <webMaster>nemtodamulher@substack.com</webMaster>
+    <itunes:owner>
+      <itunes:name>Newsletter Nem Toda Mulher</itunes:name>
+      <itunes:email>nemtodamulher@substack.com</itunes:email>
+    </itunes:owner>
+    <item>
+      <title>Nunca soube a escalação de um time de futebol</title>
+      <guid>https://nemtodamulher.substack.com/p/nunca-soube</guid>
+      <description>Uma conversa sobre futebol e meninos</description>
+      <enclosure url="https://substack.example/cover.gif" length="0" type="image/jpeg"/>
+    </item>
+  </channel>
+</rss>`;
+
+test('a newsletter whose host fills in the iTunes block is not a podcast', () => {
+  // Substack emits <itunes:owner> on every publication it hosts, podcast or
+  // not, and nothing else that looks like a show: no itunes:type, no podcast:
+  // namespace, and image enclosures rather than audio ones. On the strength of
+  // that one tag the whole platform -- thousands of feeds here -- was filed
+  // under /podcasts.
+  //
+  // The same correction the video branch already makes: an attachment is not a
+  // genre, and the tag has to be corroborated by what the feed actually ships.
+  assert.equal(parseFeed(SUBSTACK_RSS).kind, 'blog');
+});
+
+test('a show still reads as a podcast on its tags and its audio', () => {
+  // The other side of the guard above: everything that genuinely is a podcast
+  // must stay one. Linux Matters declares itunes:type and podcast:guid *and*
+  // attaches an mp3, so it passes on either half of the test.
+  assert.equal(parseFeed(PODCAST_RSS).kind, 'podcast');
+});
+
+test('a podcast that has not released an episode yet is still a podcast', () => {
+  // A declaration stands on its own. itunes:type and the podcast: namespace are
+  // written by podcast hosting for podcast directories, so there is nothing to
+  // corroborate -- and a show with no episodes has nothing to corroborate with.
+  const rss = `<?xml version="1.0"?>
+<rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd"><channel>
+  <title>A new show</title>
+  <link>https://new.example/</link>
+  <description>Coming soon</description>
+  <itunes:type>episodic</itunes:type>
+</channel></rss>`;
+
+  assert.equal(parseFeed(rss).kind, 'podcast');
+});
+
 test('audio without the podcast namespaces is a blog, not music', () => {
   // Attaching an mp3 to a post says nothing about what the feed is: a narrated
   // article, a conference talk and a cross-posted episode all look like this,