Skip malformed niche answers instead of ending the weekly run

scripts/ad-networks/niche-sort.mjs

@@ -15,18 +15,24 @@ import { jobAlert, safeError } from './alerts.mjs';
 export function validateNicheBatch(input, output) {
   const domains = new Set(input.map(row => row.domain));
   if (domains.size !== input.length) throw new Error('Duplicate input domains');
-  const rows = [], seen = new Set();
+  // The model sometimes drops a site, mangles a line, or answers a site twice in a long batch. Keep
+  // only clean answers to sites it was asked about, once each. A site without one isn't checkpointed,
+  // so the next run asks again. An answer with nothing usable is still a failed call.
+  const answers = new Map(), counts = new Map();
   for (const line of output.split(/\r?\n/).filter(line => line.trim())) {
-    const raw = JSON.parse(line);
-    if (!raw || Object.keys(raw).sort().join(',') !== 'confidence,domain,niche') throw new Error('Unexpected niche result fields');
-    const row = validateNicheResult(raw);
-    if (row.domain !== raw.domain || !domains.has(row.domain)) throw new Error(`Unexpected domain: ${row.domain}`);
-    if (seen.has(row.domain)) throw new Error(`Duplicate result: ${row.domain}`);
-    seen.add(row.domain); rows.push(row);
+    let raw;
+    try { raw = JSON.parse(line); } catch { continue; }
+    // Count every answer to an input site before checking it, so a bad second answer still
+    // marks the site as answered twice.
+    if (domains.has(raw?.domain)) counts.set(raw.domain, (counts.get(raw.domain) ?? 0) + 1);
+    try {
+      if (!raw || Object.keys(raw).sort().join(',') !== 'confidence,domain,niche') continue;
+      const row = validateNicheResult(raw);
+      if (row.domain === raw.domain && domains.has(row.domain)) answers.set(row.domain, row);
+    } catch { continue; }
   }
-  // The model sometimes drops a site from a long batch. Keep the answers it gave: a missing site
-  // isn't checkpointed, so the next run asks again. An empty answer is still a failed call.
-  if (!seen.size) throw new Error(`No niche results for a batch of ${domains.size}`);
+  const rows = [...answers.values()].filter(row => counts.get(row.domain) === 1);
+  if (!rows.length) throw new Error(`No usable niche results for a batch of ${domains.size}`);
   return rows;
 }
 

tests/pages/ad-networks/niche-sort.test.mjs

@@ -9,18 +9,24 @@ import { adNetworkNicheShares, hasAdNetworkNicheCoverage } from '../../../lib/ad
 const input = [{ domain: 'example.com' }, { domain: 'example.org' }];
 const result = (domain, niche = 'food', confidence = 'high') => ({ domain, niche, confidence });
 const lines = rows => rows.map(row => JSON.stringify(row)).join('\n');
-test('model validation takes only input domains once each, with known labels and confidence', () => {
+test('model validation keeps only clean answers to input domains, once each', () => {
   const correct = [result('example.com'), result('example.org', null, 'low')];
   assert.deepEqual(validateNicheBatch(input, lines(correct)), correct);
-  // A dropped site is left for the next run; an empty answer fails.
+  // A dropped, mangled, or repeated answer is left for the next run; nothing usable fails.
   assert.deepEqual(validateNicheBatch(input, lines([correct[0]])), [correct[0]]);
-  assert.throws(() => validateNicheBatch(input, ''), /No niche results/);
-  for (const bad of [[correct[0], correct[0]], [...correct, result('extra.com')],
-    [result('example.com', 'invented'), correct[1]], [result('example.com', 'food', 1.5), correct[1]],
-    [{ ...correct[0], comment: 'extra' }, correct[1]], [result('https://example.com'), correct[1]]]) {
-    assert.throws(() => validateNicheBatch(input, lines(bad)));
+  for (const bad of [result('extra.com'), result('example.com', 'invented'), result('example.com', 'food', 1.5),
+    { ...correct[0], comment: 'extra' }, result('https://example.com'), result('bad domain')]) {
+    assert.deepEqual(validateNicheBatch(input, lines([bad, correct[1]])), [correct[1]]);
+  }
+  assert.deepEqual(validateNicheBatch(input, `not json\n${lines([correct[1]])}`), [correct[1]]);
+  // A site answered twice is dropped, even when one of the answers is malformed, in either order.
+  const malformed = { ...result('example.com', 'tech'), comment: 'extra' };
+  for (const twice of [[correct[0], result('example.com', 'tech')], [correct[0], malformed], [malformed, correct[0]]]) {
+    assert.deepEqual(validateNicheBatch(input, lines([...twice, correct[1]])), [correct[1]]);
+  }
+  for (const empty of ['', '```json\n{}\n```', lines([correct[0], correct[0]])]) {
+    assert.throws(() => validateNicheBatch(input, empty), /No usable niche results/);
   }
-  assert.throws(() => validateNicheBatch(input, '```json\n{}\n```'));
   assert.throws(() => nicheSortArgs(['--environment', 'production', '--apply']), /confirmation/);
 });
 test('niche imports withhold uncertain guesses, serialize against sync, and rollback on failure', async () => {