diff --git a/scripts/ad-networks/niche-sort.mjs b/scripts/ad-networks/niche-sort.mjs index 5e1ab18b..32b100ff 100644 --- a/scripts/ad-networks/niche-sort.mjs +++ b/scripts/ad-networks/niche-sort.mjs @@ -15,18 +15,24 @@ import { jobAlert, safeError } from './alerts.mjs'; export function validateNicheBatch(input, output) { const domains = new Set(input.map(row => row.domain)); if (domains.size !== input.length) throw new Error('Duplicate input domains'); - const rows = [], seen = new Set(); + // The model sometimes drops a site, mangles a line, or answers a site twice in a long batch. Keep + // only clean answers to sites it was asked about, once each. A site without one isn't checkpointed, + // so the next run asks again. An answer with nothing usable is still a failed call. + const answers = new Map(), counts = new Map(); for (const line of output.split(/\r?\n/).filter(line => line.trim())) { - const raw = JSON.parse(line); - if (!raw || Object.keys(raw).sort().join(',') !== 'confidence,domain,niche') throw new Error('Unexpected niche result fields'); - const row = validateNicheResult(raw); - if (row.domain !== raw.domain || !domains.has(row.domain)) throw new Error(`Unexpected domain: ${row.domain}`); - if (seen.has(row.domain)) throw new Error(`Duplicate result: ${row.domain}`); - seen.add(row.domain); rows.push(row); + let raw; + try { raw = JSON.parse(line); } catch { continue; } + // Count every answer to an input site before checking it, so a bad second answer still + // marks the site as answered twice. + if (domains.has(raw?.domain)) counts.set(raw.domain, (counts.get(raw.domain) ?? 0) + 1); + try { + if (!raw || Object.keys(raw).sort().join(',') !== 'confidence,domain,niche') continue; + const row = validateNicheResult(raw); + if (row.domain === raw.domain && domains.has(row.domain)) answers.set(row.domain, row); + } catch { continue; } } - // The model sometimes drops a site from a long batch. Keep the answers it gave: a missing site - // isn't checkpointed, so the next run asks again. An empty answer is still a failed call. - if (!seen.size) throw new Error(`No niche results for a batch of ${domains.size}`); + const rows = [...answers.values()].filter(row => counts.get(row.domain) === 1); + if (!rows.length) throw new Error(`No usable niche results for a batch of ${domains.size}`); return rows; } diff --git a/tests/pages/ad-networks/niche-sort.test.mjs b/tests/pages/ad-networks/niche-sort.test.mjs index 5c0b019d..cc9f2d99 100644 --- a/tests/pages/ad-networks/niche-sort.test.mjs +++ b/tests/pages/ad-networks/niche-sort.test.mjs @@ -9,18 +9,24 @@ import { adNetworkNicheShares, hasAdNetworkNicheCoverage } from '../../../lib/ad const input = [{ domain: 'example.com' }, { domain: 'example.org' }]; const result = (domain, niche = 'food', confidence = 'high') => ({ domain, niche, confidence }); const lines = rows => rows.map(row => JSON.stringify(row)).join('\n'); -test('model validation takes only input domains once each, with known labels and confidence', () => { +test('model validation keeps only clean answers to input domains, once each', () => { const correct = [result('example.com'), result('example.org', null, 'low')]; assert.deepEqual(validateNicheBatch(input, lines(correct)), correct); - // A dropped site is left for the next run; an empty answer fails. + // A dropped, mangled, or repeated answer is left for the next run; nothing usable fails. assert.deepEqual(validateNicheBatch(input, lines([correct[0]])), [correct[0]]); - assert.throws(() => validateNicheBatch(input, ''), /No niche results/); - for (const bad of [[correct[0], correct[0]], [...correct, result('extra.com')], - [result('example.com', 'invented'), correct[1]], [result('example.com', 'food', 1.5), correct[1]], - [{ ...correct[0], comment: 'extra' }, correct[1]], [result('https://example.com'), correct[1]]]) { - assert.throws(() => validateNicheBatch(input, lines(bad))); + for (const bad of [result('extra.com'), result('example.com', 'invented'), result('example.com', 'food', 1.5), + { ...correct[0], comment: 'extra' }, result('https://example.com'), result('bad domain')]) { + assert.deepEqual(validateNicheBatch(input, lines([bad, correct[1]])), [correct[1]]); + } + assert.deepEqual(validateNicheBatch(input, `not json\n${lines([correct[1]])}`), [correct[1]]); + // A site answered twice is dropped, even when one of the answers is malformed, in either order. + const malformed = { ...result('example.com', 'tech'), comment: 'extra' }; + for (const twice of [[correct[0], result('example.com', 'tech')], [correct[0], malformed], [malformed, correct[0]]]) { + assert.deepEqual(validateNicheBatch(input, lines([...twice, correct[1]])), [correct[1]]); + } + for (const empty of ['', '```json\n{}\n```', lines([correct[0], correct[0]])]) { + assert.throws(() => validateNicheBatch(input, empty), /No usable niche results/); } - assert.throws(() => validateNicheBatch(input, '```json\n{}\n```')); assert.throws(() => nicheSortArgs(['--environment', 'production', '--apply']), /confirmation/); }); test('niche imports withhold uncertain guesses, serialize against sync, and rollback on failure', async () => {