From 1f1aaad17b1d799360f7c87a31c80a734796c9ea Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 19:07:52 -0700 Subject: [PATCH 1/8] feat(quality): add deterministic near-duplicate detection --- src/rag-eval/near-duplicates.ts | 272 ++++++++++++++++++++++++++++++++ 1 file changed, 272 insertions(+) create mode 100644 src/rag-eval/near-duplicates.ts diff --git a/src/rag-eval/near-duplicates.ts b/src/rag-eval/near-duplicates.ts new file mode 100644 index 0000000..bb187b5 --- /dev/null +++ b/src/rag-eval/near-duplicates.ts @@ -0,0 +1,272 @@ +import type { KnowledgePage } from '../types' + +export interface NearDuplicateDetectionOptions { + /** Jaccard similarity at or above which a pair is reported. Defaults to 0.82. */ + threshold?: number + /** Minimum normalized characters before a page participates. Defaults to 80. */ + minCharacters?: number + /** Word-shingle width. Defaults to 5. */ + wordShingleSize?: number + /** Character-shingle width used for text with too few words. Defaults to 16. */ + characterShingleSize?: number + /** Ignore a shingle occurring in more pages than this. Defaults to 64. */ + maxDocumentFrequency?: number + /** Stop adding candidate pairs after this many. Defaults to 250,000. */ + maxCandidatePairs?: number + /** Maximum duplicate pairs retained in the report. Defaults to 1,000. */ + maxReportedPairs?: number +} + +export interface NearDuplicatePair { + leftPageId: string + leftPath: string + rightPageId: string + rightPath: string + similarity: number + intersectionSize: number + unionSize: number + exact: boolean +} + +export interface NearDuplicateReport { + algorithm: 'deterministic-shingle-jaccard-v1' + threshold: number + pageCount: number + eligiblePageCount: number + comparedPairCount: number + duplicatePairCount: number + duplicatePageCount: number + duplicatePageRate: number + truncated: boolean + pairs: NearDuplicatePair[] +} + +interface PreparedPage { + page: KnowledgePage + normalized: string + shingles: Set +} + +/** + * Detect exact and near-duplicate pages without a model, embedding service, or clock. + * + * The detector uses word shingles when prose has enough tokens and Unicode + * character shingles otherwise. An inverted index proposes candidate pairs; + * high-document-frequency shingles are ignored so common boilerplate cannot + * make candidate generation quadratic. Exact normalized copies are always + * proposed, even when every shingle is common. + */ +export function detectNearDuplicatePages( + pages: readonly KnowledgePage[], + options: NearDuplicateDetectionOptions = {}, +): NearDuplicateReport { + const threshold = finiteUnitInterval(options.threshold ?? 0.82, 'threshold') + const minCharacters = positiveInteger(options.minCharacters ?? 80, 'minCharacters') + const wordShingleSize = positiveInteger(options.wordShingleSize ?? 5, 'wordShingleSize') + const characterShingleSize = positiveInteger( + options.characterShingleSize ?? 16, + 'characterShingleSize', + ) + const maxDocumentFrequency = positiveInteger( + options.maxDocumentFrequency ?? 64, + 'maxDocumentFrequency', + ) + const maxCandidatePairs = positiveInteger( + options.maxCandidatePairs ?? 250_000, + 'maxCandidatePairs', + ) + const maxReportedPairs = nonNegativeInteger( + options.maxReportedPairs ?? 1_000, + 'maxReportedPairs', + ) + + const prepared = pages + .map((page) => preparePage(page, wordShingleSize, characterShingleSize)) + .filter((item): item is PreparedPage => item !== null && item.normalized.length >= minCharacters) + .sort((left, right) => left.page.path.localeCompare(right.page.path)) + + const candidateKeys = new Set() + const exactGroups = new Map() + const byShingle = new Map() + for (const [index, item] of prepared.entries()) { + exactGroups.set(item.normalized, [...(exactGroups.get(item.normalized) ?? []), index]) + for (const shingle of item.shingles) { + byShingle.set(shingle, [...(byShingle.get(shingle) ?? []), index]) + } + } + + let truncated = false + const addPair = (left: number, right: number): void => { + if (left === right) return + if (candidateKeys.size >= maxCandidatePairs) { + truncated = true + return + } + candidateKeys.add(pairKey(Math.min(left, right), Math.max(left, right))) + } + + for (const group of exactGroups.values()) addGroupPairs(group, addPair) + for (const shingle of [...byShingle.keys()].sort()) { + const group = byShingle.get(shingle)! + if (group.length > maxDocumentFrequency) continue + addGroupPairs(group, addPair) + if (truncated) break + } + + const duplicatePages = new Set() + const pairs: NearDuplicatePair[] = [] + let duplicatePairCount = 0 + let comparedPairCount = 0 + for (const key of [...candidateKeys].sort(pairKeyOrder)) { + const [leftIndex, rightIndex] = parsePairKey(key) + const left = prepared[leftIndex]! + const right = prepared[rightIndex]! + comparedPairCount += 1 + const exact = left.normalized === right.normalized + const overlap = jaccard(left.shingles, right.shingles) + if (!exact && overlap.similarity < threshold) continue + duplicatePairCount += 1 + duplicatePages.add(left.page.id) + duplicatePages.add(right.page.id) + if (pairs.length >= maxReportedPairs) { + truncated = true + continue + } + pairs.push({ + leftPageId: left.page.id, + leftPath: left.page.path, + rightPageId: right.page.id, + rightPath: right.page.path, + similarity: exact ? 1 : round(overlap.similarity), + intersectionSize: overlap.intersectionSize, + unionSize: overlap.unionSize, + exact, + }) + } + + pairs.sort( + (left, right) => + right.similarity - left.similarity || + left.leftPath.localeCompare(right.leftPath) || + left.rightPath.localeCompare(right.rightPath), + ) + + return { + algorithm: 'deterministic-shingle-jaccard-v1', + threshold, + pageCount: pages.length, + eligiblePageCount: prepared.length, + comparedPairCount, + duplicatePairCount, + duplicatePageCount: duplicatePages.size, + duplicatePageRate: prepared.length === 0 ? 0 : duplicatePages.size / prepared.length, + truncated, + pairs, + } +} + +function preparePage( + page: KnowledgePage, + wordShingleSize: number, + characterShingleSize: number, +): PreparedPage | null { + const normalized = normalizePageText(`${page.title}\n${page.text}`) + if (normalized.length === 0) return null + const words = normalized.match(/[\p{L}\p{N}]+/gu) ?? [] + const shingles = + words.length >= wordShingleSize + 2 + ? sequenceShingles(words, wordShingleSize) + : characterShingles(normalized.replace(/\s+/g, ''), characterShingleSize) + if (shingles.size === 0) shingles.add(normalized) + return { page, normalized, shingles } +} + +export function normalizePageText(value: string): string { + return value + .normalize('NFKC') + .toLocaleLowerCase('und') + .replace(/https?:\/\/\S+/g, '') + .replace(/\s+/g, ' ') + .trim() +} + +function sequenceShingles(items: readonly string[], width: number): Set { + const out = new Set() + if (items.length < width) return out + for (let index = 0; index <= items.length - width; index += 1) { + out.add(items.slice(index, index + width).join('\u001f')) + } + return out +} + +function characterShingles(value: string, width: number): Set { + const chars = [...value] + const out = new Set() + if (chars.length < width) return out + for (let index = 0; index <= chars.length - width; index += 1) { + out.add(chars.slice(index, index + width).join('')) + } + return out +} + +function addGroupPairs(group: readonly number[], add: (left: number, right: number) => void): void { + for (let left = 0; left < group.length; left += 1) { + for (let right = left + 1; right < group.length; right += 1) { + add(group[left]!, group[right]!) + } + } +} + +function jaccard(left: ReadonlySet, right: ReadonlySet): { + similarity: number + intersectionSize: number + unionSize: number +} { + const [small, large] = left.size <= right.size ? [left, right] : [right, left] + let intersectionSize = 0 + for (const item of small) if (large.has(item)) intersectionSize += 1 + const unionSize = left.size + right.size - intersectionSize + return { + similarity: unionSize === 0 ? 1 : intersectionSize / unionSize, + intersectionSize, + unionSize, + } +} + +function pairKey(left: number, right: number): string { + return `${left}:${right}` +} + +function parsePairKey(value: string): [number, number] { + const [left, right] = value.split(':') + return [Number(left), Number(right)] +} + +function pairKeyOrder(left: string, right: string): number { + const [leftA, leftB] = parsePairKey(left) + const [rightA, rightB] = parsePairKey(right) + return leftA - rightA || leftB - rightB +} + +function finiteUnitInterval(value: number, name: string): number { + if (!Number.isFinite(value) || value < 0 || value > 1) { + throw new Error(`${name} must be a finite number in [0, 1]`) + } + return value +} + +function positiveInteger(value: number, name: string): number { + if (!Number.isInteger(value) || value < 1) throw new Error(`${name} must be a positive integer`) + return value +} + +function nonNegativeInteger(value: number, name: string): number { + if (!Number.isInteger(value) || value < 0) { + throw new Error(`${name} must be a non-negative integer`) + } + return value +} + +function round(value: number): number { + return Math.round(value * 1_000_000) / 1_000_000 +} From 253d870daa6349c6aaf704bb106a1d6afc324e03 Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 19:11:27 -0700 Subject: [PATCH 2/8] fix(quality): bound candidate generation deterministically --- src/rag-eval/near-duplicates.ts | 55 +++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/src/rag-eval/near-duplicates.ts b/src/rag-eval/near-duplicates.ts index bb187b5..86b70d0 100644 --- a/src/rag-eval/near-duplicates.ts +++ b/src/rag-eval/near-duplicates.ts @@ -33,6 +33,7 @@ export interface NearDuplicateReport { threshold: number pageCount: number eligiblePageCount: number + candidatePairCount: number comparedPairCount: number duplicatePairCount: number duplicatePageCount: number @@ -54,14 +55,16 @@ interface PreparedPage { * character shingles otherwise. An inverted index proposes candidate pairs; * high-document-frequency shingles are ignored so common boilerplate cannot * make candidate generation quadratic. Exact normalized copies are always - * proposed, even when every shingle is common. + * proposed before ordinary shingle candidates, even when every shingle is + * common. Every limit and sort order is explicit so the same corpus produces + * the same report on every run. */ export function detectNearDuplicatePages( pages: readonly KnowledgePage[], options: NearDuplicateDetectionOptions = {}, ): NearDuplicateReport { const threshold = finiteUnitInterval(options.threshold ?? 0.82, 'threshold') - const minCharacters = positiveInteger(options.minCharacters ?? 80, 'minCharacters') + const minCharacters = nonNegativeInteger(options.minCharacters ?? 80, 'minCharacters') const wordShingleSize = positiveInteger(options.wordShingleSize ?? 5, 'wordShingleSize') const characterShingleSize = positiveInteger( options.characterShingleSize ?? 16, @@ -96,24 +99,30 @@ export function detectNearDuplicatePages( } let truncated = false - const addPair = (left: number, right: number): void => { - if (left === right) return + const addPair = (left: number, right: number): boolean => { + if (left === right) return true + const key = pairKey(Math.min(left, right), Math.max(left, right)) + if (candidateKeys.has(key)) return true if (candidateKeys.size >= maxCandidatePairs) { truncated = true - return + return false } - candidateKeys.add(pairKey(Math.min(left, right), Math.max(left, right))) + candidateKeys.add(key) + return true } - for (const group of exactGroups.values()) addGroupPairs(group, addPair) - for (const shingle of [...byShingle.keys()].sort()) { - const group = byShingle.get(shingle)! - if (group.length > maxDocumentFrequency) continue - addGroupPairs(group, addPair) - if (truncated) break + for (const normalized of [...exactGroups.keys()].sort()) { + if (!addGroupPairs(exactGroups.get(normalized)!, addPair)) break + } + if (!truncated) { + for (const shingle of [...byShingle.keys()].sort()) { + const group = byShingle.get(shingle)! + if (group.length > maxDocumentFrequency) continue + if (!addGroupPairs(group, addPair)) break + } } - const duplicatePages = new Set() + const duplicatePageIndices = new Set() const pairs: NearDuplicatePair[] = [] let duplicatePairCount = 0 let comparedPairCount = 0 @@ -126,8 +135,8 @@ export function detectNearDuplicatePages( const overlap = jaccard(left.shingles, right.shingles) if (!exact && overlap.similarity < threshold) continue duplicatePairCount += 1 - duplicatePages.add(left.page.id) - duplicatePages.add(right.page.id) + duplicatePageIndices.add(leftIndex) + duplicatePageIndices.add(rightIndex) if (pairs.length >= maxReportedPairs) { truncated = true continue @@ -156,10 +165,12 @@ export function detectNearDuplicatePages( threshold, pageCount: pages.length, eligiblePageCount: prepared.length, + candidatePairCount: candidateKeys.size, comparedPairCount, duplicatePairCount, - duplicatePageCount: duplicatePages.size, - duplicatePageRate: prepared.length === 0 ? 0 : duplicatePages.size / prepared.length, + duplicatePageCount: duplicatePageIndices.size, + duplicatePageRate: + prepared.length === 0 ? 0 : round(duplicatePageIndices.size / prepared.length), truncated, pairs, } @@ -184,7 +195,7 @@ function preparePage( export function normalizePageText(value: string): string { return value .normalize('NFKC') - .toLocaleLowerCase('und') + .toLowerCase() .replace(/https?:\/\/\S+/g, '') .replace(/\s+/g, ' ') .trim() @@ -209,12 +220,16 @@ function characterShingles(value: string, width: number): Set { return out } -function addGroupPairs(group: readonly number[], add: (left: number, right: number) => void): void { +function addGroupPairs( + group: readonly number[], + add: (left: number, right: number) => boolean, +): boolean { for (let left = 0; left < group.length; left += 1) { for (let right = left + 1; right < group.length; right += 1) { - add(group[left]!, group[right]!) + if (!add(group[left]!, group[right]!)) return false } } + return true } function jaccard(left: ReadonlySet, right: ReadonlySet): { From 88d435fb6ced08fe9792f7b5280715c24b1188f8 Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 19:12:08 -0700 Subject: [PATCH 3/8] feat(quality): expose near-duplicate configuration and metrics --- src/rag-eval/contracts.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/rag-eval/contracts.ts b/src/rag-eval/contracts.ts index 2e6922b..bd27e3e 100644 --- a/src/rag-eval/contracts.ts +++ b/src/rag-eval/contracts.ts @@ -1,6 +1,7 @@ import type { ComparisonCost, JudgeConfig, Scenario } from '@tangle-network/agent-eval/campaign' import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' import type { RagGapFinding } from '../rag-improvement-loop' +import type { NearDuplicateDetectionOptions, NearDuplicateReport } from './near-duplicates' export type RagEvalProvider = | 'agent-knowledge' @@ -166,6 +167,10 @@ export interface KnowledgeBaseQualityOptions { strict?: boolean minCitationRate?: number maxStaleSourceRate?: number + /** Deterministic exact/near-duplicate detector configuration. */ + nearDuplicates?: NearDuplicateDetectionOptions + /** Maximum allowed fraction of eligible pages implicated in a duplicate pair. Defaults to 1. */ + maxNearDuplicatePageRate?: number } export interface KnowledgeBaseQualityReport { @@ -179,7 +184,13 @@ export interface KnowledgeBaseQualityReport { duplicate_source_hash_rate: number lint_error_count: number lint_warning_count: number + /** Added in the deterministic near-duplicate quality pass. */ + near_duplicate_page_rate?: number + near_duplicate_page_count?: number + near_duplicate_pair_count?: number } + /** Exact report from the deterministic detector; present when produced by this package. */ + nearDuplicates?: NearDuplicateReport findings: readonly RagGapFinding[] } From d501f5dc22d7b642ed5b65a8bd0f13f50abb73a5 Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 19:13:25 -0700 Subject: [PATCH 4/8] feat(quality): score and gate near-duplicate page rate --- src/rag-eval/knowledge-base.ts | 45 +++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/src/rag-eval/knowledge-base.ts b/src/rag-eval/knowledge-base.ts index 503a0bb..24fc157 100644 --- a/src/rag-eval/knowledge-base.ts +++ b/src/rag-eval/knowledge-base.ts @@ -3,6 +3,7 @@ import type { RagGapFinding } from '../rag-improvement-loop' import type { KnowledgeIndex } from '../types' import { validateKnowledgeIndex } from '../validate' import type { KnowledgeBaseQualityOptions, KnowledgeBaseQualityReport } from './contracts' +import { detectNearDuplicatePages } from './near-duplicates' export function scoreKnowledgeBaseIndex( index: KnowledgeIndex, @@ -25,6 +26,11 @@ export function scoreKnowledgeBaseIndex( (page) => sourceRefs(page.text).length > 0, ).length const pagesWithSources = index.pages.filter((page) => page.sourceIds.length > 0).length + const nearDuplicates = detectNearDuplicatePages(index.pages, options.nearDuplicates) + const maxNearDuplicatePageRate = unitInterval( + options.maxNearDuplicatePageRate ?? 1, + 'maxNearDuplicatePageRate', + ) const metrics = { page_count: index.pages.length, source_count: index.sources.length, @@ -35,6 +41,9 @@ export function scoreKnowledgeBaseIndex( index.sources.length === 0 ? 0 : duplicateSourceHashCount / index.sources.length, lint_error_count: validation.findings.filter((finding) => finding.severity === 'error').length, lint_warning_count: lintFindings.filter((finding) => finding.severity === 'warning').length, + near_duplicate_page_rate: nearDuplicates.duplicatePageRate, + near_duplicate_page_count: nearDuplicates.duplicatePageCount, + near_duplicate_pair_count: nearDuplicates.duplicatePairCount, } const findings: RagGapFinding[] = [] if (metrics.lint_error_count > 0) { @@ -64,7 +73,34 @@ export function scoreKnowledgeBaseIndex( evidence: { stale_source_rate: metrics.stale_source_rate }, }) } - return { ok: findings.length === 0, metrics, findings } + if (nearDuplicates.truncated && maxNearDuplicatePageRate < 1) { + findings.push({ + id: 'kb:near-duplicate-analysis-truncated', + kind: 'unknown', + severity: 'error', + message: + 'Near-duplicate analysis reached a configured bound, so the duplicate-page gate cannot be evaluated completely.', + evidence: { + candidate_pair_count: nearDuplicates.candidatePairCount, + compared_pair_count: nearDuplicates.comparedPairCount, + duplicate_pair_count: nearDuplicates.duplicatePairCount, + }, + }) + } else if (metrics.near_duplicate_page_rate > maxNearDuplicatePageRate) { + findings.push({ + id: 'kb:near-duplicate-page-rate', + kind: 'unknown', + severity: 'error', + message: 'Near-duplicate page rate exceeds the configured maximum.', + evidence: { + near_duplicate_page_rate: metrics.near_duplicate_page_rate, + near_duplicate_page_count: metrics.near_duplicate_page_count, + near_duplicate_pair_count: metrics.near_duplicate_pair_count, + threshold: nearDuplicates.threshold, + }, + }) + } + return { ok: findings.length === 0, metrics, nearDuplicates, findings } } function sourceRefs(text: string): string[] { @@ -77,3 +113,10 @@ function sourceRefs(text: string): string[] { } return refs } + +function unitInterval(value: number, name: string): number { + if (!Number.isFinite(value) || value < 0 || value > 1) { + throw new Error(`${name} must be a finite number in [0, 1]`) + } + return value +} From 5f6cbba0d843775fcaa7cd535d58d5300fece888 Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 19:13:49 -0700 Subject: [PATCH 5/8] feat(quality): export deterministic near-duplicate analysis --- src/rag-eval.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/rag-eval.ts b/src/rag-eval.ts index 1d49a38..da6ccd3 100644 --- a/src/rag-eval.ts +++ b/src/rag-eval.ts @@ -20,6 +20,12 @@ export type { RagRequiredContext, } from './rag-eval/contracts' export { scoreKnowledgeBaseIndex } from './rag-eval/knowledge-base' +export type { + NearDuplicateDetectionOptions, + NearDuplicatePair, + NearDuplicateReport, +} from './rag-eval/near-duplicates' +export { detectNearDuplicatePages, normalizePageText } from './rag-eval/near-duplicates' export { normalizeExternalRagScores, toDeepEvalTestCases, From e5655f9f02494777520cbe8ff99f5edd7c334db4 Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 19:15:36 -0700 Subject: [PATCH 6/8] test(quality): verify deterministic duplicate metrics and gates --- src/rag-eval/near-duplicates.test.ts | 206 +++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 src/rag-eval/near-duplicates.test.ts diff --git a/src/rag-eval/near-duplicates.test.ts b/src/rag-eval/near-duplicates.test.ts new file mode 100644 index 0000000..15ee4ff --- /dev/null +++ b/src/rag-eval/near-duplicates.test.ts @@ -0,0 +1,206 @@ +import { describe, expect, it } from 'vitest' +import type { KnowledgeIndex, KnowledgePage } from '../types' +import { scoreKnowledgeBaseIndex } from './knowledge-base' +import { detectNearDuplicatePages, normalizePageText } from './near-duplicates' + +const repeated = + 'A verified research page explains the mechanism, records the experiment, names the evidence, and preserves the result for later agents. ' + +function page(id: string, title: string, text: string): KnowledgePage { + return { + id, + path: `knowledge/${id}.md`, + title, + text, + frontmatter: { id, title }, + sourceIds: [], + tags: [], + outLinks: [], + } +} + +function index(pages: KnowledgePage[]): KnowledgeIndex { + return { + root: '/kb', + generatedAt: '2026-08-17T00:00:00.000Z', + sources: [], + pages, + graph: { nodes: [], edges: [] }, + } +} + +describe('detectNearDuplicatePages', () => { + it('reports exact normalized copies with stable page identity', () => { + const pages = [ + page('copy-a', 'Same title', repeated.repeat(2)), + page('copy-b', 'Same title', repeated.repeat(2)), + ] + + const report = detectNearDuplicatePages(pages) + + expect(report).toMatchObject({ + pageCount: 2, + eligiblePageCount: 2, + candidatePairCount: 1, + comparedPairCount: 1, + duplicatePairCount: 1, + duplicatePageCount: 2, + duplicatePageRate: 1, + truncated: false, + }) + expect(report.pairs).toEqual([ + expect.objectContaining({ + leftPageId: 'copy-a', + rightPageId: 'copy-b', + similarity: 1, + exact: true, + }), + ]) + }) + + it('finds near copies but not unrelated prose', () => { + const base = `${repeated.repeat(2)} The decisive measured value is forty two.` + const revised = `${repeated.repeat(2)} The decisive measured value is forty three.` + const unrelated = + 'A kitchen inventory lists pans, knives, towels, plates, and groceries for a weekend dinner. '.repeat( + 3, + ) + + const report = detectNearDuplicatePages( + [ + page('base', 'Experiment', base), + page('revision', 'Experiment', revised), + page('unrelated', 'Kitchen', unrelated), + ], + { threshold: 0.7 }, + ) + + expect(report.duplicatePairCount).toBe(1) + expect(report.pairs[0]).toMatchObject({ + leftPageId: 'base', + rightPageId: 'revision', + exact: false, + }) + expect(report.pairs[0]!.similarity).toBeGreaterThanOrEqual(0.7) + }) + + it('is deterministic under input reordering', () => { + const pages = [ + page('z', 'Same title', repeated.repeat(2)), + page('a', 'Same title', repeated.repeat(2)), + page('m', 'Different title', `${repeated.repeat(2)} extra material`), + ] + + const forward = detectNearDuplicatePages(pages, { threshold: 0.7 }) + const reverse = detectNearDuplicatePages([...pages].reverse(), { threshold: 0.7 }) + + expect(reverse).toEqual(forward) + }) + + it('excludes short pages and reports bounded candidate truncation', () => { + const short = detectNearDuplicatePages( + [page('short-a', 'Short', 'same'), page('short-b', 'Short', 'same')], + { minCharacters: 80 }, + ) + expect(short).toMatchObject({ eligiblePageCount: 0, duplicatePairCount: 0 }) + + const bounded = detectNearDuplicatePages( + Array.from({ length: 4 }, (_, index) => + page(`copy-${index}`, 'Same title', repeated.repeat(2)), + ), + { maxCandidatePairs: 2 }, + ) + expect(bounded).toMatchObject({ + candidatePairCount: 2, + comparedPairCount: 2, + truncated: true, + }) + }) + + it('normalizes Unicode width, case, whitespace, and URL identity', () => { + expect(normalizePageText('ＡLPHA https://example.com/a\nBeta')).toBe( + 'alpha beta', + ) + }) + + it('refuses malformed detector limits', () => { + expect(() => detectNearDuplicatePages([], { threshold: 1.1 })).toThrow(/threshold/) + expect(() => detectNearDuplicatePages([], { maxCandidatePairs: 0 })).toThrow( + /maxCandidatePairs/, + ) + expect(() => detectNearDuplicatePages([], { minCharacters: -1 })).toThrow(/minCharacters/) + }) +}) + +describe('scoreKnowledgeBaseIndex near-duplicate quality', () => { + const duplicatePages = [ + page('copy-a', 'Same title', repeated.repeat(2)), + page('copy-b', 'Same title', repeated.repeat(2)), + page( + 'independent', + 'Independent result', + 'A separate proof studies a different theorem, with distinct assumptions and a different verification artifact. '.repeat( + 3, + ), + ), + ] + + it('always reports deterministic duplicate metrics without failing by default', () => { + const report = scoreKnowledgeBaseIndex(index(duplicatePages)) + + expect(report.ok).toBe(true) + expect(report.metrics).toMatchObject({ + near_duplicate_page_rate: 0.666667, + near_duplicate_page_count: 2, + near_duplicate_pair_count: 1, + }) + expect(report.nearDuplicates?.pairs).toHaveLength(1) + }) + + it('fails a configured duplicate-page gate with measured evidence', () => { + const report = scoreKnowledgeBaseIndex(index(duplicatePages), { + maxNearDuplicatePageRate: 0.5, + }) + + expect(report.ok).toBe(false) + expect(report.findings).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: 'kb:near-duplicate-page-rate', + severity: 'error', + evidence: expect.objectContaining({ + near_duplicate_page_rate: 0.666667, + near_duplicate_pair_count: 1, + }), + }), + ]), + ) + }) + + it('fails closed when a configured gate rests on truncated analysis', () => { + const report = scoreKnowledgeBaseIndex( + index( + Array.from({ length: 4 }, (_, position) => + page(`copy-${position}`, 'Same title', repeated.repeat(2)), + ), + ), + { + nearDuplicates: { maxCandidatePairs: 1 }, + maxNearDuplicatePageRate: 0.9, + }, + ) + + expect(report.ok).toBe(false) + expect(report.findings).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: 'kb:near-duplicate-analysis-truncated' }), + ]), + ) + }) + + it('refuses a malformed quality threshold', () => { + expect(() => + scoreKnowledgeBaseIndex(index(duplicatePages), { maxNearDuplicatePageRate: -0.1 }), + ).toThrow(/maxNearDuplicatePageRate/) + }) +}) From 26f80fa28e994865efa37d205994286d19242c33 Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 19:21:32 -0700 Subject: [PATCH 7/8] style(quality): apply canonical near-duplicate formatting --- src/rag-eval/near-duplicates.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/rag-eval/near-duplicates.ts b/src/rag-eval/near-duplicates.ts index 86b70d0..f2510ca 100644 --- a/src/rag-eval/near-duplicates.ts +++ b/src/rag-eval/near-duplicates.ts @@ -78,14 +78,13 @@ export function detectNearDuplicatePages( options.maxCandidatePairs ?? 250_000, 'maxCandidatePairs', ) - const maxReportedPairs = nonNegativeInteger( - options.maxReportedPairs ?? 1_000, - 'maxReportedPairs', - ) + const maxReportedPairs = nonNegativeInteger(options.maxReportedPairs ?? 1_000, 'maxReportedPairs') const prepared = pages .map((page) => preparePage(page, wordShingleSize, characterShingleSize)) - .filter((item): item is PreparedPage => item !== null && item.normalized.length >= minCharacters) + .filter( + (item): item is PreparedPage => item !== null && item.normalized.length >= minCharacters, + ) .sort((left, right) => left.page.path.localeCompare(right.page.path)) const candidateKeys = new Set() @@ -232,7 +231,10 @@ function addGroupPairs( return true } -function jaccard(left: ReadonlySet, right: ReadonlySet): { +function jaccard( + left: ReadonlySet, + right: ReadonlySet, +): { similarity: number intersectionSize: number unionSize: number From c7007d779d1bbed76de324d626347eb1e61e655c Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 19:22:46 -0700 Subject: [PATCH 8/8] style(quality): format detector tests --- src/rag-eval/near-duplicates.test.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/rag-eval/near-duplicates.test.ts b/src/rag-eval/near-duplicates.test.ts index 15ee4ff..24cb2e9 100644 --- a/src/rag-eval/near-duplicates.test.ts +++ b/src/rag-eval/near-duplicates.test.ts @@ -118,9 +118,7 @@ describe('detectNearDuplicatePages', () => { }) it('normalizes Unicode width, case, whitespace, and URL identity', () => { - expect(normalizePageText('ＡLPHA https://example.com/a\nBeta')).toBe( - 'alpha beta', - ) + expect(normalizePageText('ＡLPHA https://example.com/a\nBeta')).toBe('alpha beta') }) it('refuses malformed detector limits', () => {