diff --git a/src/rag-eval.ts b/src/rag-eval.ts index 1d49a38..da6ccd3 100644 --- a/src/rag-eval.ts +++ b/src/rag-eval.ts @@ -20,6 +20,12 @@ export type { RagRequiredContext, } from './rag-eval/contracts' export { scoreKnowledgeBaseIndex } from './rag-eval/knowledge-base' +export type { + NearDuplicateDetectionOptions, + NearDuplicatePair, + NearDuplicateReport, +} from './rag-eval/near-duplicates' +export { detectNearDuplicatePages, normalizePageText } from './rag-eval/near-duplicates' export { normalizeExternalRagScores, toDeepEvalTestCases, diff --git a/src/rag-eval/contracts.ts b/src/rag-eval/contracts.ts index 2e6922b..bd27e3e 100644 --- a/src/rag-eval/contracts.ts +++ b/src/rag-eval/contracts.ts @@ -1,6 +1,7 @@ import type { ComparisonCost, JudgeConfig, Scenario } from '@tangle-network/agent-eval/campaign' import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' import type { RagGapFinding } from '../rag-improvement-loop' +import type { NearDuplicateDetectionOptions, NearDuplicateReport } from './near-duplicates' export type RagEvalProvider = | 'agent-knowledge' @@ -166,6 +167,10 @@ export interface KnowledgeBaseQualityOptions { strict?: boolean minCitationRate?: number maxStaleSourceRate?: number + /** Deterministic exact/near-duplicate detector configuration. */ + nearDuplicates?: NearDuplicateDetectionOptions + /** Maximum allowed fraction of eligible pages implicated in a duplicate pair. Defaults to 1. */ + maxNearDuplicatePageRate?: number } export interface KnowledgeBaseQualityReport { @@ -179,7 +184,13 @@ export interface KnowledgeBaseQualityReport { duplicate_source_hash_rate: number lint_error_count: number lint_warning_count: number + /** Added in the deterministic near-duplicate quality pass. */ + near_duplicate_page_rate?: number + near_duplicate_page_count?: number + near_duplicate_pair_count?: number } + /** Exact report from the deterministic detector; present when produced by this package. */ + nearDuplicates?: NearDuplicateReport findings: readonly RagGapFinding[] } diff --git a/src/rag-eval/knowledge-base.ts b/src/rag-eval/knowledge-base.ts index 503a0bb..24fc157 100644 --- a/src/rag-eval/knowledge-base.ts +++ b/src/rag-eval/knowledge-base.ts @@ -3,6 +3,7 @@ import type { RagGapFinding } from '../rag-improvement-loop' import type { KnowledgeIndex } from '../types' import { validateKnowledgeIndex } from '../validate' import type { KnowledgeBaseQualityOptions, KnowledgeBaseQualityReport } from './contracts' +import { detectNearDuplicatePages } from './near-duplicates' export function scoreKnowledgeBaseIndex( index: KnowledgeIndex, @@ -25,6 +26,11 @@ export function scoreKnowledgeBaseIndex( (page) => sourceRefs(page.text).length > 0, ).length const pagesWithSources = index.pages.filter((page) => page.sourceIds.length > 0).length + const nearDuplicates = detectNearDuplicatePages(index.pages, options.nearDuplicates) + const maxNearDuplicatePageRate = unitInterval( + options.maxNearDuplicatePageRate ?? 1, + 'maxNearDuplicatePageRate', + ) const metrics = { page_count: index.pages.length, source_count: index.sources.length, @@ -35,6 +41,9 @@ export function scoreKnowledgeBaseIndex( index.sources.length === 0 ? 0 : duplicateSourceHashCount / index.sources.length, lint_error_count: validation.findings.filter((finding) => finding.severity === 'error').length, lint_warning_count: lintFindings.filter((finding) => finding.severity === 'warning').length, + near_duplicate_page_rate: nearDuplicates.duplicatePageRate, + near_duplicate_page_count: nearDuplicates.duplicatePageCount, + near_duplicate_pair_count: nearDuplicates.duplicatePairCount, } const findings: RagGapFinding[] = [] if (metrics.lint_error_count > 0) { @@ -64,7 +73,34 @@ export function scoreKnowledgeBaseIndex( evidence: { stale_source_rate: metrics.stale_source_rate }, }) } - return { ok: findings.length === 0, metrics, findings } + if (nearDuplicates.truncated && maxNearDuplicatePageRate < 1) { + findings.push({ + id: 'kb:near-duplicate-analysis-truncated', + kind: 'unknown', + severity: 'error', + message: + 'Near-duplicate analysis reached a configured bound, so the duplicate-page gate cannot be evaluated completely.', + evidence: { + candidate_pair_count: nearDuplicates.candidatePairCount, + compared_pair_count: nearDuplicates.comparedPairCount, + duplicate_pair_count: nearDuplicates.duplicatePairCount, + }, + }) + } else if (metrics.near_duplicate_page_rate > maxNearDuplicatePageRate) { + findings.push({ + id: 'kb:near-duplicate-page-rate', + kind: 'unknown', + severity: 'error', + message: 'Near-duplicate page rate exceeds the configured maximum.', + evidence: { + near_duplicate_page_rate: metrics.near_duplicate_page_rate, + near_duplicate_page_count: metrics.near_duplicate_page_count, + near_duplicate_pair_count: metrics.near_duplicate_pair_count, + threshold: nearDuplicates.threshold, + }, + }) + } + return { ok: findings.length === 0, metrics, nearDuplicates, findings } } function sourceRefs(text: string): string[] { @@ -77,3 +113,10 @@ function sourceRefs(text: string): string[] { } return refs } + +function unitInterval(value: number, name: string): number { + if (!Number.isFinite(value) || value < 0 || value > 1) { + throw new Error(`${name} must be a finite number in [0, 1]`) + } + return value +} diff --git a/src/rag-eval/near-duplicates.test.ts b/src/rag-eval/near-duplicates.test.ts new file mode 100644 index 0000000..24cb2e9 --- /dev/null +++ b/src/rag-eval/near-duplicates.test.ts @@ -0,0 +1,204 @@ +import { describe, expect, it } from 'vitest' +import type { KnowledgeIndex, KnowledgePage } from '../types' +import { scoreKnowledgeBaseIndex } from './knowledge-base' +import { detectNearDuplicatePages, normalizePageText } from './near-duplicates' + +const repeated = + 'A verified research page explains the mechanism, records the experiment, names the evidence, and preserves the result for later agents. ' + +function page(id: string, title: string, text: string): KnowledgePage { + return { + id, + path: `knowledge/${id}.md`, + title, + text, + frontmatter: { id, title }, + sourceIds: [], + tags: [], + outLinks: [], + } +} + +function index(pages: KnowledgePage[]): KnowledgeIndex { + return { + root: '/kb', + generatedAt: '2026-08-17T00:00:00.000Z', + sources: [], + pages, + graph: { nodes: [], edges: [] }, + } +} + +describe('detectNearDuplicatePages', () => { + it('reports exact normalized copies with stable page identity', () => { + const pages = [ + page('copy-a', 'Same title', repeated.repeat(2)), + page('copy-b', 'Same title', repeated.repeat(2)), + ] + + const report = detectNearDuplicatePages(pages) + + expect(report).toMatchObject({ + pageCount: 2, + eligiblePageCount: 2, + candidatePairCount: 1, + comparedPairCount: 1, + duplicatePairCount: 1, + duplicatePageCount: 2, + duplicatePageRate: 1, + truncated: false, + }) + expect(report.pairs).toEqual([ + expect.objectContaining({ + leftPageId: 'copy-a', + rightPageId: 'copy-b', + similarity: 1, + exact: true, + }), + ]) + }) + + it('finds near copies but not unrelated prose', () => { + const base = `${repeated.repeat(2)} The decisive measured value is forty two.` + const revised = `${repeated.repeat(2)} The decisive measured value is forty three.` + const unrelated = + 'A kitchen inventory lists pans, knives, towels, plates, and groceries for a weekend dinner. '.repeat( + 3, + ) + + const report = detectNearDuplicatePages( + [ + page('base', 'Experiment', base), + page('revision', 'Experiment', revised), + page('unrelated', 'Kitchen', unrelated), + ], + { threshold: 0.7 }, + ) + + expect(report.duplicatePairCount).toBe(1) + expect(report.pairs[0]).toMatchObject({ + leftPageId: 'base', + rightPageId: 'revision', + exact: false, + }) + expect(report.pairs[0]!.similarity).toBeGreaterThanOrEqual(0.7) + }) + + it('is deterministic under input reordering', () => { + const pages = [ + page('z', 'Same title', repeated.repeat(2)), + page('a', 'Same title', repeated.repeat(2)), + page('m', 'Different title', `${repeated.repeat(2)} extra material`), + ] + + const forward = detectNearDuplicatePages(pages, { threshold: 0.7 }) + const reverse = detectNearDuplicatePages([...pages].reverse(), { threshold: 0.7 }) + + expect(reverse).toEqual(forward) + }) + + it('excludes short pages and reports bounded candidate truncation', () => { + const short = detectNearDuplicatePages( + [page('short-a', 'Short', 'same'), page('short-b', 'Short', 'same')], + { minCharacters: 80 }, + ) + expect(short).toMatchObject({ eligiblePageCount: 0, duplicatePairCount: 0 }) + + const bounded = detectNearDuplicatePages( + Array.from({ length: 4 }, (_, index) => + page(`copy-${index}`, 'Same title', repeated.repeat(2)), + ), + { maxCandidatePairs: 2 }, + ) + expect(bounded).toMatchObject({ + candidatePairCount: 2, + comparedPairCount: 2, + truncated: true, + }) + }) + + it('normalizes Unicode width, case, whitespace, and URL identity', () => { + expect(normalizePageText('ALPHA https://example.com/a\nBeta')).toBe('alpha beta') + }) + + it('refuses malformed detector limits', () => { + expect(() => detectNearDuplicatePages([], { threshold: 1.1 })).toThrow(/threshold/) + expect(() => detectNearDuplicatePages([], { maxCandidatePairs: 0 })).toThrow( + /maxCandidatePairs/, + ) + expect(() => detectNearDuplicatePages([], { minCharacters: -1 })).toThrow(/minCharacters/) + }) +}) + +describe('scoreKnowledgeBaseIndex near-duplicate quality', () => { + const duplicatePages = [ + page('copy-a', 'Same title', repeated.repeat(2)), + page('copy-b', 'Same title', repeated.repeat(2)), + page( + 'independent', + 'Independent result', + 'A separate proof studies a different theorem, with distinct assumptions and a different verification artifact. '.repeat( + 3, + ), + ), + ] + + it('always reports deterministic duplicate metrics without failing by default', () => { + const report = scoreKnowledgeBaseIndex(index(duplicatePages)) + + expect(report.ok).toBe(true) + expect(report.metrics).toMatchObject({ + near_duplicate_page_rate: 0.666667, + near_duplicate_page_count: 2, + near_duplicate_pair_count: 1, + }) + expect(report.nearDuplicates?.pairs).toHaveLength(1) + }) + + it('fails a configured duplicate-page gate with measured evidence', () => { + const report = scoreKnowledgeBaseIndex(index(duplicatePages), { + maxNearDuplicatePageRate: 0.5, + }) + + expect(report.ok).toBe(false) + expect(report.findings).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: 'kb:near-duplicate-page-rate', + severity: 'error', + evidence: expect.objectContaining({ + near_duplicate_page_rate: 0.666667, + near_duplicate_pair_count: 1, + }), + }), + ]), + ) + }) + + it('fails closed when a configured gate rests on truncated analysis', () => { + const report = scoreKnowledgeBaseIndex( + index( + Array.from({ length: 4 }, (_, position) => + page(`copy-${position}`, 'Same title', repeated.repeat(2)), + ), + ), + { + nearDuplicates: { maxCandidatePairs: 1 }, + maxNearDuplicatePageRate: 0.9, + }, + ) + + expect(report.ok).toBe(false) + expect(report.findings).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: 'kb:near-duplicate-analysis-truncated' }), + ]), + ) + }) + + it('refuses a malformed quality threshold', () => { + expect(() => + scoreKnowledgeBaseIndex(index(duplicatePages), { maxNearDuplicatePageRate: -0.1 }), + ).toThrow(/maxNearDuplicatePageRate/) + }) +}) diff --git a/src/rag-eval/near-duplicates.ts b/src/rag-eval/near-duplicates.ts new file mode 100644 index 0000000..f2510ca --- /dev/null +++ b/src/rag-eval/near-duplicates.ts @@ -0,0 +1,289 @@ +import type { KnowledgePage } from '../types' + +export interface NearDuplicateDetectionOptions { + /** Jaccard similarity at or above which a pair is reported. Defaults to 0.82. */ + threshold?: number + /** Minimum normalized characters before a page participates. Defaults to 80. */ + minCharacters?: number + /** Word-shingle width. Defaults to 5. */ + wordShingleSize?: number + /** Character-shingle width used for text with too few words. Defaults to 16. */ + characterShingleSize?: number + /** Ignore a shingle occurring in more pages than this. Defaults to 64. */ + maxDocumentFrequency?: number + /** Stop adding candidate pairs after this many. Defaults to 250,000. */ + maxCandidatePairs?: number + /** Maximum duplicate pairs retained in the report. Defaults to 1,000. */ + maxReportedPairs?: number +} + +export interface NearDuplicatePair { + leftPageId: string + leftPath: string + rightPageId: string + rightPath: string + similarity: number + intersectionSize: number + unionSize: number + exact: boolean +} + +export interface NearDuplicateReport { + algorithm: 'deterministic-shingle-jaccard-v1' + threshold: number + pageCount: number + eligiblePageCount: number + candidatePairCount: number + comparedPairCount: number + duplicatePairCount: number + duplicatePageCount: number + duplicatePageRate: number + truncated: boolean + pairs: NearDuplicatePair[] +} + +interface PreparedPage { + page: KnowledgePage + normalized: string + shingles: Set +} + +/** + * Detect exact and near-duplicate pages without a model, embedding service, or clock. + * + * The detector uses word shingles when prose has enough tokens and Unicode + * character shingles otherwise. An inverted index proposes candidate pairs; + * high-document-frequency shingles are ignored so common boilerplate cannot + * make candidate generation quadratic. Exact normalized copies are always + * proposed before ordinary shingle candidates, even when every shingle is + * common. Every limit and sort order is explicit so the same corpus produces + * the same report on every run. + */ +export function detectNearDuplicatePages( + pages: readonly KnowledgePage[], + options: NearDuplicateDetectionOptions = {}, +): NearDuplicateReport { + const threshold = finiteUnitInterval(options.threshold ?? 0.82, 'threshold') + const minCharacters = nonNegativeInteger(options.minCharacters ?? 80, 'minCharacters') + const wordShingleSize = positiveInteger(options.wordShingleSize ?? 5, 'wordShingleSize') + const characterShingleSize = positiveInteger( + options.characterShingleSize ?? 16, + 'characterShingleSize', + ) + const maxDocumentFrequency = positiveInteger( + options.maxDocumentFrequency ?? 64, + 'maxDocumentFrequency', + ) + const maxCandidatePairs = positiveInteger( + options.maxCandidatePairs ?? 250_000, + 'maxCandidatePairs', + ) + const maxReportedPairs = nonNegativeInteger(options.maxReportedPairs ?? 1_000, 'maxReportedPairs') + + const prepared = pages + .map((page) => preparePage(page, wordShingleSize, characterShingleSize)) + .filter( + (item): item is PreparedPage => item !== null && item.normalized.length >= minCharacters, + ) + .sort((left, right) => left.page.path.localeCompare(right.page.path)) + + const candidateKeys = new Set() + const exactGroups = new Map() + const byShingle = new Map() + for (const [index, item] of prepared.entries()) { + exactGroups.set(item.normalized, [...(exactGroups.get(item.normalized) ?? []), index]) + for (const shingle of item.shingles) { + byShingle.set(shingle, [...(byShingle.get(shingle) ?? []), index]) + } + } + + let truncated = false + const addPair = (left: number, right: number): boolean => { + if (left === right) return true + const key = pairKey(Math.min(left, right), Math.max(left, right)) + if (candidateKeys.has(key)) return true + if (candidateKeys.size >= maxCandidatePairs) { + truncated = true + return false + } + candidateKeys.add(key) + return true + } + + for (const normalized of [...exactGroups.keys()].sort()) { + if (!addGroupPairs(exactGroups.get(normalized)!, addPair)) break + } + if (!truncated) { + for (const shingle of [...byShingle.keys()].sort()) { + const group = byShingle.get(shingle)! + if (group.length > maxDocumentFrequency) continue + if (!addGroupPairs(group, addPair)) break + } + } + + const duplicatePageIndices = new Set() + const pairs: NearDuplicatePair[] = [] + let duplicatePairCount = 0 + let comparedPairCount = 0 + for (const key of [...candidateKeys].sort(pairKeyOrder)) { + const [leftIndex, rightIndex] = parsePairKey(key) + const left = prepared[leftIndex]! + const right = prepared[rightIndex]! + comparedPairCount += 1 + const exact = left.normalized === right.normalized + const overlap = jaccard(left.shingles, right.shingles) + if (!exact && overlap.similarity < threshold) continue + duplicatePairCount += 1 + duplicatePageIndices.add(leftIndex) + duplicatePageIndices.add(rightIndex) + if (pairs.length >= maxReportedPairs) { + truncated = true + continue + } + pairs.push({ + leftPageId: left.page.id, + leftPath: left.page.path, + rightPageId: right.page.id, + rightPath: right.page.path, + similarity: exact ? 1 : round(overlap.similarity), + intersectionSize: overlap.intersectionSize, + unionSize: overlap.unionSize, + exact, + }) + } + + pairs.sort( + (left, right) => + right.similarity - left.similarity || + left.leftPath.localeCompare(right.leftPath) || + left.rightPath.localeCompare(right.rightPath), + ) + + return { + algorithm: 'deterministic-shingle-jaccard-v1', + threshold, + pageCount: pages.length, + eligiblePageCount: prepared.length, + candidatePairCount: candidateKeys.size, + comparedPairCount, + duplicatePairCount, + duplicatePageCount: duplicatePageIndices.size, + duplicatePageRate: + prepared.length === 0 ? 0 : round(duplicatePageIndices.size / prepared.length), + truncated, + pairs, + } +} + +function preparePage( + page: KnowledgePage, + wordShingleSize: number, + characterShingleSize: number, +): PreparedPage | null { + const normalized = normalizePageText(`${page.title}\n${page.text}`) + if (normalized.length === 0) return null + const words = normalized.match(/[\p{L}\p{N}]+/gu) ?? [] + const shingles = + words.length >= wordShingleSize + 2 + ? sequenceShingles(words, wordShingleSize) + : characterShingles(normalized.replace(/\s+/g, ''), characterShingleSize) + if (shingles.size === 0) shingles.add(normalized) + return { page, normalized, shingles } +} + +export function normalizePageText(value: string): string { + return value + .normalize('NFKC') + .toLowerCase() + .replace(/https?:\/\/\S+/g, '') + .replace(/\s+/g, ' ') + .trim() +} + +function sequenceShingles(items: readonly string[], width: number): Set { + const out = new Set() + if (items.length < width) return out + for (let index = 0; index <= items.length - width; index += 1) { + out.add(items.slice(index, index + width).join('\u001f')) + } + return out +} + +function characterShingles(value: string, width: number): Set { + const chars = [...value] + const out = new Set() + if (chars.length < width) return out + for (let index = 0; index <= chars.length - width; index += 1) { + out.add(chars.slice(index, index + width).join('')) + } + return out +} + +function addGroupPairs( + group: readonly number[], + add: (left: number, right: number) => boolean, +): boolean { + for (let left = 0; left < group.length; left += 1) { + for (let right = left + 1; right < group.length; right += 1) { + if (!add(group[left]!, group[right]!)) return false + } + } + return true +} + +function jaccard( + left: ReadonlySet, + right: ReadonlySet, +): { + similarity: number + intersectionSize: number + unionSize: number +} { + const [small, large] = left.size <= right.size ? [left, right] : [right, left] + let intersectionSize = 0 + for (const item of small) if (large.has(item)) intersectionSize += 1 + const unionSize = left.size + right.size - intersectionSize + return { + similarity: unionSize === 0 ? 1 : intersectionSize / unionSize, + intersectionSize, + unionSize, + } +} + +function pairKey(left: number, right: number): string { + return `${left}:${right}` +} + +function parsePairKey(value: string): [number, number] { + const [left, right] = value.split(':') + return [Number(left), Number(right)] +} + +function pairKeyOrder(left: string, right: string): number { + const [leftA, leftB] = parsePairKey(left) + const [rightA, rightB] = parsePairKey(right) + return leftA - rightA || leftB - rightB +} + +function finiteUnitInterval(value: number, name: string): number { + if (!Number.isFinite(value) || value < 0 || value > 1) { + throw new Error(`${name} must be a finite number in [0, 1]`) + } + return value +} + +function positiveInteger(value: number, name: string): number { + if (!Number.isInteger(value) || value < 1) throw new Error(`${name} must be a positive integer`) + return value +} + +function nonNegativeInteger(value: number, name: string): number { + if (!Number.isInteger(value) || value < 0) { + throw new Error(`${name} must be a non-negative integer`) + } + return value +} + +function round(value: number): number { + return Math.round(value * 1_000_000) / 1_000_000 +}