feat(quality): measure and gate near-duplicate knowledge pages

src/rag-eval.ts

@@ -20,6 +20,12 @@ export type {
   RagRequiredContext,
 } from './rag-eval/contracts'
 export { scoreKnowledgeBaseIndex } from './rag-eval/knowledge-base'
+export type {
+  NearDuplicateDetectionOptions,
+  NearDuplicatePair,
+  NearDuplicateReport,
+} from './rag-eval/near-duplicates'
+export { detectNearDuplicatePages, normalizePageText } from './rag-eval/near-duplicates'
 export {
   normalizeExternalRagScores,
   toDeepEvalTestCases,

src/rag-eval/contracts.ts

@@ -1,6 +1,7 @@
 import type { ComparisonCost, JudgeConfig, Scenario } from '@tangle-network/agent-eval/campaign'
 import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface'
 import type { RagGapFinding } from '../rag-improvement-loop'
+import type { NearDuplicateDetectionOptions, NearDuplicateReport } from './near-duplicates'
 
 export type RagEvalProvider =
   | 'agent-knowledge'
@@ -166,6 +167,10 @@ export interface KnowledgeBaseQualityOptions {
   strict?: boolean
   minCitationRate?: number
   maxStaleSourceRate?: number
+  /** Deterministic exact/near-duplicate detector configuration. */
+  nearDuplicates?: NearDuplicateDetectionOptions
+  /** Maximum allowed fraction of eligible pages implicated in a duplicate pair. Defaults to 1. */
+  maxNearDuplicatePageRate?: number
 }
 
 export interface KnowledgeBaseQualityReport {
@@ -179,7 +184,13 @@ export interface KnowledgeBaseQualityReport {
     duplicate_source_hash_rate: number
     lint_error_count: number
     lint_warning_count: number
+    /** Added in the deterministic near-duplicate quality pass. */
+    near_duplicate_page_rate?: number
+    near_duplicate_page_count?: number
+    near_duplicate_pair_count?: number
   }
+  /** Exact report from the deterministic detector; present when produced by this package. */
+  nearDuplicates?: NearDuplicateReport
   findings: readonly RagGapFinding[]
 }
 

src/rag-eval/knowledge-base.ts

@@ -3,6 +3,7 @@ import type { RagGapFinding } from '../rag-improvement-loop'
 import type { KnowledgeIndex } from '../types'
 import { validateKnowledgeIndex } from '../validate'
 import type { KnowledgeBaseQualityOptions, KnowledgeBaseQualityReport } from './contracts'
+import { detectNearDuplicatePages } from './near-duplicates'
 
 export function scoreKnowledgeBaseIndex(
   index: KnowledgeIndex,
@@ -25,6 +26,11 @@ export function scoreKnowledgeBaseIndex(
     (page) => sourceRefs(page.text).length > 0,
   ).length
   const pagesWithSources = index.pages.filter((page) => page.sourceIds.length > 0).length
+  const nearDuplicates = detectNearDuplicatePages(index.pages, options.nearDuplicates)
+  const maxNearDuplicatePageRate = unitInterval(
+    options.maxNearDuplicatePageRate ?? 1,
+    'maxNearDuplicatePageRate',
+  )
   const metrics = {
     page_count: index.pages.length,
     source_count: index.sources.length,
@@ -35,6 +41,9 @@ export function scoreKnowledgeBaseIndex(
       index.sources.length === 0 ? 0 : duplicateSourceHashCount / index.sources.length,
     lint_error_count: validation.findings.filter((finding) => finding.severity === 'error').length,
     lint_warning_count: lintFindings.filter((finding) => finding.severity === 'warning').length,
+    near_duplicate_page_rate: nearDuplicates.duplicatePageRate,
+    near_duplicate_page_count: nearDuplicates.duplicatePageCount,
+    near_duplicate_pair_count: nearDuplicates.duplicatePairCount,
   }
   const findings: RagGapFinding[] = []
   if (metrics.lint_error_count > 0) {
@@ -64,7 +73,34 @@ export function scoreKnowledgeBaseIndex(
       evidence: { stale_source_rate: metrics.stale_source_rate },
     })
   }
-  return { ok: findings.length === 0, metrics, findings }
+  if (nearDuplicates.truncated && maxNearDuplicatePageRate < 1) {
+    findings.push({
+      id: 'kb:near-duplicate-analysis-truncated',
+      kind: 'unknown',
+      severity: 'error',
+      message:
+        'Near-duplicate analysis reached a configured bound, so the duplicate-page gate cannot be evaluated completely.',
+      evidence: {
+        candidate_pair_count: nearDuplicates.candidatePairCount,
+        compared_pair_count: nearDuplicates.comparedPairCount,
+        duplicate_pair_count: nearDuplicates.duplicatePairCount,
+      },
+    })
+  } else if (metrics.near_duplicate_page_rate > maxNearDuplicatePageRate) {
+    findings.push({
+      id: 'kb:near-duplicate-page-rate',
+      kind: 'unknown',
+      severity: 'error',
+      message: 'Near-duplicate page rate exceeds the configured maximum.',
+      evidence: {
+        near_duplicate_page_rate: metrics.near_duplicate_page_rate,
+        near_duplicate_page_count: metrics.near_duplicate_page_count,
+        near_duplicate_pair_count: metrics.near_duplicate_pair_count,
+        threshold: nearDuplicates.threshold,
+      },
+    })
+  }
+  return { ok: findings.length === 0, metrics, nearDuplicates, findings }
 }
 
 function sourceRefs(text: string): string[] {
@@ -77,3 +113,10 @@ function sourceRefs(text: string): string[] {
   }
   return refs
 }
+
+function unitInterval(value: number, name: string): number {
+  if (!Number.isFinite(value) || value < 0 || value > 1) {
+    throw new Error(`${name} must be a finite number in [0, 1]`)
+  }
+  return value
+}

src/rag-eval/near-duplicates.test.ts

@@ -0,0 +1,204 @@
+import { describe, expect, it } from 'vitest'
+import type { KnowledgeIndex, KnowledgePage } from '../types'
+import { scoreKnowledgeBaseIndex } from './knowledge-base'
+import { detectNearDuplicatePages, normalizePageText } from './near-duplicates'
+
+const repeated =
+  'A verified research page explains the mechanism, records the experiment, names the evidence, and preserves the result for later agents. '
+
+function page(id: string, title: string, text: string): KnowledgePage {
+  return {
+    id,
+    path: `knowledge/${id}.md`,
+    title,
+    text,
+    frontmatter: { id, title },
+    sourceIds: [],
+    tags: [],
+    outLinks: [],
+  }
+}
+
+function index(pages: KnowledgePage[]): KnowledgeIndex {
+  return {
+    root: '/kb',
+    generatedAt: '2026-08-17T00:00:00.000Z',
+    sources: [],
+    pages,
+    graph: { nodes: [], edges: [] },
+  }
+}
+
+describe('detectNearDuplicatePages', () => {
+  it('reports exact normalized copies with stable page identity', () => {
+    const pages = [
+      page('copy-a', 'Same title', repeated.repeat(2)),
+      page('copy-b', 'Same title', repeated.repeat(2)),
+    ]
+
+    const report = detectNearDuplicatePages(pages)
+
+    expect(report).toMatchObject({
+      pageCount: 2,
+      eligiblePageCount: 2,
+      candidatePairCount: 1,
+      comparedPairCount: 1,
+      duplicatePairCount: 1,
+      duplicatePageCount: 2,
+      duplicatePageRate: 1,
+      truncated: false,
+    })
+    expect(report.pairs).toEqual([
+      expect.objectContaining({
+        leftPageId: 'copy-a',
+        rightPageId: 'copy-b',
+        similarity: 1,
+        exact: true,
+      }),
+    ])
+  })
+
+  it('finds near copies but not unrelated prose', () => {
+    const base = `${repeated.repeat(2)} The decisive measured value is forty two.`
+    const revised = `${repeated.repeat(2)} The decisive measured value is forty three.`
+    const unrelated =
+      'A kitchen inventory lists pans, knives, towels, plates, and groceries for a weekend dinner. '.repeat(
+        3,
+      )
+
+    const report = detectNearDuplicatePages(
+      [
+        page('base', 'Experiment', base),
+        page('revision', 'Experiment', revised),
+        page('unrelated', 'Kitchen', unrelated),
+      ],
+      { threshold: 0.7 },
+    )
+
+    expect(report.duplicatePairCount).toBe(1)
+    expect(report.pairs[0]).toMatchObject({
+      leftPageId: 'base',
+      rightPageId: 'revision',
+      exact: false,
+    })
+    expect(report.pairs[0]!.similarity).toBeGreaterThanOrEqual(0.7)
+  })
+
+  it('is deterministic under input reordering', () => {
+    const pages = [
+      page('z', 'Same title', repeated.repeat(2)),
+      page('a', 'Same title', repeated.repeat(2)),
+      page('m', 'Different title', `${repeated.repeat(2)} extra material`),
+    ]
+
+    const forward = detectNearDuplicatePages(pages, { threshold: 0.7 })
+    const reverse = detectNearDuplicatePages([...pages].reverse(), { threshold: 0.7 })
+
+    expect(reverse).toEqual(forward)
+  })
+
+  it('excludes short pages and reports bounded candidate truncation', () => {
+    const short = detectNearDuplicatePages(
+      [page('short-a', 'Short', 'same'), page('short-b', 'Short', 'same')],
+      { minCharacters: 80 },
+    )
+    expect(short).toMatchObject({ eligiblePageCount: 0, duplicatePairCount: 0 })
+
+    const bounded = detectNearDuplicatePages(
+      Array.from({ length: 4 }, (_, index) =>
+        page(`copy-${index}`, 'Same title', repeated.repeat(2)),
+      ),
+      { maxCandidatePairs: 2 },
+    )
+    expect(bounded).toMatchObject({
+      candidatePairCount: 2,
+      comparedPairCount: 2,
+      truncated: true,
+    })
+  })
+
+  it('normalizes Unicode width, case, whitespace, and URL identity', () => {
+    expect(normalizePageText('ＡLPHA  https://example.com/a\nBeta')).toBe('alpha <url> beta')
+  })
+
+  it('refuses malformed detector limits', () => {
+    expect(() => detectNearDuplicatePages([], { threshold: 1.1 })).toThrow(/threshold/)
+    expect(() => detectNearDuplicatePages([], { maxCandidatePairs: 0 })).toThrow(
+      /maxCandidatePairs/,
+    )
+    expect(() => detectNearDuplicatePages([], { minCharacters: -1 })).toThrow(/minCharacters/)
+  })
+})
+
+describe('scoreKnowledgeBaseIndex near-duplicate quality', () => {
+  const duplicatePages = [
+    page('copy-a', 'Same title', repeated.repeat(2)),
+    page('copy-b', 'Same title', repeated.repeat(2)),
+    page(
+      'independent',
+      'Independent result',
+      'A separate proof studies a different theorem, with distinct assumptions and a different verification artifact. '.repeat(
+        3,
+      ),
+    ),
+  ]
+
+  it('always reports deterministic duplicate metrics without failing by default', () => {
+    const report = scoreKnowledgeBaseIndex(index(duplicatePages))
+
+    expect(report.ok).toBe(true)
+    expect(report.metrics).toMatchObject({
+      near_duplicate_page_rate: 0.666667,
+      near_duplicate_page_count: 2,
+      near_duplicate_pair_count: 1,
+    })
+    expect(report.nearDuplicates?.pairs).toHaveLength(1)
+  })
+
+  it('fails a configured duplicate-page gate with measured evidence', () => {
+    const report = scoreKnowledgeBaseIndex(index(duplicatePages), {
+      maxNearDuplicatePageRate: 0.5,
+    })
+
+    expect(report.ok).toBe(false)
+    expect(report.findings).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          id: 'kb:near-duplicate-page-rate',
+          severity: 'error',
+          evidence: expect.objectContaining({
+            near_duplicate_page_rate: 0.666667,
+            near_duplicate_pair_count: 1,
+          }),
+        }),
+      ]),
+    )
+  })
+
+  it('fails closed when a configured gate rests on truncated analysis', () => {
+    const report = scoreKnowledgeBaseIndex(
+      index(
+        Array.from({ length: 4 }, (_, position) =>
+          page(`copy-${position}`, 'Same title', repeated.repeat(2)),
+        ),
+      ),
+      {
+        nearDuplicates: { maxCandidatePairs: 1 },
+        maxNearDuplicatePageRate: 0.9,
+      },
+    )
+
+    expect(report.ok).toBe(false)
+    expect(report.findings).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ id: 'kb:near-duplicate-analysis-truncated' }),
+      ]),
+    )
+  })
+
+  it('refuses a malformed quality threshold', () => {
+    expect(() =>
+      scoreKnowledgeBaseIndex(index(duplicatePages), { maxNearDuplicatePageRate: -0.1 }),
+    ).toThrow(/maxNearDuplicatePageRate/)
+  })
+})