From 16e28856ed1b84e9e02f3ea2abcbff631ae1870b Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 19:39:50 -0700 Subject: [PATCH 1/7] feat(improvement): remeasure selected candidate changes before promotion --- src/kb-improvement/selected-candidate.ts | 481 +++++++++++++++++++++++ 1 file changed, 481 insertions(+) create mode 100644 src/kb-improvement/selected-candidate.ts diff --git a/src/kb-improvement/selected-candidate.ts b/src/kb-improvement/selected-candidate.ts new file mode 100644 index 0000000..d9c8102 --- /dev/null +++ b/src/kb-improvement/selected-candidate.ts @@ -0,0 +1,481 @@ +import { createHash } from 'node:crypto' +import { join } from 'node:path' +import { canonicalJson, contentHash } from '@tangle-network/agent-eval' +import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' +import { z } from 'zod' +import { + isMissingFile, + readRegularFileWithinRoot, + writeJsonDurableWithinRoot, +} from '../durable-fs' +import { + applyKnowledgeFileTransaction, + assertKnowledgeMutationPath, + finishKnowledgeFileTransaction, + type KnowledgeFileMutation, + type KnowledgeFileTransaction, + type KnowledgeFileTransactionPlanEntry, + knowledgeFileTransactionPlanHash, + prepareKnowledgeFileTransaction, +} from '../file-transaction' +import { writeKnowledgeIndex } from '../indexer' +import { stableId } from '../ids' +import { withKnowledgeMutation } from '../mutation-lock' +import type { RagKnowledgeImprovementPhase } from '../rag-improvement-loop' +import type { + KnowledgeImprovementCandidateRef, + KnowledgeImprovementOptions, + KnowledgeImprovementResult, +} from './contracts' +import { + EVALUATION_PHASES, + immutableRefSchema, + KnowledgeImprovementCandidateRefSchema, + safePathSegmentSchema, +} from './contracts' +import { improveKnowledgeBase } from './run' +import { knowledgeImprovementRunDir } from './state' +import { knowledgeFilePlanEntries } from './transition' +import { hashKnowledgeBase, withKnowledgeImprovementComparison } from './workspace' + +const DERIVED_KNOWLEDGE_PATHS = new Set(['knowledge/index.md']) +const selectionPathSchema = z.string().min(1).transform((path, context) => { + try { + return assertKnowledgeMutationPath(path) + } catch (error) { + context.addIssue({ + code: 'custom', + message: error instanceof Error ? error.message : String(error), + }) + return z.NEVER + } +}) +const selectionMetadataSchema = z.record(z.string(), z.json()) +const measuredSelectionLifecycleSchema = z + .object({ + kind: z.literal('measured-knowledge-change-selection'), + version: z.literal(1), + selectionDigest: z.string().regex(/^[a-f0-9]{64}$/), + sourceCandidateId: z.string().min(1), + sourceEvidenceHash: z.string().regex(/^[a-f0-9]{64}$/), + sourcePlanHash: z.string().regex(/^[a-f0-9]{64}$/), + selectedPaths: z.array(selectionPathSchema), + selectedCandidateHash: z.string().regex(/^[a-f0-9]{64}$/), + selectedPlanHash: z.string().regex(/^[a-f0-9]{64}$/), + }) + .strict() +const measuredSelectionReceiptSchema = z + .object({ + kind: z.literal('measured-knowledge-change-selection-receipt'), + version: z.literal(1), + receiptHash: z.string().regex(/^[a-f0-9]{64}$/), + selectionDigest: z.string().regex(/^[a-f0-9]{64}$/), + sourceCandidate: KnowledgeImprovementCandidateRefSchema, + sourcePlanHash: z.string().regex(/^[a-f0-9]{64}$/), + selectedPaths: z.array(selectionPathSchema), + derivedImplementationRef: immutableRefSchema, + runId: z.string().min(1), + derivedCandidateId: z.string().min(1), + derivedCandidateStatus: z.string().min(1), + selectedCandidateHash: z.string().regex(/^[a-f0-9]{64}$/), + selectedPlanHash: z.string().regex(/^[a-f0-9]{64}$/), + selectedEvidenceHash: z.string().regex(/^[a-f0-9]{64}$/), + rationale: z.string().trim().min(1).optional(), + metadata: selectionMetadataSchema.optional(), + }) + .strict() + +export type KnowledgeEvaluationPhase = Exclude< + RagKnowledgeImprovementPhase, + 'knowledge-acquisition' | 'knowledge-update' +> + +export interface ImproveSelectedKnowledgeCandidateOptions + extends Omit< + KnowledgeImprovementOptions, + | 'root' + | 'goal' + | 'implementationRef' + | 'runId' + | 'step' + | 'knowledgeResearch' + | 'acquireKnowledge' + | 'updateKnowledge' + | 'enabledPhases' + | 'requiredPhases' + > { + root: string + goal: string + /** Identity of the helper, policy, or human procedure choosing the subset. */ + implementationRef: string + /** Previously measured whole candidate from which the subset is derived. */ + sourceCandidate: KnowledgeImprovementCandidateRef + /** Exact changed file paths to carry into the derived candidate. */ + selectedPaths: readonly string[] + /** Optional explicit run identity. The default includes the selection digest. */ + runId?: string + /** Human-readable reason for selecting this subset. */ + rationale?: string + /** JSON-safe policy output retained in the selection receipt. */ + selectionMetadata?: Record + /** Evaluation-only phases. `knowledge-update` is inserted by this helper. */ + enabledEvaluationPhases?: readonly KnowledgeEvaluationPhase[] + /** Evaluation-only phases that must complete. `knowledge-update` is always required. */ + requiredEvaluationPhases?: readonly KnowledgeEvaluationPhase[] +} + +export type MeasuredKnowledgeSelectionReceipt = z.infer + +export interface ImproveSelectedKnowledgeCandidateResult extends KnowledgeImprovementResult { + selection: MeasuredKnowledgeSelectionReceipt +} + +/** + * Derive a subset from an already measured candidate, then measure the subset as + * a new candidate before it can be promoted. + * + * Directly filtering an activation plan is unsafe: a whole candidate can pass + * while one page subset breaks links, removes supporting sources, or changes a + * readiness score. This helper instead applies the chosen changed paths to an + * isolated baseline, recomputes the index, runs the ordinary improvement + * evaluator, and returns the ordinary candidate reference. Promotion therefore + * remains one path and can never admit an unmeasured hybrid. + */ +export async function improveSelectedKnowledgeCandidate( + options: ImproveSelectedKnowledgeCandidateOptions, +): Promise { + const sourceCandidate = Object.freeze( + KnowledgeImprovementCandidateRefSchema.parse(options.sourceCandidate), + ) + const requestedImplementationRef = immutableRefSchema.parse(options.implementationRef) + const enabledEvaluationPhases = normalizeEvaluationPhases( + options.enabledEvaluationPhases ?? EVALUATION_PHASES, + 'enabledEvaluationPhases', + ) + const requiredEvaluationPhases = normalizeEvaluationPhases( + options.requiredEvaluationPhases ?? [], + 'requiredEvaluationPhases', + ) + const metadata = options.selectionMetadata + ? selectionMetadataSchema.parse(structuredClone(options.selectionMetadata)) + : undefined + + return withKnowledgeImprovementComparison( + { root: options.root, candidate: sourceCandidate }, + async (source) => { + const sourcePlan = await knowledgeFilePlanEntries(source.baseline.root, source.candidate.root) + const sourcePlanHash = knowledgeFileTransactionPlanHash(sourcePlan) + if (sourcePlanHash !== sourceCandidate.promotionPlanHash) { + throw new Error('source knowledge candidate plan no longer matches its measured identity') + } + const changedSourcePlan = sourcePlan.filter(planEntryChanged).filter(notDerivedPath) + const selectedPaths = normalizeSelectedPaths(options.selectedPaths, changedSourcePlan) + const selectionMaterial = immutableJson({ + kind: 'measured-knowledge-change-selection' as const, + version: 1 as const, + sourceCandidate, + sourcePlanHash, + selectedPaths, + ...(options.rationale?.trim() ? { rationale: options.rationale.trim() } : {}), + ...(metadata ? { metadata } : {}), + }) + const selectionDigest = contentHash(selectionMaterial) + const derivedImplementationRef = immutableRefSchema.parse( + `sha256:${contentHash({ + engine: 'agent-knowledge/measured-selection-v1', + implementationRef: requestedImplementationRef, + selectionDigest, + })}`, + ) + const runId = + options.runId ?? + stableId( + 'kimpsel', + canonicalJson({ + root: options.root, + goal: options.goal, + derivedImplementationRef, + selectionDigest, + }), + ) + const selectedEntries = selectedPaths.map((path) => { + const entry = changedSourcePlan.find((candidate) => candidate.path === path) + if (!entry) throw new Error(`selected knowledge path disappeared: ${path}`) + return entry + }) + const selectionMutationPlanHash = knowledgeFileTransactionPlanHash(selectedEntries) + let lifecycleSelection: z.infer | undefined + + const result = await improveKnowledgeBase({ + ...improvementOptions(options), + root: options.root, + goal: options.goal, + implementationRef: derivedImplementationRef, + runId, + enabledPhases: ['knowledge-update', ...enabledEvaluationPhases], + requiredPhases: ['knowledge-update', ...requiredEvaluationPhases], + async updateKnowledge(input) { + const purpose = `knowledge-selected-candidate:${selectionDigest}` + const recoveryOwner = `knowledge-selected-candidate:${sourceCandidate.candidateId}` + await withKnowledgeMutation( + input.candidateRoot, + async (lock) => { + lock.assertOwned() + if (!lock.recovery && selectedEntries.length > 0) { + const transaction = await prepareKnowledgeFileTransaction({ + root: input.candidateRoot, + transactionRoot: lock.transactionRoot, + purpose, + recoveryOwner, + mutations: await selectionMutations(source.candidate.root, selectedEntries), + now: options.now, + }) + if (transaction) { + assertSelectionTransaction(transaction, selectionMutationPlanHash) + await applyKnowledgeFileTransaction({ + root: input.candidateRoot, + transactionRoot: lock.transactionRoot, + transaction, + beforeCommit: lock.assertOwned, + }) + await finishKnowledgeFileTransaction({ + root: input.candidateRoot, + transactionRoot: lock.transactionRoot, + transaction, + assertOwned: lock.assertOwned, + }) + } + } + await writeKnowledgeIndex(input.candidateRoot) + lock.assertOwned() + }, + { + resumeTransaction: { + purpose, + recoveryOwner, + validate: (transaction) => + assertSelectionTransaction(transaction, selectionMutationPlanHash), + }, + }, + ) + + const selectedPlan = await knowledgeFilePlanEntries( + source.baseline.root, + input.candidateRoot, + ) + assertExactSelectedChanges(selectedPlan, selectedPaths) + const selectedPlanHash = knowledgeFileTransactionPlanHash(selectedPlan) + const selectedCandidateHash = await hashKnowledgeBase(input.candidateRoot) + lifecycleSelection = measuredSelectionLifecycleSchema.parse({ + kind: 'measured-knowledge-change-selection', + version: 1, + selectionDigest, + sourceCandidateId: sourceCandidate.candidateId, + sourceEvidenceHash: sourceCandidate.evidenceHash, + sourcePlanHash, + selectedPaths, + selectedCandidateHash, + selectedPlanHash, + }) + return { + applied: selectedPaths.length > 0, + summary: + selectedPaths.length > 0 + ? `Applied ${selectedPaths.length} selected measured change(s).` + : 'Selected no changes; measuring the exact baseline as a null candidate.', + metadata: { selection: lifecycleSelection }, + } + }, + }) + + const candidate = result.candidate + if (!candidate?.candidateHash || !candidate.promotionPlanHash || !candidate.evidenceHash) { + throw new Error('selected knowledge candidate did not produce measured candidate evidence') + } + const recordedSelection = measuredSelectionLifecycleSchema.parse( + result.lifecycle?.knowledgeUpdate?.metadata?.selection ?? lifecycleSelection, + ) + if ( + recordedSelection.selectionDigest !== selectionDigest || + recordedSelection.selectedCandidateHash !== candidate.candidateHash || + recordedSelection.selectedPlanHash !== candidate.promotionPlanHash || + canonicalJson(recordedSelection.selectedPaths) !== canonicalJson(selectedPaths) + ) { + throw new Error('selected knowledge candidate evidence does not bind the measured subset') + } + + const receiptWithoutHash = immutableJson({ + kind: 'measured-knowledge-change-selection-receipt' as const, + version: 1 as const, + selectionDigest, + sourceCandidate, + sourcePlanHash, + selectedPaths, + derivedImplementationRef, + runId: result.runId, + derivedCandidateId: candidate.candidateId, + derivedCandidateStatus: candidate.status, + selectedCandidateHash: candidate.candidateHash, + selectedPlanHash: candidate.promotionPlanHash, + selectedEvidenceHash: candidate.evidenceHash, + ...(options.rationale?.trim() ? { rationale: options.rationale.trim() } : {}), + ...(metadata ? { metadata } : {}), + }) + const receipt = measuredSelectionReceiptSchema.parse({ + ...receiptWithoutHash, + receiptHash: contentHash(receiptWithoutHash), + }) + await persistSelectionReceipt(options.root, receipt) + return { ...result, selection: receipt } + }, + ) +} + +function improvementOptions( + options: ImproveSelectedKnowledgeCandidateOptions, +): Omit< + KnowledgeImprovementOptions, + | 'root' + | 'goal' + | 'implementationRef' + | 'runId' + | 'step' + | 'knowledgeResearch' + | 'acquireKnowledge' + | 'updateKnowledge' + | 'enabledPhases' + | 'requiredPhases' +> { + const { + root: _root, + goal: _goal, + implementationRef: _implementationRef, + sourceCandidate: _sourceCandidate, + selectedPaths: _selectedPaths, + runId: _runId, + rationale: _rationale, + selectionMetadata: _selectionMetadata, + enabledEvaluationPhases: _enabledEvaluationPhases, + requiredEvaluationPhases: _requiredEvaluationPhases, + ...rest + } = options + return rest +} + +function normalizeEvaluationPhases( + phases: readonly RagKnowledgeImprovementPhase[], + field: string, +): KnowledgeEvaluationPhase[] { + const unique = [...new Set(phases)] + for (const phase of unique) { + if (phase === 'knowledge-acquisition' || phase === 'knowledge-update') { + throw new Error(`${field} cannot contain the selection-owned phase '${phase}'`) + } + } + return unique as KnowledgeEvaluationPhase[] +} + +function normalizeSelectedPaths( + paths: readonly string[], + changedPlan: readonly KnowledgeFileTransactionPlanEntry[], +): string[] { + const available = new Set(changedPlan.map((entry) => entry.path)) + const selected: string[] = [] + const seen = new Set() + for (const input of paths) { + const path = selectionPathSchema.parse(input) + if (DERIVED_KNOWLEDGE_PATHS.has(path)) { + throw new Error(`derived knowledge path cannot be selected directly: ${path}`) + } + if (seen.has(path)) throw new Error(`selected knowledge path is repeated: ${path}`) + if (!available.has(path)) { + throw new Error(`selected knowledge path is not a changed source-candidate file: ${path}`) + } + seen.add(path) + selected.push(path) + } + return selected.sort((left, right) => left.localeCompare(right)) +} + +function planEntryChanged(entry: KnowledgeFileTransactionPlanEntry): boolean { + return ( + entry.beforeHash !== entry.afterHash || + (entry.beforeHash !== null && entry.afterHash !== null && entry.beforeMode !== entry.afterMode) + ) +} + +function notDerivedPath(entry: KnowledgeFileTransactionPlanEntry): boolean { + return !DERIVED_KNOWLEDGE_PATHS.has(entry.path) +} + +async function selectionMutations( + sourceCandidateRoot: string, + entries: readonly KnowledgeFileTransactionPlanEntry[], +): Promise { + return Promise.all( + entries.map(async (entry) => { + if (entry.afterHash === null) return { path: entry.path, content: null } + const file = await readRegularFileWithinRoot(sourceCandidateRoot, entry.path) + const actualHash = createHash('sha256').update(file.bytes).digest('hex') + if (actualHash !== entry.afterHash || file.mode !== entry.afterMode) { + throw new Error(`source candidate changed before subset materialization: ${entry.path}`) + } + return { path: entry.path, content: file.bytes, mode: file.mode } + }), + ) +} + +function assertSelectionTransaction( + transaction: KnowledgeFileTransaction, + expectedPlanHash: string, +): void { + if (knowledgeFileTransactionPlanHash(transaction.entries) !== expectedPlanHash) { + throw new Error('selected knowledge transaction does not match its approved path set') + } +} + +function assertExactSelectedChanges( + plan: readonly KnowledgeFileTransactionPlanEntry[], + selectedPaths: readonly string[], +): void { + const actual = plan + .filter(planEntryChanged) + .filter(notDerivedPath) + .map((entry) => entry.path) + .sort((left, right) => left.localeCompare(right)) + if (canonicalJson(actual) !== canonicalJson(selectedPaths)) { + throw new Error( + `selected knowledge candidate changed the wrong files: expected ${selectedPaths.join(', ') || '(none)'}, got ${actual.join(', ') || '(none)'}`, + ) + } +} + +async function persistSelectionReceipt( + root: string, + receipt: MeasuredKnowledgeSelectionReceipt, +): Promise { + const runDir = knowledgeImprovementRunDir(root, receipt.runId) + const relativePath = join( + 'candidates', + safePathSegmentSchema.parse(receipt.derivedCandidateId), + 'selection.json', + ).replace(/\\/g, '/') + try { + const existing = measuredSelectionReceiptSchema.parse( + JSON.parse((await readRegularFileWithinRoot(runDir, relativePath)).bytes.toString('utf8')), + ) + if (canonicalJson(existing) !== canonicalJson(receipt)) { + throw new Error('measured knowledge selection receipt conflicts with durable content') + } + return + } catch (error) { + if (!isMissingFile(error)) throw error + } + await writeJsonDurableWithinRoot(runDir, relativePath, receipt) +} + +function immutableJson(value: T): T { + if (value === null || typeof value !== 'object') return value + for (const child of Object.values(value)) immutableJson(child) + return Object.freeze(value) +} From 53479f1b0101d18150e9456f6aaea5bb6cf03672 Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 19:40:35 -0700 Subject: [PATCH 2/7] feat(improvement): export measured partial promotion --- src/kb-improvement.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/kb-improvement.ts b/src/kb-improvement.ts index a5fe46a..da26469 100644 --- a/src/kb-improvement.ts +++ b/src/kb-improvement.ts @@ -40,6 +40,13 @@ export type { } from './kb-improvement/optimization' export { optimizeKnowledgeBasePolicy } from './kb-improvement/optimization' export { improveKnowledgeBase } from './kb-improvement/run' +export type { + ImproveSelectedKnowledgeCandidateOptions, + ImproveSelectedKnowledgeCandidateResult, + KnowledgeEvaluationPhase, + MeasuredKnowledgeSelectionReceipt, +} from './kb-improvement/selected-candidate' +export { improveSelectedKnowledgeCandidate } from './kb-improvement/selected-candidate' export type { KnowledgeImprovementEvent } from './kb-improvement/state' export { knowledgeImprovementRunDir, From 596cb8d2db330e4d61629ddcd08117a4e2a38285 Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 19:42:26 -0700 Subject: [PATCH 3/7] test(improvement): prove selected changes are remeasured before promotion --- .../kb-improvement/selected-candidate.test.ts | 231 ++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 tests/kb-improvement/selected-candidate.test.ts diff --git a/tests/kb-improvement/selected-candidate.test.ts b/tests/kb-improvement/selected-candidate.test.ts new file mode 100644 index 0000000..7c21cb9 --- /dev/null +++ b/tests/kb-improvement/selected-candidate.test.ts @@ -0,0 +1,231 @@ +import { readFile, rm, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { + hashKnowledgeBase, + improveSelectedKnowledgeCandidate, + knowledgeImprovementCandidateRef, + knowledgeImprovementRunDir, + promoteKnowledgeCandidate, + withKnowledgeImprovementComparison, +} from '../../src/index' +import { + improveTestKnowledgeBase as improveKnowledgeBase, + passingMetric, + TEST_KNOWLEDGE_IMPLEMENTATION_REF, + withKb, +} from '../support/kb-improvement' + +describe('improveSelectedKnowledgeCandidate', () => { + it('remeasures the exact selected subset before ordinary promotion', async () => { + await withKb(async (root) => { + await writeFile(join(root, 'knowledge', 'original.md'), '# Original\n') + const baseHash = await hashKnowledgeBase(root) + const source = await improveKnowledgeBase({ + root, + goal: 'Generate a broad candidate from which a measured subset will be selected', + runId: 'broad-source-candidate', + updateKnowledge: async ({ candidateRoot }) => { + await writeFile(join(candidateRoot, 'knowledge', 'keep.md'), '# Keep\n') + await writeFile(join(candidateRoot, 'knowledge', 'drop.md'), '# Drop\n') + await writeFile(join(candidateRoot, 'knowledge', 'original.md'), '# Changed\n') + return { applied: true, summary: 'created a three-file broad candidate' } + }, + evaluate: passingMetric, + }) + const sourceCandidate = knowledgeImprovementCandidateRef(source) + let evaluatedCandidateRoot = '' + + const selected = await improveSelectedKnowledgeCandidate({ + root, + goal: 'Measure only the useful files from the broad candidate', + implementationRef: TEST_KNOWLEDGE_IMPLEMENTATION_REF, + sourceCandidate, + selectedPaths: ['knowledge/original.md', 'knowledge/keep.md'], + rationale: 'The dropped page is redundant with an existing source.', + selectionMetadata: { reviewer: 'test-reviewer', policyVersion: 1 }, + evaluate(input) { + if (input.candidateRoot !== input.baselineRoot) { + evaluatedCandidateRoot = input.candidateRoot + } + return passingMetric() + }, + }) + const candidate = knowledgeImprovementCandidateRef(selected) + + expect(candidate.baseHash).toBe(baseHash) + expect(selected.selection).toMatchObject({ + kind: 'measured-knowledge-change-selection-receipt', + sourceCandidate, + selectedPaths: ['knowledge/keep.md', 'knowledge/original.md'], + selectedCandidateHash: candidate.candidateHash, + selectedPlanHash: candidate.promotionPlanHash, + selectedEvidenceHash: candidate.evidenceHash, + }) + expect(evaluatedCandidateRoot).not.toBe('') + await expect(readFile(join(evaluatedCandidateRoot, 'knowledge', 'keep.md'), 'utf8')).resolves.toBe( + '# Keep\n', + ) + await expect( + readFile(join(evaluatedCandidateRoot, 'knowledge', 'original.md'), 'utf8'), + ).resolves.toBe('# Changed\n') + await expect( + readFile(join(evaluatedCandidateRoot, 'knowledge', 'drop.md'), 'utf8'), + ).rejects.toMatchObject({ code: 'ENOENT' }) + + await withKnowledgeImprovementComparison({ root, candidate }, async (comparison) => { + await expect( + readFile(join(comparison.candidate.root, 'knowledge', 'keep.md'), 'utf8'), + ).resolves.toBe('# Keep\n') + await expect( + readFile(join(comparison.candidate.root, 'knowledge', 'drop.md'), 'utf8'), + ).rejects.toMatchObject({ code: 'ENOENT' }) + }) + + const promoted = await promoteKnowledgeCandidate({ root, candidate }) + expect(promoted).toMatchObject({ promoted: true, blocked: false }) + await expect(readFile(join(root, 'knowledge', 'keep.md'), 'utf8')).resolves.toBe('# Keep\n') + await expect(readFile(join(root, 'knowledge', 'original.md'), 'utf8')).resolves.toBe( + '# Changed\n', + ) + await expect(readFile(join(root, 'knowledge', 'drop.md'), 'utf8')).rejects.toMatchObject({ + code: 'ENOENT', + }) + + const receiptPath = join( + knowledgeImprovementRunDir(root, candidate.runId), + 'candidates', + candidate.candidateId, + 'selection.json', + ) + const storedReceipt = JSON.parse(await readFile(receiptPath, 'utf8')) + expect(storedReceipt).toEqual(selected.selection) + }) + }) + + it('makes a harmful subset fail its own evaluator instead of inheriting the whole-candidate pass', async () => { + await withKb(async (root) => { + const source = await improveKnowledgeBase({ + root, + goal: 'Create a whole candidate whose two pages work together', + runId: 'whole-pair-source', + updateKnowledge: async ({ candidateRoot }) => { + await writeFile(join(candidateRoot, 'knowledge', 'claim.md'), '# Claim\nUses support.\n') + await writeFile(join(candidateRoot, 'knowledge', 'support.md'), '# Support\nEvidence.\n') + return { applied: true, summary: 'created claim and support pages' } + }, + evaluate: passingMetric, + }) + const sourceCandidate = knowledgeImprovementCandidateRef(source) + + const selected = await improveSelectedKnowledgeCandidate({ + root, + goal: 'Test whether the claim page survives without its support page', + implementationRef: TEST_KNOWLEDGE_IMPLEMENTATION_REF, + sourceCandidate, + selectedPaths: ['knowledge/claim.md'], + async evaluate({ candidateRoot }) { + try { + await readFile(join(candidateRoot, 'knowledge', 'support.md'), 'utf8') + return passingMetric() + } catch { + return { + score: 0, + passed: false, + notes: 'claim is missing its supporting page', + provenance: { + evaluator: 'selected-candidate-dependency-check', + version: '1', + method: 'deterministic', + }, + } + } + }, + }) + + expect(selected.candidate).toMatchObject({ status: 'rejected' }) + expect(selected.evaluation).toMatchObject({ passed: false }) + await expect( + promoteKnowledgeCandidate({ root, candidate: knowledgeImprovementCandidateRef(selected) }), + ).rejects.toThrow(/not ready for promotion/) + await expect(readFile(join(root, 'knowledge', 'claim.md'), 'utf8')).rejects.toMatchObject({ + code: 'ENOENT', + }) + }) + }) + + it('rejects unknown, repeated, and generated paths before opening a derived run', async () => { + await withKb(async (root) => { + const source = await improveKnowledgeBase({ + root, + goal: 'Create one selectable page', + runId: 'selection-shape-source', + updateKnowledge: async ({ candidateRoot }) => { + await writeFile(join(candidateRoot, 'knowledge', 'page.md'), '# Page\n') + return { applied: true, summary: 'created one page' } + }, + evaluate: passingMetric, + }) + const sourceCandidate = knowledgeImprovementCandidateRef(source) + const common = { + root, + goal: 'Refuse a malformed selection', + implementationRef: TEST_KNOWLEDGE_IMPLEMENTATION_REF, + sourceCandidate, + evaluate: passingMetric, + } + + await expect( + improveSelectedKnowledgeCandidate({ ...common, selectedPaths: ['knowledge/missing.md'] }), + ).rejects.toThrow(/not a changed source-candidate file/) + await expect( + improveSelectedKnowledgeCandidate({ + ...common, + selectedPaths: ['knowledge/page.md', 'knowledge/page.md'], + }), + ).rejects.toThrow(/repeated/) + await expect( + improveSelectedKnowledgeCandidate({ ...common, selectedPaths: ['knowledge/index.md'] }), + ).rejects.toThrow(/derived knowledge path/) + }) + }) + + it('reopens the same measured selection without rerunning its update', async () => { + await withKb(async (root) => { + const source = await improveKnowledgeBase({ + root, + goal: 'Create a resumable source candidate', + runId: 'selected-resume-source', + updateKnowledge: async ({ candidateRoot }) => { + await writeFile(join(candidateRoot, 'knowledge', 'selected.md'), '# Selected\n') + return { applied: true, summary: 'created selected page' } + }, + evaluate: passingMetric, + }) + const options = { + root, + goal: 'Measure a stable selected candidate', + runId: 'selected-resume-derived', + implementationRef: TEST_KNOWLEDGE_IMPLEMENTATION_REF, + sourceCandidate: knowledgeImprovementCandidateRef(source), + selectedPaths: ['knowledge/selected.md'], + evaluate: passingMetric, + } + + const first = await improveSelectedKnowledgeCandidate(options) + const mutableRoot = join( + knowledgeImprovementRunDir(root, first.runId), + 'candidates', + first.candidate!.candidateId, + 'workspace', + ) + await rm(mutableRoot, { recursive: true, force: true }) + const second = await improveSelectedKnowledgeCandidate(options) + + expect(second.selection).toEqual(first.selection) + expect(knowledgeImprovementCandidateRef(second)).toEqual( + knowledgeImprovementCandidateRef(first), + ) + }) + }) +}) From 2aaf8485d92b6c49c2380d80f819d37a636451be Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 19:45:41 -0700 Subject: [PATCH 4/7] test(improvement): inspect the selected snapshot before it is released --- .../kb-improvement/selected-candidate.test.ts | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/tests/kb-improvement/selected-candidate.test.ts b/tests/kb-improvement/selected-candidate.test.ts index 7c21cb9..676cdf6 100644 --- a/tests/kb-improvement/selected-candidate.test.ts +++ b/tests/kb-improvement/selected-candidate.test.ts @@ -34,7 +34,7 @@ describe('improveSelectedKnowledgeCandidate', () => { evaluate: passingMetric, }) const sourceCandidate = knowledgeImprovementCandidateRef(source) - let evaluatedCandidateRoot = '' + let evaluatedSelectedSnapshot = false const selected = await improveSelectedKnowledgeCandidate({ root, @@ -44,15 +44,24 @@ describe('improveSelectedKnowledgeCandidate', () => { selectedPaths: ['knowledge/original.md', 'knowledge/keep.md'], rationale: 'The dropped page is redundant with an existing source.', selectionMetadata: { reviewer: 'test-reviewer', policyVersion: 1 }, - evaluate(input) { - if (input.candidateRoot !== input.baselineRoot) { - evaluatedCandidateRoot = input.candidateRoot - } + async evaluate(input) { + expect(input.candidateRoot).not.toBe(input.baselineRoot) + await expect( + readFile(join(input.candidateRoot, 'knowledge', 'keep.md'), 'utf8'), + ).resolves.toBe('# Keep\n') + await expect( + readFile(join(input.candidateRoot, 'knowledge', 'original.md'), 'utf8'), + ).resolves.toBe('# Changed\n') + await expect( + readFile(join(input.candidateRoot, 'knowledge', 'drop.md'), 'utf8'), + ).rejects.toMatchObject({ code: 'ENOENT' }) + evaluatedSelectedSnapshot = true return passingMetric() }, }) const candidate = knowledgeImprovementCandidateRef(selected) + expect(evaluatedSelectedSnapshot).toBe(true) expect(candidate.baseHash).toBe(baseHash) expect(selected.selection).toMatchObject({ kind: 'measured-knowledge-change-selection-receipt', @@ -62,16 +71,6 @@ describe('improveSelectedKnowledgeCandidate', () => { selectedPlanHash: candidate.promotionPlanHash, selectedEvidenceHash: candidate.evidenceHash, }) - expect(evaluatedCandidateRoot).not.toBe('') - await expect(readFile(join(evaluatedCandidateRoot, 'knowledge', 'keep.md'), 'utf8')).resolves.toBe( - '# Keep\n', - ) - await expect( - readFile(join(evaluatedCandidateRoot, 'knowledge', 'original.md'), 'utf8'), - ).resolves.toBe('# Changed\n') - await expect( - readFile(join(evaluatedCandidateRoot, 'knowledge', 'drop.md'), 'utf8'), - ).rejects.toMatchObject({ code: 'ENOENT' }) await withKnowledgeImprovementComparison({ root, candidate }, async (comparison) => { await expect( From 97bab16f514a7d8ba02d9df762eda61a78c7c2bf Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 19:52:30 -0700 Subject: [PATCH 5/7] style(improvement): format measured selection implementation --- src/kb-improvement/selected-candidate.ts | 38 +++++++++++++----------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/src/kb-improvement/selected-candidate.ts b/src/kb-improvement/selected-candidate.ts index d9c8102..934d799 100644 --- a/src/kb-improvement/selected-candidate.ts +++ b/src/kb-improvement/selected-candidate.ts @@ -3,11 +3,7 @@ import { join } from 'node:path' import { canonicalJson, contentHash } from '@tangle-network/agent-eval' import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' import { z } from 'zod' -import { - isMissingFile, - readRegularFileWithinRoot, - writeJsonDurableWithinRoot, -} from '../durable-fs' +import { isMissingFile, readRegularFileWithinRoot, writeJsonDurableWithinRoot } from '../durable-fs' import { applyKnowledgeFileTransaction, assertKnowledgeMutationPath, @@ -18,8 +14,8 @@ import { knowledgeFileTransactionPlanHash, prepareKnowledgeFileTransaction, } from '../file-transaction' -import { writeKnowledgeIndex } from '../indexer' import { stableId } from '../ids' +import { writeKnowledgeIndex } from '../indexer' import { withKnowledgeMutation } from '../mutation-lock' import type { RagKnowledgeImprovementPhase } from '../rag-improvement-loop' import type { @@ -39,17 +35,20 @@ import { knowledgeFilePlanEntries } from './transition' import { hashKnowledgeBase, withKnowledgeImprovementComparison } from './workspace' const DERIVED_KNOWLEDGE_PATHS = new Set(['knowledge/index.md']) -const selectionPathSchema = z.string().min(1).transform((path, context) => { - try { - return assertKnowledgeMutationPath(path) - } catch (error) { - context.addIssue({ - code: 'custom', - message: error instanceof Error ? error.message : String(error), - }) - return z.NEVER - } -}) +const selectionPathSchema = z + .string() + .min(1) + .transform((path, context) => { + try { + return assertKnowledgeMutationPath(path) + } catch (error) { + context.addIssue({ + code: 'custom', + message: error instanceof Error ? error.message : String(error), + }) + return z.NEVER + } + }) const selectionMetadataSchema = z.record(z.string(), z.json()) const measuredSelectionLifecycleSchema = z .object({ @@ -476,6 +475,9 @@ async function persistSelectionReceipt( function immutableJson(value: T): T { if (value === null || typeof value !== 'object') return value - for (const child of Object.values(value)) immutableJson(child) + const children: readonly unknown[] = Array.isArray(value) + ? value + : Object.values(value as Record) + for (const child of children) immutableJson(child) return Object.freeze(value) } From 4d11513c922ac4a689bfc61d81d136e7c167d4bb Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 20:02:01 -0700 Subject: [PATCH 6/7] test(improvement): assert rejected and resumed selection semantics --- .../kb-improvement/selected-candidate.test.ts | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/tests/kb-improvement/selected-candidate.test.ts b/tests/kb-improvement/selected-candidate.test.ts index 676cdf6..8978215 100644 --- a/tests/kb-improvement/selected-candidate.test.ts +++ b/tests/kb-improvement/selected-candidate.test.ts @@ -1,4 +1,4 @@ -import { readFile, rm, writeFile } from 'node:fs/promises' +import { readFile, writeFile } from 'node:fs/promises' import { join } from 'node:path' import { describe, expect, it } from 'vitest' import { @@ -144,9 +144,7 @@ describe('improveSelectedKnowledgeCandidate', () => { expect(selected.candidate).toMatchObject({ status: 'rejected' }) expect(selected.evaluation).toMatchObject({ passed: false }) - await expect( - promoteKnowledgeCandidate({ root, candidate: knowledgeImprovementCandidateRef(selected) }), - ).rejects.toThrow(/not ready for promotion/) + expect(() => knowledgeImprovementCandidateRef(selected)).toThrow(/not ready for promotion/) await expect(readFile(join(root, 'knowledge', 'claim.md'), 'utf8')).rejects.toMatchObject({ code: 'ENOENT', }) @@ -189,7 +187,7 @@ describe('improveSelectedKnowledgeCandidate', () => { }) }) - it('reopens the same measured selection without rerunning its update', async () => { + it('reopens the same measured selection without rerunning its evaluator', async () => { await withKb(async (root) => { const source = await improveKnowledgeBase({ root, @@ -201,6 +199,7 @@ describe('improveSelectedKnowledgeCandidate', () => { }, evaluate: passingMetric, }) + let evaluationCalls = 0 const options = { root, goal: 'Measure a stable selected candidate', @@ -208,23 +207,21 @@ describe('improveSelectedKnowledgeCandidate', () => { implementationRef: TEST_KNOWLEDGE_IMPLEMENTATION_REF, sourceCandidate: knowledgeImprovementCandidateRef(source), selectedPaths: ['knowledge/selected.md'], - evaluate: passingMetric, + evaluate() { + evaluationCalls += 1 + return passingMetric() + }, } const first = await improveSelectedKnowledgeCandidate(options) - const mutableRoot = join( - knowledgeImprovementRunDir(root, first.runId), - 'candidates', - first.candidate!.candidateId, - 'workspace', - ) - await rm(mutableRoot, { recursive: true, force: true }) + expect(evaluationCalls).toBe(1) const second = await improveSelectedKnowledgeCandidate(options) + expect(evaluationCalls).toBe(1) expect(second.selection).toEqual(first.selection) expect(knowledgeImprovementCandidateRef(second)).toEqual( knowledgeImprovementCandidateRef(first), ) }) }) -}) +}) \ No newline at end of file From 3d5dd9b50eebf49898c10fb09e4f206c93b62fb9 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sun, 16 Aug 2026 23:08:55 -0600 Subject: [PATCH 7/7] test(improvement): assert the message the rejected-candidate ref actually throws candidateRefFor throws "knowledge candidate '' is not ready". The assertion expected "not ready for promotion", a wording that exists only on the promote path this call never reaches, so the test failed on every run. Match the thrown message and terminate the file with a newline for biome. --- tests/kb-improvement/selected-candidate.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/kb-improvement/selected-candidate.test.ts b/tests/kb-improvement/selected-candidate.test.ts index 8978215..f32be67 100644 --- a/tests/kb-improvement/selected-candidate.test.ts +++ b/tests/kb-improvement/selected-candidate.test.ts @@ -144,7 +144,7 @@ describe('improveSelectedKnowledgeCandidate', () => { expect(selected.candidate).toMatchObject({ status: 'rejected' }) expect(selected.evaluation).toMatchObject({ passed: false }) - expect(() => knowledgeImprovementCandidateRef(selected)).toThrow(/not ready for promotion/) + expect(() => knowledgeImprovementCandidateRef(selected)).toThrow(/is not ready/) await expect(readFile(join(root, 'knowledge', 'claim.md'), 'utf8')).rejects.toMatchObject({ code: 'ENOENT', }) @@ -224,4 +224,4 @@ describe('improveSelectedKnowledgeCandidate', () => { ) }) }) -}) \ No newline at end of file +})