diff --git a/src/kb-improvement.ts b/src/kb-improvement.ts index a5fe46a..da26469 100644 --- a/src/kb-improvement.ts +++ b/src/kb-improvement.ts @@ -40,6 +40,13 @@ export type { } from './kb-improvement/optimization' export { optimizeKnowledgeBasePolicy } from './kb-improvement/optimization' export { improveKnowledgeBase } from './kb-improvement/run' +export type { + ImproveSelectedKnowledgeCandidateOptions, + ImproveSelectedKnowledgeCandidateResult, + KnowledgeEvaluationPhase, + MeasuredKnowledgeSelectionReceipt, +} from './kb-improvement/selected-candidate' +export { improveSelectedKnowledgeCandidate } from './kb-improvement/selected-candidate' export type { KnowledgeImprovementEvent } from './kb-improvement/state' export { knowledgeImprovementRunDir, diff --git a/src/kb-improvement/selected-candidate.ts b/src/kb-improvement/selected-candidate.ts new file mode 100644 index 0000000..934d799 --- /dev/null +++ b/src/kb-improvement/selected-candidate.ts @@ -0,0 +1,483 @@ +import { createHash } from 'node:crypto' +import { join } from 'node:path' +import { canonicalJson, contentHash } from '@tangle-network/agent-eval' +import type { AgentCandidateJsonValue as JsonValue } from '@tangle-network/agent-interface' +import { z } from 'zod' +import { isMissingFile, readRegularFileWithinRoot, writeJsonDurableWithinRoot } from '../durable-fs' +import { + applyKnowledgeFileTransaction, + assertKnowledgeMutationPath, + finishKnowledgeFileTransaction, + type KnowledgeFileMutation, + type KnowledgeFileTransaction, + type KnowledgeFileTransactionPlanEntry, + knowledgeFileTransactionPlanHash, + prepareKnowledgeFileTransaction, +} from '../file-transaction' +import { stableId } from '../ids' +import { writeKnowledgeIndex } from '../indexer' +import { withKnowledgeMutation } from '../mutation-lock' +import type { RagKnowledgeImprovementPhase } from '../rag-improvement-loop' +import type { + KnowledgeImprovementCandidateRef, + KnowledgeImprovementOptions, + KnowledgeImprovementResult, +} from './contracts' +import { + EVALUATION_PHASES, + immutableRefSchema, + KnowledgeImprovementCandidateRefSchema, + safePathSegmentSchema, +} from './contracts' +import { improveKnowledgeBase } from './run' +import { knowledgeImprovementRunDir } from './state' +import { knowledgeFilePlanEntries } from './transition' +import { hashKnowledgeBase, withKnowledgeImprovementComparison } from './workspace' + +const DERIVED_KNOWLEDGE_PATHS = new Set(['knowledge/index.md']) +const selectionPathSchema = z + .string() + .min(1) + .transform((path, context) => { + try { + return assertKnowledgeMutationPath(path) + } catch (error) { + context.addIssue({ + code: 'custom', + message: error instanceof Error ? error.message : String(error), + }) + return z.NEVER + } + }) +const selectionMetadataSchema = z.record(z.string(), z.json()) +const measuredSelectionLifecycleSchema = z + .object({ + kind: z.literal('measured-knowledge-change-selection'), + version: z.literal(1), + selectionDigest: z.string().regex(/^[a-f0-9]{64}$/), + sourceCandidateId: z.string().min(1), + sourceEvidenceHash: z.string().regex(/^[a-f0-9]{64}$/), + sourcePlanHash: z.string().regex(/^[a-f0-9]{64}$/), + selectedPaths: z.array(selectionPathSchema), + selectedCandidateHash: z.string().regex(/^[a-f0-9]{64}$/), + selectedPlanHash: z.string().regex(/^[a-f0-9]{64}$/), + }) + .strict() +const measuredSelectionReceiptSchema = z + .object({ + kind: z.literal('measured-knowledge-change-selection-receipt'), + version: z.literal(1), + receiptHash: z.string().regex(/^[a-f0-9]{64}$/), + selectionDigest: z.string().regex(/^[a-f0-9]{64}$/), + sourceCandidate: KnowledgeImprovementCandidateRefSchema, + sourcePlanHash: z.string().regex(/^[a-f0-9]{64}$/), + selectedPaths: z.array(selectionPathSchema), + derivedImplementationRef: immutableRefSchema, + runId: z.string().min(1), + derivedCandidateId: z.string().min(1), + derivedCandidateStatus: z.string().min(1), + selectedCandidateHash: z.string().regex(/^[a-f0-9]{64}$/), + selectedPlanHash: z.string().regex(/^[a-f0-9]{64}$/), + selectedEvidenceHash: z.string().regex(/^[a-f0-9]{64}$/), + rationale: z.string().trim().min(1).optional(), + metadata: selectionMetadataSchema.optional(), + }) + .strict() + +export type KnowledgeEvaluationPhase = Exclude< + RagKnowledgeImprovementPhase, + 'knowledge-acquisition' | 'knowledge-update' +> + +export interface ImproveSelectedKnowledgeCandidateOptions + extends Omit< + KnowledgeImprovementOptions, + | 'root' + | 'goal' + | 'implementationRef' + | 'runId' + | 'step' + | 'knowledgeResearch' + | 'acquireKnowledge' + | 'updateKnowledge' + | 'enabledPhases' + | 'requiredPhases' + > { + root: string + goal: string + /** Identity of the helper, policy, or human procedure choosing the subset. */ + implementationRef: string + /** Previously measured whole candidate from which the subset is derived. */ + sourceCandidate: KnowledgeImprovementCandidateRef + /** Exact changed file paths to carry into the derived candidate. */ + selectedPaths: readonly string[] + /** Optional explicit run identity. The default includes the selection digest. */ + runId?: string + /** Human-readable reason for selecting this subset. */ + rationale?: string + /** JSON-safe policy output retained in the selection receipt. */ + selectionMetadata?: Record + /** Evaluation-only phases. `knowledge-update` is inserted by this helper. */ + enabledEvaluationPhases?: readonly KnowledgeEvaluationPhase[] + /** Evaluation-only phases that must complete. `knowledge-update` is always required. */ + requiredEvaluationPhases?: readonly KnowledgeEvaluationPhase[] +} + +export type MeasuredKnowledgeSelectionReceipt = z.infer + +export interface ImproveSelectedKnowledgeCandidateResult extends KnowledgeImprovementResult { + selection: MeasuredKnowledgeSelectionReceipt +} + +/** + * Derive a subset from an already measured candidate, then measure the subset as + * a new candidate before it can be promoted. + * + * Directly filtering an activation plan is unsafe: a whole candidate can pass + * while one page subset breaks links, removes supporting sources, or changes a + * readiness score. This helper instead applies the chosen changed paths to an + * isolated baseline, recomputes the index, runs the ordinary improvement + * evaluator, and returns the ordinary candidate reference. Promotion therefore + * remains one path and can never admit an unmeasured hybrid. + */ +export async function improveSelectedKnowledgeCandidate( + options: ImproveSelectedKnowledgeCandidateOptions, +): Promise { + const sourceCandidate = Object.freeze( + KnowledgeImprovementCandidateRefSchema.parse(options.sourceCandidate), + ) + const requestedImplementationRef = immutableRefSchema.parse(options.implementationRef) + const enabledEvaluationPhases = normalizeEvaluationPhases( + options.enabledEvaluationPhases ?? EVALUATION_PHASES, + 'enabledEvaluationPhases', + ) + const requiredEvaluationPhases = normalizeEvaluationPhases( + options.requiredEvaluationPhases ?? [], + 'requiredEvaluationPhases', + ) + const metadata = options.selectionMetadata + ? selectionMetadataSchema.parse(structuredClone(options.selectionMetadata)) + : undefined + + return withKnowledgeImprovementComparison( + { root: options.root, candidate: sourceCandidate }, + async (source) => { + const sourcePlan = await knowledgeFilePlanEntries(source.baseline.root, source.candidate.root) + const sourcePlanHash = knowledgeFileTransactionPlanHash(sourcePlan) + if (sourcePlanHash !== sourceCandidate.promotionPlanHash) { + throw new Error('source knowledge candidate plan no longer matches its measured identity') + } + const changedSourcePlan = sourcePlan.filter(planEntryChanged).filter(notDerivedPath) + const selectedPaths = normalizeSelectedPaths(options.selectedPaths, changedSourcePlan) + const selectionMaterial = immutableJson({ + kind: 'measured-knowledge-change-selection' as const, + version: 1 as const, + sourceCandidate, + sourcePlanHash, + selectedPaths, + ...(options.rationale?.trim() ? { rationale: options.rationale.trim() } : {}), + ...(metadata ? { metadata } : {}), + }) + const selectionDigest = contentHash(selectionMaterial) + const derivedImplementationRef = immutableRefSchema.parse( + `sha256:${contentHash({ + engine: 'agent-knowledge/measured-selection-v1', + implementationRef: requestedImplementationRef, + selectionDigest, + })}`, + ) + const runId = + options.runId ?? + stableId( + 'kimpsel', + canonicalJson({ + root: options.root, + goal: options.goal, + derivedImplementationRef, + selectionDigest, + }), + ) + const selectedEntries = selectedPaths.map((path) => { + const entry = changedSourcePlan.find((candidate) => candidate.path === path) + if (!entry) throw new Error(`selected knowledge path disappeared: ${path}`) + return entry + }) + const selectionMutationPlanHash = knowledgeFileTransactionPlanHash(selectedEntries) + let lifecycleSelection: z.infer | undefined + + const result = await improveKnowledgeBase({ + ...improvementOptions(options), + root: options.root, + goal: options.goal, + implementationRef: derivedImplementationRef, + runId, + enabledPhases: ['knowledge-update', ...enabledEvaluationPhases], + requiredPhases: ['knowledge-update', ...requiredEvaluationPhases], + async updateKnowledge(input) { + const purpose = `knowledge-selected-candidate:${selectionDigest}` + const recoveryOwner = `knowledge-selected-candidate:${sourceCandidate.candidateId}` + await withKnowledgeMutation( + input.candidateRoot, + async (lock) => { + lock.assertOwned() + if (!lock.recovery && selectedEntries.length > 0) { + const transaction = await prepareKnowledgeFileTransaction({ + root: input.candidateRoot, + transactionRoot: lock.transactionRoot, + purpose, + recoveryOwner, + mutations: await selectionMutations(source.candidate.root, selectedEntries), + now: options.now, + }) + if (transaction) { + assertSelectionTransaction(transaction, selectionMutationPlanHash) + await applyKnowledgeFileTransaction({ + root: input.candidateRoot, + transactionRoot: lock.transactionRoot, + transaction, + beforeCommit: lock.assertOwned, + }) + await finishKnowledgeFileTransaction({ + root: input.candidateRoot, + transactionRoot: lock.transactionRoot, + transaction, + assertOwned: lock.assertOwned, + }) + } + } + await writeKnowledgeIndex(input.candidateRoot) + lock.assertOwned() + }, + { + resumeTransaction: { + purpose, + recoveryOwner, + validate: (transaction) => + assertSelectionTransaction(transaction, selectionMutationPlanHash), + }, + }, + ) + + const selectedPlan = await knowledgeFilePlanEntries( + source.baseline.root, + input.candidateRoot, + ) + assertExactSelectedChanges(selectedPlan, selectedPaths) + const selectedPlanHash = knowledgeFileTransactionPlanHash(selectedPlan) + const selectedCandidateHash = await hashKnowledgeBase(input.candidateRoot) + lifecycleSelection = measuredSelectionLifecycleSchema.parse({ + kind: 'measured-knowledge-change-selection', + version: 1, + selectionDigest, + sourceCandidateId: sourceCandidate.candidateId, + sourceEvidenceHash: sourceCandidate.evidenceHash, + sourcePlanHash, + selectedPaths, + selectedCandidateHash, + selectedPlanHash, + }) + return { + applied: selectedPaths.length > 0, + summary: + selectedPaths.length > 0 + ? `Applied ${selectedPaths.length} selected measured change(s).` + : 'Selected no changes; measuring the exact baseline as a null candidate.', + metadata: { selection: lifecycleSelection }, + } + }, + }) + + const candidate = result.candidate + if (!candidate?.candidateHash || !candidate.promotionPlanHash || !candidate.evidenceHash) { + throw new Error('selected knowledge candidate did not produce measured candidate evidence') + } + const recordedSelection = measuredSelectionLifecycleSchema.parse( + result.lifecycle?.knowledgeUpdate?.metadata?.selection ?? lifecycleSelection, + ) + if ( + recordedSelection.selectionDigest !== selectionDigest || + recordedSelection.selectedCandidateHash !== candidate.candidateHash || + recordedSelection.selectedPlanHash !== candidate.promotionPlanHash || + canonicalJson(recordedSelection.selectedPaths) !== canonicalJson(selectedPaths) + ) { + throw new Error('selected knowledge candidate evidence does not bind the measured subset') + } + + const receiptWithoutHash = immutableJson({ + kind: 'measured-knowledge-change-selection-receipt' as const, + version: 1 as const, + selectionDigest, + sourceCandidate, + sourcePlanHash, + selectedPaths, + derivedImplementationRef, + runId: result.runId, + derivedCandidateId: candidate.candidateId, + derivedCandidateStatus: candidate.status, + selectedCandidateHash: candidate.candidateHash, + selectedPlanHash: candidate.promotionPlanHash, + selectedEvidenceHash: candidate.evidenceHash, + ...(options.rationale?.trim() ? { rationale: options.rationale.trim() } : {}), + ...(metadata ? { metadata } : {}), + }) + const receipt = measuredSelectionReceiptSchema.parse({ + ...receiptWithoutHash, + receiptHash: contentHash(receiptWithoutHash), + }) + await persistSelectionReceipt(options.root, receipt) + return { ...result, selection: receipt } + }, + ) +} + +function improvementOptions( + options: ImproveSelectedKnowledgeCandidateOptions, +): Omit< + KnowledgeImprovementOptions, + | 'root' + | 'goal' + | 'implementationRef' + | 'runId' + | 'step' + | 'knowledgeResearch' + | 'acquireKnowledge' + | 'updateKnowledge' + | 'enabledPhases' + | 'requiredPhases' +> { + const { + root: _root, + goal: _goal, + implementationRef: _implementationRef, + sourceCandidate: _sourceCandidate, + selectedPaths: _selectedPaths, + runId: _runId, + rationale: _rationale, + selectionMetadata: _selectionMetadata, + enabledEvaluationPhases: _enabledEvaluationPhases, + requiredEvaluationPhases: _requiredEvaluationPhases, + ...rest + } = options + return rest +} + +function normalizeEvaluationPhases( + phases: readonly RagKnowledgeImprovementPhase[], + field: string, +): KnowledgeEvaluationPhase[] { + const unique = [...new Set(phases)] + for (const phase of unique) { + if (phase === 'knowledge-acquisition' || phase === 'knowledge-update') { + throw new Error(`${field} cannot contain the selection-owned phase '${phase}'`) + } + } + return unique as KnowledgeEvaluationPhase[] +} + +function normalizeSelectedPaths( + paths: readonly string[], + changedPlan: readonly KnowledgeFileTransactionPlanEntry[], +): string[] { + const available = new Set(changedPlan.map((entry) => entry.path)) + const selected: string[] = [] + const seen = new Set() + for (const input of paths) { + const path = selectionPathSchema.parse(input) + if (DERIVED_KNOWLEDGE_PATHS.has(path)) { + throw new Error(`derived knowledge path cannot be selected directly: ${path}`) + } + if (seen.has(path)) throw new Error(`selected knowledge path is repeated: ${path}`) + if (!available.has(path)) { + throw new Error(`selected knowledge path is not a changed source-candidate file: ${path}`) + } + seen.add(path) + selected.push(path) + } + return selected.sort((left, right) => left.localeCompare(right)) +} + +function planEntryChanged(entry: KnowledgeFileTransactionPlanEntry): boolean { + return ( + entry.beforeHash !== entry.afterHash || + (entry.beforeHash !== null && entry.afterHash !== null && entry.beforeMode !== entry.afterMode) + ) +} + +function notDerivedPath(entry: KnowledgeFileTransactionPlanEntry): boolean { + return !DERIVED_KNOWLEDGE_PATHS.has(entry.path) +} + +async function selectionMutations( + sourceCandidateRoot: string, + entries: readonly KnowledgeFileTransactionPlanEntry[], +): Promise { + return Promise.all( + entries.map(async (entry) => { + if (entry.afterHash === null) return { path: entry.path, content: null } + const file = await readRegularFileWithinRoot(sourceCandidateRoot, entry.path) + const actualHash = createHash('sha256').update(file.bytes).digest('hex') + if (actualHash !== entry.afterHash || file.mode !== entry.afterMode) { + throw new Error(`source candidate changed before subset materialization: ${entry.path}`) + } + return { path: entry.path, content: file.bytes, mode: file.mode } + }), + ) +} + +function assertSelectionTransaction( + transaction: KnowledgeFileTransaction, + expectedPlanHash: string, +): void { + if (knowledgeFileTransactionPlanHash(transaction.entries) !== expectedPlanHash) { + throw new Error('selected knowledge transaction does not match its approved path set') + } +} + +function assertExactSelectedChanges( + plan: readonly KnowledgeFileTransactionPlanEntry[], + selectedPaths: readonly string[], +): void { + const actual = plan + .filter(planEntryChanged) + .filter(notDerivedPath) + .map((entry) => entry.path) + .sort((left, right) => left.localeCompare(right)) + if (canonicalJson(actual) !== canonicalJson(selectedPaths)) { + throw new Error( + `selected knowledge candidate changed the wrong files: expected ${selectedPaths.join(', ') || '(none)'}, got ${actual.join(', ') || '(none)'}`, + ) + } +} + +async function persistSelectionReceipt( + root: string, + receipt: MeasuredKnowledgeSelectionReceipt, +): Promise { + const runDir = knowledgeImprovementRunDir(root, receipt.runId) + const relativePath = join( + 'candidates', + safePathSegmentSchema.parse(receipt.derivedCandidateId), + 'selection.json', + ).replace(/\\/g, '/') + try { + const existing = measuredSelectionReceiptSchema.parse( + JSON.parse((await readRegularFileWithinRoot(runDir, relativePath)).bytes.toString('utf8')), + ) + if (canonicalJson(existing) !== canonicalJson(receipt)) { + throw new Error('measured knowledge selection receipt conflicts with durable content') + } + return + } catch (error) { + if (!isMissingFile(error)) throw error + } + await writeJsonDurableWithinRoot(runDir, relativePath, receipt) +} + +function immutableJson(value: T): T { + if (value === null || typeof value !== 'object') return value + const children: readonly unknown[] = Array.isArray(value) + ? value + : Object.values(value as Record) + for (const child of children) immutableJson(child) + return Object.freeze(value) +} diff --git a/tests/kb-improvement/selected-candidate.test.ts b/tests/kb-improvement/selected-candidate.test.ts new file mode 100644 index 0000000..f32be67 --- /dev/null +++ b/tests/kb-improvement/selected-candidate.test.ts @@ -0,0 +1,227 @@ +import { readFile, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { + hashKnowledgeBase, + improveSelectedKnowledgeCandidate, + knowledgeImprovementCandidateRef, + knowledgeImprovementRunDir, + promoteKnowledgeCandidate, + withKnowledgeImprovementComparison, +} from '../../src/index' +import { + improveTestKnowledgeBase as improveKnowledgeBase, + passingMetric, + TEST_KNOWLEDGE_IMPLEMENTATION_REF, + withKb, +} from '../support/kb-improvement' + +describe('improveSelectedKnowledgeCandidate', () => { + it('remeasures the exact selected subset before ordinary promotion', async () => { + await withKb(async (root) => { + await writeFile(join(root, 'knowledge', 'original.md'), '# Original\n') + const baseHash = await hashKnowledgeBase(root) + const source = await improveKnowledgeBase({ + root, + goal: 'Generate a broad candidate from which a measured subset will be selected', + runId: 'broad-source-candidate', + updateKnowledge: async ({ candidateRoot }) => { + await writeFile(join(candidateRoot, 'knowledge', 'keep.md'), '# Keep\n') + await writeFile(join(candidateRoot, 'knowledge', 'drop.md'), '# Drop\n') + await writeFile(join(candidateRoot, 'knowledge', 'original.md'), '# Changed\n') + return { applied: true, summary: 'created a three-file broad candidate' } + }, + evaluate: passingMetric, + }) + const sourceCandidate = knowledgeImprovementCandidateRef(source) + let evaluatedSelectedSnapshot = false + + const selected = await improveSelectedKnowledgeCandidate({ + root, + goal: 'Measure only the useful files from the broad candidate', + implementationRef: TEST_KNOWLEDGE_IMPLEMENTATION_REF, + sourceCandidate, + selectedPaths: ['knowledge/original.md', 'knowledge/keep.md'], + rationale: 'The dropped page is redundant with an existing source.', + selectionMetadata: { reviewer: 'test-reviewer', policyVersion: 1 }, + async evaluate(input) { + expect(input.candidateRoot).not.toBe(input.baselineRoot) + await expect( + readFile(join(input.candidateRoot, 'knowledge', 'keep.md'), 'utf8'), + ).resolves.toBe('# Keep\n') + await expect( + readFile(join(input.candidateRoot, 'knowledge', 'original.md'), 'utf8'), + ).resolves.toBe('# Changed\n') + await expect( + readFile(join(input.candidateRoot, 'knowledge', 'drop.md'), 'utf8'), + ).rejects.toMatchObject({ code: 'ENOENT' }) + evaluatedSelectedSnapshot = true + return passingMetric() + }, + }) + const candidate = knowledgeImprovementCandidateRef(selected) + + expect(evaluatedSelectedSnapshot).toBe(true) + expect(candidate.baseHash).toBe(baseHash) + expect(selected.selection).toMatchObject({ + kind: 'measured-knowledge-change-selection-receipt', + sourceCandidate, + selectedPaths: ['knowledge/keep.md', 'knowledge/original.md'], + selectedCandidateHash: candidate.candidateHash, + selectedPlanHash: candidate.promotionPlanHash, + selectedEvidenceHash: candidate.evidenceHash, + }) + + await withKnowledgeImprovementComparison({ root, candidate }, async (comparison) => { + await expect( + readFile(join(comparison.candidate.root, 'knowledge', 'keep.md'), 'utf8'), + ).resolves.toBe('# Keep\n') + await expect( + readFile(join(comparison.candidate.root, 'knowledge', 'drop.md'), 'utf8'), + ).rejects.toMatchObject({ code: 'ENOENT' }) + }) + + const promoted = await promoteKnowledgeCandidate({ root, candidate }) + expect(promoted).toMatchObject({ promoted: true, blocked: false }) + await expect(readFile(join(root, 'knowledge', 'keep.md'), 'utf8')).resolves.toBe('# Keep\n') + await expect(readFile(join(root, 'knowledge', 'original.md'), 'utf8')).resolves.toBe( + '# Changed\n', + ) + await expect(readFile(join(root, 'knowledge', 'drop.md'), 'utf8')).rejects.toMatchObject({ + code: 'ENOENT', + }) + + const receiptPath = join( + knowledgeImprovementRunDir(root, candidate.runId), + 'candidates', + candidate.candidateId, + 'selection.json', + ) + const storedReceipt = JSON.parse(await readFile(receiptPath, 'utf8')) + expect(storedReceipt).toEqual(selected.selection) + }) + }) + + it('makes a harmful subset fail its own evaluator instead of inheriting the whole-candidate pass', async () => { + await withKb(async (root) => { + const source = await improveKnowledgeBase({ + root, + goal: 'Create a whole candidate whose two pages work together', + runId: 'whole-pair-source', + updateKnowledge: async ({ candidateRoot }) => { + await writeFile(join(candidateRoot, 'knowledge', 'claim.md'), '# Claim\nUses support.\n') + await writeFile(join(candidateRoot, 'knowledge', 'support.md'), '# Support\nEvidence.\n') + return { applied: true, summary: 'created claim and support pages' } + }, + evaluate: passingMetric, + }) + const sourceCandidate = knowledgeImprovementCandidateRef(source) + + const selected = await improveSelectedKnowledgeCandidate({ + root, + goal: 'Test whether the claim page survives without its support page', + implementationRef: TEST_KNOWLEDGE_IMPLEMENTATION_REF, + sourceCandidate, + selectedPaths: ['knowledge/claim.md'], + async evaluate({ candidateRoot }) { + try { + await readFile(join(candidateRoot, 'knowledge', 'support.md'), 'utf8') + return passingMetric() + } catch { + return { + score: 0, + passed: false, + notes: 'claim is missing its supporting page', + provenance: { + evaluator: 'selected-candidate-dependency-check', + version: '1', + method: 'deterministic', + }, + } + } + }, + }) + + expect(selected.candidate).toMatchObject({ status: 'rejected' }) + expect(selected.evaluation).toMatchObject({ passed: false }) + expect(() => knowledgeImprovementCandidateRef(selected)).toThrow(/is not ready/) + await expect(readFile(join(root, 'knowledge', 'claim.md'), 'utf8')).rejects.toMatchObject({ + code: 'ENOENT', + }) + }) + }) + + it('rejects unknown, repeated, and generated paths before opening a derived run', async () => { + await withKb(async (root) => { + const source = await improveKnowledgeBase({ + root, + goal: 'Create one selectable page', + runId: 'selection-shape-source', + updateKnowledge: async ({ candidateRoot }) => { + await writeFile(join(candidateRoot, 'knowledge', 'page.md'), '# Page\n') + return { applied: true, summary: 'created one page' } + }, + evaluate: passingMetric, + }) + const sourceCandidate = knowledgeImprovementCandidateRef(source) + const common = { + root, + goal: 'Refuse a malformed selection', + implementationRef: TEST_KNOWLEDGE_IMPLEMENTATION_REF, + sourceCandidate, + evaluate: passingMetric, + } + + await expect( + improveSelectedKnowledgeCandidate({ ...common, selectedPaths: ['knowledge/missing.md'] }), + ).rejects.toThrow(/not a changed source-candidate file/) + await expect( + improveSelectedKnowledgeCandidate({ + ...common, + selectedPaths: ['knowledge/page.md', 'knowledge/page.md'], + }), + ).rejects.toThrow(/repeated/) + await expect( + improveSelectedKnowledgeCandidate({ ...common, selectedPaths: ['knowledge/index.md'] }), + ).rejects.toThrow(/derived knowledge path/) + }) + }) + + it('reopens the same measured selection without rerunning its evaluator', async () => { + await withKb(async (root) => { + const source = await improveKnowledgeBase({ + root, + goal: 'Create a resumable source candidate', + runId: 'selected-resume-source', + updateKnowledge: async ({ candidateRoot }) => { + await writeFile(join(candidateRoot, 'knowledge', 'selected.md'), '# Selected\n') + return { applied: true, summary: 'created selected page' } + }, + evaluate: passingMetric, + }) + let evaluationCalls = 0 + const options = { + root, + goal: 'Measure a stable selected candidate', + runId: 'selected-resume-derived', + implementationRef: TEST_KNOWLEDGE_IMPLEMENTATION_REF, + sourceCandidate: knowledgeImprovementCandidateRef(source), + selectedPaths: ['knowledge/selected.md'], + evaluate() { + evaluationCalls += 1 + return passingMetric() + }, + } + + const first = await improveSelectedKnowledgeCandidate(options) + expect(evaluationCalls).toBe(1) + const second = await improveSelectedKnowledgeCandidate(options) + + expect(evaluationCalls).toBe(1) + expect(second.selection).toEqual(first.selection) + expect(knowledgeImprovementCandidateRef(second)).toEqual( + knowledgeImprovementCandidateRef(first), + ) + }) + }) +})