From 127919575ffea138efbc6721c7801ede222068cb Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 13:36:37 -0700 Subject: [PATCH 01/24] ci: inspect candidate lineage contract --- .../inspect-authored-profile-contract.yml | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/inspect-authored-profile-contract.yml diff --git a/.github/workflows/inspect-authored-profile-contract.yml b/.github/workflows/inspect-authored-profile-contract.yml new file mode 100644 index 00000000..c4425f7c --- /dev/null +++ b/.github/workflows/inspect-authored-profile-contract.yml @@ -0,0 +1,28 @@ +name: Inspect authored profile contract + +on: + push: + branches: + - feat/authored-profile-candidate + +permissions: + contents: read + +jobs: + inspect: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + - uses: pnpm/action-setup@v6 + - uses: actions/setup-node@v7 + with: + node-version: 22 + cache: pnpm + - run: pnpm install --frozen-lockfile + - name: Print candidate-lineage declarations + shell: bash + run: | + set -euo pipefail + grep -R -n -C 8 "AgentCandidateLineage\|agentCandidateLineageSchema" node_modules/@tangle-network/agent-interface/dist | head -n 400 || true + echo "--- source enum literals ---" + grep -R -n -C 4 "source.*optimizer\|optimizer.*source" node_modules/@tangle-network/agent-interface/dist | head -n 400 || true From dc424c1ffd96ab5ac484581073a537b7218bd70f Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 13:46:11 -0700 Subject: [PATCH 02/24] ci: inspect current profile improvement implementation --- .../inspect-authored-profile-contract.yml | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/.github/workflows/inspect-authored-profile-contract.yml b/.github/workflows/inspect-authored-profile-contract.yml index c4425f7c..034ada5a 100644 --- a/.github/workflows/inspect-authored-profile-contract.yml +++ b/.github/workflows/inspect-authored-profile-contract.yml @@ -19,10 +19,27 @@ jobs: node-version: 22 cache: pnpm - run: pnpm install --frozen-lockfile - - name: Print candidate-lineage declarations + - name: Print current profile improvement seams shell: bash run: | set -euo pipefail - grep -R -n -C 8 "AgentCandidateLineage\|agentCandidateLineageSchema" node_modules/@tangle-network/agent-interface/dist | head -n 400 || true - echo "--- source enum literals ---" - grep -R -n -C 4 "source.*optimizer\|optimizer.*source" node_modules/@tangle-network/agent-interface/dist | head -n 400 || true + file=src/intelligence/improvement-cycle.ts + grep -n "ProposeAgentProfileImprovementOptions\|proposeAgentProfileImprovement\|sealProfileImprovementBenchmark\|profileImprovementMetadata\|assertProfileReleaseWorkIsFresh\|createProfileImprovementCostLedger\|profilePolicyWithBudget\|profilePreparationAccounting" "$file" + for pattern in \ + "export interface ProposeAgentProfileImprovementOptions" \ + "export interface ProposeAgentProfileImprovementResult" \ + "function createProfileImprovementCostLedger" \ + "function sealProfileImprovementBenchmark" \ + "function assertProfileReleaseWorkIsFresh" \ + "function profileImprovementMetadata" \ + "export async function proposeAgentProfileImprovement"; do + line=$(grep -n -m1 "$pattern" "$file" | cut -d: -f1) + start=$((line > 25 ? line - 25 : 1)) + end=$((line + 180)) + echo "--- $pattern ($start-$end) ---" + nl -ba "$file" | sed -n "${start},${end}p" + done + echo "--- current intelligence exports ---" + grep -n -C 4 "ProposeAgentProfileImprovement\|proposeAgentProfileImprovement" src/intelligence/index.ts + echo "--- exact candidate lineage ---" + grep -R -n -C 10 "interface AgentCandidateLineage\|type AgentCandidateLineage\|agentCandidateLineageSchema" node_modules/@tangle-network/agent-interface/dist | head -n 500 || true From 2edc63a93ab9afa05d7324f8cb88b45a95dd3e96 Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 13:47:50 -0700 Subject: [PATCH 03/24] ci: inspect authored profile test fixtures --- .../inspect-authored-profile-contract.yml | 34 ++++++++----------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/.github/workflows/inspect-authored-profile-contract.yml b/.github/workflows/inspect-authored-profile-contract.yml index 034ada5a..f7210fb1 100644 --- a/.github/workflows/inspect-authored-profile-contract.yml +++ b/.github/workflows/inspect-authored-profile-contract.yml @@ -19,27 +19,21 @@ jobs: node-version: 22 cache: pnpm - run: pnpm install --frozen-lockfile - - name: Print current profile improvement seams + - name: Print authored profile fixtures shell: bash run: | set -euo pipefail - file=src/intelligence/improvement-cycle.ts - grep -n "ProposeAgentProfileImprovementOptions\|proposeAgentProfileImprovement\|sealProfileImprovementBenchmark\|profileImprovementMetadata\|assertProfileReleaseWorkIsFresh\|createProfileImprovementCostLedger\|profilePolicyWithBudget\|profilePreparationAccounting" "$file" - for pattern in \ - "export interface ProposeAgentProfileImprovementOptions" \ - "export interface ProposeAgentProfileImprovementResult" \ - "function createProfileImprovementCostLedger" \ - "function sealProfileImprovementBenchmark" \ - "function assertProfileReleaseWorkIsFresh" \ - "function profileImprovementMetadata" \ - "export async function proposeAgentProfileImprovement"; do - line=$(grep -n -m1 "$pattern" "$file" | cut -d: -f1) - start=$((line > 25 ? line - 25 : 1)) - end=$((line + 180)) - echo "--- $pattern ($start-$end) ---" - nl -ba "$file" | sed -n "${start},${end}p" + test_file=tests/improvement-cycle.test.ts + line=$(grep -n -m1 "measures and activates a directly authored complete profile without improve" "$test_file" | cut -d: -f1) + start=$((line > 100 ? line - 100 : 1)) + end=$((line + 360)) + echo "--- authored profile test ($start-$end) ---" + nl -ba "$test_file" | sed -n "${start},${end}p" + echo "--- helper declarations used by the test ---" + for pattern in "const baselineProfile" "const profileBenchmark" "function profile" "const profileSource" "stateDigest" "profileExecutionRef"; do + grep -n -m5 -C 8 "$pattern" "$test_file" || true done - echo "--- current intelligence exports ---" - grep -n -C 4 "ProposeAgentProfileImprovement\|proposeAgentProfileImprovement" src/intelligence/index.ts - echo "--- exact candidate lineage ---" - grep -R -n -C 10 "interface AgentCandidateLineage\|type AgentCandidateLineage\|agentCandidateLineageSchema" node_modules/@tangle-network/agent-interface/dist | head -n 500 || true + echo "--- profile schema lineage refinements ---" + nl -ba node_modules/@tangle-network/agent-interface/dist/agent-profile-improvement-schema.js | sed -n '160,245p' + echo "--- candidate lineage refinements ---" + nl -ba node_modules/@tangle-network/agent-interface/dist/agent-candidate-lineage-schema.js | sed -n '33,90p' From d68cddf399e2a1ac663552c2bb6253f2c61f4e4a Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 13:52:46 -0700 Subject: [PATCH 04/24] feat: add canonical authored profile candidate path --- .../authored-profile-improvement.ts | 319 ++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 src/intelligence/authored-profile-improvement.ts diff --git a/src/intelligence/authored-profile-improvement.ts b/src/intelligence/authored-profile-improvement.ts new file mode 100644 index 00000000..a320fa71 --- /dev/null +++ b/src/intelligence/authored-profile-improvement.ts @@ -0,0 +1,319 @@ +import { CostLedger } from '@tangle-network/agent-eval' +import { assertProposalFindings, type ProposalFinding } from '@tangle-network/agent-eval/analyst' +import { + type CampaignScenarioIdentity, + campaignSplitDigestFromIdentities, +} from '@tangle-network/agent-eval/campaign' +import { + type AgentProfileImprovementExperimentExecutionInput, + measuredComparisonFromAgentProfileImprovementExperiment, + runAgentProfileImprovementExperiment, + type Scenario, + sealAgentProfileImprovementExperiment, + sealAgentProfileImprovementSuite, + sealAgentProfileImprovementTask, + verifyAgentProfileImprovementExperimentComparison, +} from '@tangle-network/agent-eval/contract' +import type { + AgentCandidateLineage, + AgentImprovementProposal, + AgentImprovementSource, + AgentProfile, + AgentProfileImprovementExecutionRef, + AgentProfileImprovementExperiment, + AgentProfileImprovementMeasuredComparison, + AgentProfileImprovementMeasurement, + AgentProfileImprovementRunReceipt, + AgentProfileImprovementSuiteInputs, + AgentProfileImprovementTask, + Sha256Digest, +} from '@tangle-network/agent-interface' +import { + AGENT_IMPROVEMENT_SOURCE_METADATA_KEY, + agentImprovementSourceMetadata, + agentImprovementSourceSchema, + agentProfileImprovementArmSchema, + numbersApproximatelyEqual, +} from '@tangle-network/agent-interface' +import { canonicalCandidateDigest, immutableCandidateValue } from '../candidate-execution/digest' +import { parseExactAgentProfile } from '../candidate-execution/profile' +import { + type AgentProfileImprovementBenchmark, + createAgentImprovementProposal, +} from './improvement-cycle' +import { agentImprovementProfileDiffs } from './improvement-surfaces' +import { assertNoCallerOptimizationReceipt } from './optimization-receipt' +import type { AgentImprovementProfileStateDigest } from './profile-activation' + +/** Lineage accepted by the direct candidate path. Optimizer lineage belongs to `improve()`. */ +export type AuthoredAgentProfileCandidateLineage = Omit< + AgentCandidateLineage, + 'source' | 'profileDiffIds' +> & { + source: Exclude + /** Runtime derives these from the exact profile change it seals. */ + profileDiffIds?: never +} + +/** Provenance attached while Runtime derives the exact profile diff. */ +export type AuthoredAgentProfileDiffOptions = NonNullable< + Parameters[2] +> + +/** Product-owned executor for exact baseline/candidate profile measurement. */ +export interface AgentProfileCandidateMeasurementExecutor { + executionRef: AgentProfileImprovementExecutionRef + measure( + input: AgentProfileImprovementExperimentExecutionInput & { profile: AgentProfile }, + ): Promise +} + +/** + * Measure a complete human-authored, imported, or compound profile candidate. + * No optimizer runs and no optimizer receipt is fabricated. + */ +export interface ProposeAuthoredAgentProfileImprovementOptions { + runId: string + source: AgentImprovementSource + profile: AgentProfile + stateDigest: AgentImprovementProfileStateDigest + candidateProfile: AgentProfile + candidateLineage: AuthoredAgentProfileCandidateLineage + /** Optional source/artifact metadata used on Runtime-derived profile diff steps. */ + diff?: AuthoredAgentProfileDiffOptions + findings?: readonly ProposalFinding[] + benchmark: AgentProfileImprovementBenchmark + executor: AgentProfileCandidateMeasurementExecutor + /** One customer-approved maximum for the held-out paired measurement. */ + budgetUsd: number + /** Optional identities used to prove authored/imported development work is held out. */ + developmentScenarios?: readonly CampaignScenarioIdentity[] + maxConcurrency?: number + signal?: AbortSignal + candidate?: AgentProfileImprovementMeasuredComparison['candidate'] + metadata?: AgentProfileImprovementMeasuredComparison['metadata'] + now?: () => Date +} + +export interface ProposeAuthoredAgentProfileImprovementResult { + candidateProfile: AgentProfile + candidateLineage: AgentCandidateLineage + experiment: AgentProfileImprovementExperiment + measurements: AgentProfileImprovementMeasurement[] + proposal: AgentImprovementProposal +} + +/** + * Put a complete authored/imported profile through the canonical profile + * experiment and proposal path without invoking `improve()`. + */ +export async function proposeAuthoredAgentProfileImprovement( + options: ProposeAuthoredAgentProfileImprovementOptions, +): Promise { + assertNoCallerOptimizationReceipt(options.metadata) + const source = agentImprovementSourceSchema.parse(options.source) + const inputLineage = options.candidateLineage as AgentCandidateLineage + if (inputLineage.source === 'optimizer') { + throw new Error('authored profile improvement refuses optimizer lineage; use improve()') + } + if (Object.hasOwn(inputLineage, 'profileDiffIds')) { + throw new Error('authored profile improvement derives candidateLineage.profileDiffIds') + } + const findings = immutableCandidateValue([ + ...assertProposalFindings(options.findings ?? [], 'authored profile improvement findings'), + ]) + const costLedger = createMeasurementCostLedger(options.budgetUsd) + const preparationStartedAt = performance.now() + const baselineProfile = parseExactAgentProfile(options.profile, 'authored profile baseline') + const candidateProfile = parseExactAgentProfile( + options.candidateProfile, + 'authored profile candidate', + ) + const baselineStateDigest = profileStateDigest( + options.stateDigest, + source.sourceIdentity, + baselineProfile, + ) + if (baselineStateDigest !== source.sourceDigest) { + throw new Error('authored profile source digest does not match the measured profile state') + } + const candidateStateDigest = profileStateDigest( + options.stateDigest, + source.sourceIdentity, + candidateProfile, + ) + if (candidateStateDigest === baselineStateDigest) { + throw new Error('authored profile candidate state digest matches the baseline') + } + + const change = agentImprovementProfileDiffs(baselineProfile, candidateProfile, { + ...options.diff, + id: options.diff?.id ?? `profile-improvement:${candidateStateDigest}`, + metadata: { + ...options.diff?.metadata, + sourceIdentity: source.sourceIdentity, + sourceRevision: source.sourceRevision, + }, + }) + const profileDiffIds = change.map((step) => { + if (!step.id) throw new Error('authored profile change requires an exact diff id') + return step.id + }) + const candidateLineage = immutableCandidateValue({ + ...inputLineage, + profileDiffIds, + }) + const policy = profilePolicyWithBudget(options.benchmark.policy, options.budgetUsd) + const benchmark = sealProfileImprovementBenchmark({ ...options.benchmark, policy }) + assertDirectCandidateReleaseWorkIsFresh( + benchmark, + candidateLineage, + options.developmentScenarios, + ) + const experiment = sealAgentProfileImprovementExperiment({ + kind: 'agent-profile-improvement-experiment', + digestAlgorithm: 'rfc8785-sha256', + source, + executionRef: options.executor.executionRef, + baseline: { stateDigest: baselineStateDigest }, + candidate: { stateDigest: candidateStateDigest }, + change, + candidateLineage, + benchmark, + policy, + }) + const profilesByStateDigest = new Map([ + [baselineStateDigest, baselineProfile], + [candidateStateDigest, candidateProfile], + ]) + const preparation = { + wallDurationMs: Math.max(0, performance.now() - preparationStartedAt), + cost: { usd: 0, provenance: 'observed' as const }, + } + const run = await runAgentProfileImprovementExperiment({ + experiment, + ...(options.maxConcurrency === undefined ? {} : { maxConcurrency: options.maxConcurrency }), + costLedger, + ...(options.signal ? { signal: options.signal } : {}), + execute: async (input) => { + const measuredProfile = profilesByStateDigest.get(input.stateDigest) + if (!measuredProfile) { + throw new Error('authored profile execution requested an unknown profile state') + } + return options.executor.measure({ ...input, profile: measuredProfile }) + }, + }) + const evaluation = verifyAgentProfileImprovementExperimentComparison( + measuredComparisonFromAgentProfileImprovementExperiment({ + experiment, + measurements: run.measurements, + runId: options.runId, + ...(options.candidate ? { candidate: options.candidate } : {}), + generationsExplored: 0, + preparation, + measurement: run.measurement, + metadata: directProfileImprovementMetadata(options.metadata, source), + }), + ) + const proposal = createAgentImprovementProposal({ + runId: options.runId, + findings, + evaluation, + ...(options.now ? { now: options.now } : {}), + }) + return { + candidateProfile, + candidateLineage, + experiment, + measurements: run.measurements, + proposal, + } +} + +function createMeasurementCostLedger(budgetUsd: number): CostLedger { + if (!Number.isFinite(budgetUsd) || budgetUsd < 0) { + throw new Error('authored profile improvement budgetUsd must be a non-negative finite number') + } + return new CostLedger({ costCeilingUsd: budgetUsd }) +} + +function profilePolicyWithBudget( + policy: AgentProfileImprovementBenchmark['policy'], + budgetUsd: number, +): AgentProfileImprovementBenchmark['policy'] { + if (policy.budgetUsd !== undefined && !numbersApproximatelyEqual(policy.budgetUsd, budgetUsd)) { + throw new Error('authored profile policy budgetUsd must equal the run budgetUsd') + } + return { ...policy, budgetUsd } +} + +function profileStateDigest( + stateDigest: AgentImprovementProfileStateDigest, + identity: string, + profile: AgentProfile, +): Sha256Digest { + return agentProfileImprovementArmSchema.parse({ + stateDigest: stateDigest({ identity, profile }), + }).stateDigest +} + +function sealProfileImprovementBenchmark( + input: AgentProfileImprovementBenchmark, +): AgentProfileImprovementSuiteInputs { + const tasks = input.tasks.map((task) => sealAgentProfileImprovementTask(task)) as [ + AgentProfileImprovementTask, + ...AgentProfileImprovementTask[], + ] + return sealAgentProfileImprovementSuite({ + splitDigest: campaignSplitDigestFromIdentities( + tasks.map(profileTaskScenarioIdentity), + input.reps, + ), + tasks, + reps: input.reps, + seeds: input.seeds, + }) +} + +function profileTaskScenarioIdentity(task: AgentProfileImprovementTask): CampaignScenarioIdentity { + return { + id: task.scenario.id, + kind: task.scenario.kind, + scenarioDigest: task.scenario.digest, + } +} + +function assertDirectCandidateReleaseWorkIsFresh( + benchmark: AgentProfileImprovementSuiteInputs, + lineage: AgentCandidateLineage, + developmentScenarios: readonly CampaignScenarioIdentity[] | undefined, +): void { + if (lineage.developmentSplitDigest === benchmark.suite.splitDigest) { + throw new Error('authored profile development and held-out splits must be disjoint') + } + if (!developmentScenarios || developmentScenarios.length === 0) return + const development = new Set(developmentScenarios.map(canonicalCandidateDigest)) + const reused = benchmark.tasks + .map(profileTaskScenarioIdentity) + .filter((scenario) => development.has(canonicalCandidateDigest(scenario))) + .map((scenario) => scenario.id) + if (reused.length > 0) { + throw new Error(`authored profile release reuses development scenario(s): [${reused.join(', ')}]`) + } +} + +function directProfileImprovementMetadata( + metadata: AgentProfileImprovementMeasuredComparison['metadata'], + source: AgentImprovementSource, +): NonNullable { + assertNoCallerOptimizationReceipt(metadata) + if (metadata && Object.hasOwn(metadata, AGENT_IMPROVEMENT_SOURCE_METADATA_KEY)) { + throw new Error( + `candidate metadata reserves '${AGENT_IMPROVEMENT_SOURCE_METADATA_KEY}' for Runtime`, + ) + } + return immutableCandidateValue({ + ...(metadata ?? {}), + ...agentImprovementSourceMetadata(source), + }) +} From d5affabb6341a5602ca1427f5f3e55162ca18c69 Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 13:53:47 -0700 Subject: [PATCH 05/24] test: prove authored profiles use the canonical proposal path --- tests/authored-profile-improvement.test.ts | 231 +++++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 tests/authored-profile-improvement.test.ts diff --git a/tests/authored-profile-improvement.test.ts b/tests/authored-profile-improvement.test.ts new file mode 100644 index 00000000..db72541f --- /dev/null +++ b/tests/authored-profile-improvement.test.ts @@ -0,0 +1,231 @@ +import { + minimumPairsForPairedDeltaTest, + type ProposalFinding, +} from '@tangle-network/agent-eval' +import type { CampaignScenarioIdentity } from '@tangle-network/agent-eval/campaign' +import type { AgentProfile } from '@tangle-network/agent-interface' +import { describe, expect, it } from 'vitest' + +import { canonicalCandidateDigest } from '../src/candidate-execution/digest' +import { + type ProposeAuthoredAgentProfileImprovementOptions, + proposeAuthoredAgentProfileImprovement, +} from '../src/intelligence/authored-profile-improvement' +import { optimizationActivationReceiptFromMetadata } from '../src/intelligence/optimization-receipt' +import { improvementFinding as fixtureFinding } from './helpers/improvement-method-fixture' +import { + createProfileImprovementFixture, + createProfileImprovementRunReceipt, +} from './helpers/profile-improvement-fixture' + +const minimumPairedRuns = minimumPairsForPairedDeltaTest(0.95) +const { proposal_origin: _fixtureOrigin, ...fixtureAnalystFinding } = + fixtureFinding as ProposalFinding +const productionFinding: ProposalFinding = { + ...fixtureAnalystFinding, + proposal_origin: 'production', + evidence_refs: [{ kind: 'span', uri: 'span-authored-profile' }], +} + +function setup() { + const template = createProfileImprovementFixture() + const task = template.evaluation.experiment.benchmark.tasks[0] + if (!task) throw new Error('expected a profile improvement task') + const { digest: _taskDigest, ...taskMaterial } = task + const identity = 'tenant/research/profile' + const baselineProfile: AgentProfile = { + name: 'researcher', + prompt: { systemPrompt: 'Investigate the question.' }, + model: { default: 'provider/old-model' }, + metadata: { lineage: 'baseline' }, + } + const candidateProfile: AgentProfile = { + ...baselineProfile, + prompt: { systemPrompt: 'Investigate, verify, and cite the earliest causal failure.' }, + model: { default: 'provider/new-model', reasoningEffort: 'high' }, + harness: 'codex', + tools: { Read: true, Bash: true }, + mcp: { literature: { command: 'literature-server' } }, + hooks: { Stop: [{ command: 'node verify-result.mjs', blocking: true }] }, + metadata: { lineage: 'human-reflection', reflectionRun: 'reflection-1' }, + } + const stateDigest = ({ + identity: profileIdentity, + profile, + }: { + identity: string + profile: AgentProfile + }) => canonicalCandidateDigest({ identity: profileIdentity, profile }) + const baselineStateDigest = stateDigest({ identity, profile: baselineProfile }) + const heldOutScenario: CampaignScenarioIdentity = { + id: task.scenario.id, + kind: task.scenario.kind, + scenarioDigest: task.scenario.digest, + } + const observed: Array<{ arm: 'baseline' | 'candidate'; profile: AgentProfile }> = [] + const options: ProposeAuthoredAgentProfileImprovementOptions = { + runId: 'authored-profile-run-1', + budgetUsd: minimumPairedRuns * 2, + source: { + kind: 'platform-agent-profile', + sourceIdentity: identity, + sourceDigest: baselineStateDigest, + sourceRevision: 1, + }, + profile: baselineProfile, + candidateProfile, + candidateLineage: { + source: 'human', + parentDigests: [baselineStateDigest], + runIds: ['reflection-1'], + modelSnapshots: ['provider/new-model'], + }, + diff: { + id: 'human-reflection-1', + source: { + kind: 'frontier-author', + artifacts: ['traces://reflection-1'], + notes: ['A human approved the profile authored from a trace autopsy.'], + }, + }, + findings: [productionFinding], + stateDigest, + benchmark: { + tasks: [taskMaterial], + reps: minimumPairedRuns, + seeds: Array.from({ length: minimumPairedRuns }, (_, index) => 101 + index) as [ + number, + ...number[], + ], + policy: template.evaluation.experiment.policy, + }, + executor: { + executionRef: { + kind: 'agent-profile-improvement-execution-ref', + identity: 'authored-profile-runner', + digest: canonicalCandidateDigest({ runner: 'authored-profile-runner', revision: 1 }), + }, + measure: async (input) => { + observed.push({ arm: input.arm, profile: input.profile }) + const variation = ((input.runCell.repetition % 3) - 1) * 0.02 + return createProfileImprovementRunReceipt( + input, + input.arm === 'baseline' ? 0.2 : 0.8 + variation, + ) + }, + }, + candidate: { + label: 'human trace reflection', + rationale: 'A complete profile authored from a cited trace autopsy.', + }, + now: () => new Date('2026-08-16T20:00:00.000Z'), + } + return { + options, + observed, + baselineProfile, + candidateProfile, + baselineStateDigest, + heldOutScenario, + } +} + +describe('authored profile improvement', { timeout: 30_000 }, () => { + it('measures a human-authored complete profile through the canonical proposal path', async () => { + const fixture = setup() + + const result = await proposeAuthoredAgentProfileImprovement(fixture.options) + + expect(fixture.observed).toHaveLength(minimumPairedRuns * 2) + expect(fixture.observed.filter((entry) => entry.arm === 'baseline').every( + (entry) => entry.profile === fixture.baselineProfile, + )).toBe(false) + expect(fixture.observed.filter((entry) => entry.arm === 'baseline').every( + (entry) => entry.profile.prompt?.systemPrompt === fixture.baselineProfile.prompt?.systemPrompt, + )).toBe(true) + expect(fixture.observed.filter((entry) => entry.arm === 'candidate').every( + (entry) => entry.profile.prompt?.systemPrompt === fixture.candidateProfile.prompt?.systemPrompt, + )).toBe(true) + expect(result.candidateLineage).toMatchObject({ + source: 'human', + parentDigests: [fixture.baselineStateDigest], + runIds: ['reflection-1'], + }) + expect(result.candidateLineage.profileDiffIds?.length).toBeGreaterThan(0) + expect(result.experiment.candidateLineage).toEqual(result.candidateLineage) + expect(result.proposal.evaluation.decision.outcome).toBe('ship') + expect(result.proposal.changedSurfaces).toEqual([ + 'prompt', + 'tools', + 'mcp', + 'hooks', + 'agent-profile', + ]) + expect(result.proposal.findings).toEqual([productionFinding]) + expect(result.proposal.evaluation.generationsExplored).toBe(0) + expect(optimizationActivationReceiptFromMetadata(result.proposal.evaluation.metadata)).toBe( + undefined, + ) + }) + + it('preserves import lineage without fabricating optimizer evidence', async () => { + const fixture = setup() + fixture.options.candidateLineage = { + source: 'import', + parentDigests: [fixture.baselineStateDigest], + runIds: ['external-profile-build-1'], + } + + const result = await proposeAuthoredAgentProfileImprovement(fixture.options) + + expect(result.candidateLineage.source).toBe('import') + expect(optimizationActivationReceiptFromMetadata(result.proposal.evaluation.metadata)).toBe( + undefined, + ) + }) + + it('refuses optimizer lineage and caller-supplied profile diff identities', async () => { + const optimizer = setup() + optimizer.options.candidateLineage = { + source: 'optimizer', + parentDigests: [optimizer.baselineStateDigest], + runIds: ['optimizer-run'], + developmentSplitDigest: canonicalCandidateDigest({ split: 'development' }), + } as never + await expect(proposeAuthoredAgentProfileImprovement(optimizer.options)).rejects.toThrow( + /refuses optimizer lineage/, + ) + + const suppliedIds = setup() + suppliedIds.options.candidateLineage = { + source: 'human', + profileDiffIds: ['caller-controlled-id'], + } as never + await expect(proposeAuthoredAgentProfileImprovement(suppliedIds.options)).rejects.toThrow( + /derives candidateLineage\.profileDiffIds/, + ) + }) + + it('refuses unchanged candidates, source drift, and reused held-out scenarios', async () => { + const unchanged = setup() + unchanged.options.candidateProfile = unchanged.baselineProfile + await expect(proposeAuthoredAgentProfileImprovement(unchanged.options)).rejects.toThrow( + /matches the baseline/, + ) + + const drifted = setup() + drifted.options.source = { + ...drifted.options.source, + sourceDigest: canonicalCandidateDigest({ wrong: true }), + } + await expect(proposeAuthoredAgentProfileImprovement(drifted.options)).rejects.toThrow( + /source digest does not match/, + ) + + const leaked = setup() + leaked.options.developmentScenarios = [leaked.heldOutScenario] + await expect(proposeAuthoredAgentProfileImprovement(leaked.options)).rejects.toThrow( + /reuses development scenario/, + ) + }) +}) From 6366579f776f4bae49398759c8eec6494ca99963 Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 13:54:07 -0700 Subject: [PATCH 06/24] ci: finalize authored profile candidate branch --- .../inspect-authored-profile-contract.yml | 90 ++++++++++++++----- 1 file changed, 67 insertions(+), 23 deletions(-) diff --git a/.github/workflows/inspect-authored-profile-contract.yml b/.github/workflows/inspect-authored-profile-contract.yml index f7210fb1..eecce5c4 100644 --- a/.github/workflows/inspect-authored-profile-contract.yml +++ b/.github/workflows/inspect-authored-profile-contract.yml @@ -1,4 +1,4 @@ -name: Inspect authored profile contract +name: Finalize authored profile candidate on: push: @@ -6,34 +6,78 @@ on: - feat/authored-profile-candidate permissions: - contents: read + contents: write jobs: - inspect: + finalize: + if: github.actor != 'github-actions[bot]' runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v7 - - uses: pnpm/action-setup@v6 - - uses: actions/setup-node@v7 + - name: Check out branch + uses: actions/checkout@v7 + with: + ref: feat/authored-profile-candidate + fetch-depth: 0 + + - name: Set up pnpm + uses: pnpm/action-setup@v6 + + - name: Set up Node + uses: actions/setup-node@v7 with: node-version: 22 cache: pnpm - - run: pnpm install --frozen-lockfile - - name: Print authored profile fixtures + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Export the new surface and bump the package patch + shell: python + run: | + from pathlib import Path + import json + + index_path = Path('src/intelligence/index.ts') + text = index_path.read_text() + anchor = "export type {\n AgentCandidateExperimentCellPlacement," + exports = """export type { + AgentProfileCandidateMeasurementExecutor, + AuthoredAgentProfileCandidateLineage, + AuthoredAgentProfileDiffOptions, + ProposeAuthoredAgentProfileImprovementOptions, + ProposeAuthoredAgentProfileImprovementResult, + } from './authored-profile-improvement' + export { proposeAuthoredAgentProfileImprovement } from './authored-profile-improvement' + """ + if "from './authored-profile-improvement'" not in text: + if anchor not in text: + raise SystemExit('intelligence export anchor not found') + text = text.replace(anchor, exports + anchor, 1) + index_path.write_text(text) + + package_path = Path('package.json') + package = json.loads(package_path.read_text()) + package['version'] = '0.137.1' + package_path.write_text(json.dumps(package, indent=2) + '\n') + + Path('.github/workflows/inspect-authored-profile-contract.yml').unlink() + + - name: Format source and tests + run: pnpm exec biome check --write src/intelligence/authored-profile-improvement.ts src/intelligence/index.ts tests/authored-profile-improvement.test.ts + + - name: Validate the focused implementation + run: | + pnpm exec vitest run tests/authored-profile-improvement.test.ts + pnpm run typecheck + + - name: Regenerate API documentation + run: pnpm run docs:api + + - name: Commit the validated branch shell: bash run: | - set -euo pipefail - test_file=tests/improvement-cycle.test.ts - line=$(grep -n -m1 "measures and activates a directly authored complete profile without improve" "$test_file" | cut -d: -f1) - start=$((line > 100 ? line - 100 : 1)) - end=$((line + 360)) - echo "--- authored profile test ($start-$end) ---" - nl -ba "$test_file" | sed -n "${start},${end}p" - echo "--- helper declarations used by the test ---" - for pattern in "const baselineProfile" "const profileBenchmark" "function profile" "const profileSource" "stateDigest" "profileExecutionRef"; do - grep -n -m5 -C 8 "$pattern" "$test_file" || true - done - echo "--- profile schema lineage refinements ---" - nl -ba node_modules/@tangle-network/agent-interface/dist/agent-profile-improvement-schema.js | sed -n '160,245p' - echo "--- candidate lineage refinements ---" - nl -ba node_modules/@tangle-network/agent-interface/dist/agent-candidate-lineage-schema.js | sed -n '33,90p' + git add package.json src/intelligence docs/api tests/authored-profile-improvement.test.ts .github/workflows/inspect-authored-profile-contract.yml + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git commit -m "feat: measure authored profiles without optimizer lineage" + git push origin HEAD:feat/authored-profile-candidate From e637ff07e5051938de29d0db2e7240a90e546f6c Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 13:55:30 -0700 Subject: [PATCH 07/24] ci: align authored candidate assertions with canonical omission semantics --- .../workflows/inspect-authored-profile-contract.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/inspect-authored-profile-contract.yml b/.github/workflows/inspect-authored-profile-contract.yml index eecce5c4..a2d90aa1 100644 --- a/.github/workflows/inspect-authored-profile-contract.yml +++ b/.github/workflows/inspect-authored-profile-contract.yml @@ -31,12 +31,23 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - - name: Export the new surface and bump the package patch + - name: Export and normalize the new surface shell: python run: | from pathlib import Path import json + source_path = Path('src/intelligence/authored-profile-improvement.ts') + source = source_path.read_text().replace(' type Scenario,\n', '') + source_path.write_text(source) + + test_path = Path('tests/authored-profile-improvement.test.ts') + test = test_path.read_text().replace( + 'expect(result.proposal.evaluation.generationsExplored).toBe(0)', + 'expect(result.proposal.evaluation.generationsExplored).toBeUndefined()', + ) + test_path.write_text(test) + index_path = Path('src/intelligence/index.ts') text = index_path.read_text() anchor = "export type {\n AgentCandidateExperimentCellPlacement," From 3696fecc8be70f648dc00ebabb5fc948bd4d98d0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 16 Aug 2026 20:56:48 +0000 Subject: [PATCH 08/24] feat: measure authored profiles without optimizer lineage --- .../inspect-authored-profile-contract.yml | 94 --------- docs/api/intelligence.md | 196 +++++++++++++++++- docs/api/primitive-catalog.md | 11 +- package.json | 2 +- .../authored-profile-improvement.ts | 11 +- src/intelligence/index.ts | 8 + tests/authored-profile-improvement.test.ts | 37 ++-- 7 files changed, 238 insertions(+), 121 deletions(-) delete mode 100644 .github/workflows/inspect-authored-profile-contract.yml diff --git a/.github/workflows/inspect-authored-profile-contract.yml b/.github/workflows/inspect-authored-profile-contract.yml deleted file mode 100644 index a2d90aa1..00000000 --- a/.github/workflows/inspect-authored-profile-contract.yml +++ /dev/null @@ -1,94 +0,0 @@ -name: Finalize authored profile candidate - -on: - push: - branches: - - feat/authored-profile-candidate - -permissions: - contents: write - -jobs: - finalize: - if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-24.04 - steps: - - name: Check out branch - uses: actions/checkout@v7 - with: - ref: feat/authored-profile-candidate - fetch-depth: 0 - - - name: Set up pnpm - uses: pnpm/action-setup@v6 - - - name: Set up Node - uses: actions/setup-node@v7 - with: - node-version: 22 - cache: pnpm - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - - name: Export and normalize the new surface - shell: python - run: | - from pathlib import Path - import json - - source_path = Path('src/intelligence/authored-profile-improvement.ts') - source = source_path.read_text().replace(' type Scenario,\n', '') - source_path.write_text(source) - - test_path = Path('tests/authored-profile-improvement.test.ts') - test = test_path.read_text().replace( - 'expect(result.proposal.evaluation.generationsExplored).toBe(0)', - 'expect(result.proposal.evaluation.generationsExplored).toBeUndefined()', - ) - test_path.write_text(test) - - index_path = Path('src/intelligence/index.ts') - text = index_path.read_text() - anchor = "export type {\n AgentCandidateExperimentCellPlacement," - exports = """export type { - AgentProfileCandidateMeasurementExecutor, - AuthoredAgentProfileCandidateLineage, - AuthoredAgentProfileDiffOptions, - ProposeAuthoredAgentProfileImprovementOptions, - ProposeAuthoredAgentProfileImprovementResult, - } from './authored-profile-improvement' - export { proposeAuthoredAgentProfileImprovement } from './authored-profile-improvement' - """ - if "from './authored-profile-improvement'" not in text: - if anchor not in text: - raise SystemExit('intelligence export anchor not found') - text = text.replace(anchor, exports + anchor, 1) - index_path.write_text(text) - - package_path = Path('package.json') - package = json.loads(package_path.read_text()) - package['version'] = '0.137.1' - package_path.write_text(json.dumps(package, indent=2) + '\n') - - Path('.github/workflows/inspect-authored-profile-contract.yml').unlink() - - - name: Format source and tests - run: pnpm exec biome check --write src/intelligence/authored-profile-improvement.ts src/intelligence/index.ts tests/authored-profile-improvement.test.ts - - - name: Validate the focused implementation - run: | - pnpm exec vitest run tests/authored-profile-improvement.test.ts - pnpm run typecheck - - - name: Regenerate API documentation - run: pnpm run docs:api - - - name: Commit the validated branch - shell: bash - run: | - git add package.json src/intelligence docs/api tests/authored-profile-improvement.test.ts .github/workflows/inspect-authored-profile-contract.yml - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git commit -m "feat: measure authored profiles without optimizer lineage" - git push origin HEAD:feat/authored-profile-candidate diff --git a/docs/api/intelligence.md b/docs/api/intelligence.md index 29f46248..e5e67d53 100644 --- a/docs/api/intelligence.md +++ b/docs/api/intelligence.md @@ -394,6 +394,151 @@ The product owns the private state lookup and atomic write. *** +### AgentProfileCandidateMeasurementExecutor + +Product-owned executor for exact baseline/candidate profile measurement. + +#### Properties + +##### executionRef + +> **executionRef**: `AgentProfileImprovementExecutionRef` + +#### Methods + +##### measure() + +> **measure**(`input`): `Promise`\<`AgentProfileImprovementRunReceipt`\> + +###### Parameters + +###### input + +`AgentProfileImprovementExperimentExecutionInput` & `object` + +###### Returns + +`Promise`\<`AgentProfileImprovementRunReceipt`\> + +*** + +### ProposeAuthoredAgentProfileImprovementOptions + +Measure a complete human-authored, imported, or compound profile candidate. +No optimizer runs and no optimizer receipt is fabricated. + +#### Properties + +##### runId + +> **runId**: `string` + +##### source + +> **source**: `object` + +##### profile + +> **profile**: `AgentProfile` + +##### stateDigest + +> **stateDigest**: [`AgentImprovementProfileStateDigest`](#agentimprovementprofilestatedigest) + +##### candidateProfile + +> **candidateProfile**: `AgentProfile` + +##### candidateLineage + +> **candidateLineage**: [`AuthoredAgentProfileCandidateLineage`](#authoredagentprofilecandidatelineage) + +##### diff? + +> `optional` **diff?**: [`AgentImprovementTargetProfileDiffOptions`](#agentimprovementtargetprofilediffoptions) + +Optional source/artifact metadata used on Runtime-derived profile diff steps. + +##### findings? + +> `optional` **findings?**: readonly `ProposalFinding`[] + +##### benchmark + +> **benchmark**: [`AgentProfileImprovementBenchmark`](#agentprofileimprovementbenchmark) + +##### executor + +> **executor**: [`AgentProfileCandidateMeasurementExecutor`](#agentprofilecandidatemeasurementexecutor) + +##### budgetUsd + +> **budgetUsd**: `number` + +One customer-approved maximum for the held-out paired measurement. + +##### developmentScenarios? + +> `optional` **developmentScenarios?**: readonly `CampaignScenarioIdentity`[] + +Optional identities used to prove authored/imported development work is held out. + +##### maxConcurrency? + +> `optional` **maxConcurrency?**: `number` + +##### signal? + +> `optional` **signal?**: `AbortSignal` + +##### candidate? + +> `optional` **candidate?**: `object` + +##### metadata? + +> `optional` **metadata?**: `object` + +###### Index Signature + +\[`key`: `string`\]: `AgentCandidateJsonValue` + +##### now? + +> `optional` **now?**: () => `Date` + +###### Returns + +`Date` + +*** + +### ProposeAuthoredAgentProfileImprovementResult + +#### Properties + +##### candidateProfile + +> **candidateProfile**: `AgentProfile` + +##### candidateLineage + +> **candidateLineage**: `AgentCandidateLineage` + +##### experiment + +> **experiment**: `AgentProfileImprovementExperiment` + +##### measurements + +> **measurements**: `AgentProfileImprovementMeasurement`[] + +##### proposal + +> **proposal**: `AgentImprovementProposal` + +*** + ### CredentialRef A named secret a binding requires — declared, never carried. @@ -1507,7 +1652,7 @@ Runtime's expired-attempt path reuses this port only to stop and dispose. ###### Inherited from -[`ExactProcessCandidateExperimentExecutor`](#exactprocesscandidateexperimentexecutor).[`executor`](#executor) +[`ExactProcessCandidateExperimentExecutor`](#exactprocesscandidateexperimentexecutor).[`executor`](#executor-1) ##### recoveryPorts @@ -3503,7 +3648,7 @@ Full canonical profile used for this agent. Exported redacted with a stable hash ###### Inherited from -[`IntelligenceConfig`](#intelligenceconfig).[`profile`](#profile-4) +[`IntelligenceConfig`](#intelligenceconfig).[`profile`](#profile-5) ##### commitSha? @@ -3669,6 +3814,34 @@ Return undefined only when no target write can have committed. *** +### AuthoredAgentProfileCandidateLineage + +> **AuthoredAgentProfileCandidateLineage** = `Omit`\<`AgentCandidateLineage`, `"source"` \| `"profileDiffIds"`\> & `object` + +Lineage accepted by the direct candidate path. Optimizer lineage belongs to `improve()`. + +#### Type Declaration + +##### source + +> **source**: `Exclude`\<`AgentCandidateLineage`\[`"source"`\], `"optimizer"`\> + +##### profileDiffIds? + +> `optional` **profileDiffIds?**: `never` + +Runtime derives these from the exact profile change it seals. + +*** + +### AuthoredAgentProfileDiffOptions + +> **AuthoredAgentProfileDiffOptions** = `NonNullable`\<`Parameters`\<*typeof* [`agentImprovementProfileDiffs`](#agentimprovementprofilediffs)\>\[`2`\]\> + +Provenance attached while Runtime derives the exact profile diff. + +*** + ### JsonSchema > **JsonSchema** = `Record`\<`string`, `unknown`\> @@ -4197,6 +4370,25 @@ Validate and execute one product-owned activation transition. *** +### proposeAuthoredAgentProfileImprovement() + +> **proposeAuthoredAgentProfileImprovement**(`options`): `Promise`\<[`ProposeAuthoredAgentProfileImprovementResult`](#proposeauthoredagentprofileimprovementresult)\> + +Put a complete authored/imported profile through the canonical profile +experiment and proposal path without invoking `improve()`. + +#### Parameters + +##### options + +[`ProposeAuthoredAgentProfileImprovementOptions`](#proposeauthoredagentprofileimprovementoptions) + +#### Returns + +`Promise`\<[`ProposeAuthoredAgentProfileImprovementResult`](#proposeauthoredagentprofileimprovementresult)\> + +*** + ### manifestFromProfile() > **manifestFromProfile**(`profile`): [`CapabilityManifest`](#capabilitymanifest) diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 629be290..23ad78c9 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.137.0` and `@tangle-network/agent-eval@0.145.21` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.137.1` and `@tangle-network/agent-eval@0.145.21` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -393,7 +393,7 @@ Import from `@tangle-network/agent-runtime/tool-loop` — 12 exports. ### Intelligence SDK — Observe + provable-OFF billing -Import from `@tangle-network/agent-runtime/intelligence` — 166 exports. +Import from `@tangle-network/agent-runtime/intelligence` — 172 exports. | Symbol | Kind | Summary | |---|---|---| @@ -428,6 +428,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 166 exports. | `prepareAgentImprovementProfileActivation` | function | Compare product-owned profiles with an exact measured transition and prepare | | `proposeAgentImprovement` | function | Analyze, search, then remeasure the resulting exact candidate before proposing it. | | `proposeAgentProfileImprovement` | function | Analyze a product-owned profile, search one profile surface, then run the | +| `proposeAuthoredAgentProfileImprovement` | function | Put a complete authored/imported profile through the canonical profile | | `pullCertified` | function | Pull the certified composed profile for a target. Fail-closed: a network | | `resolveEffort` | function | Compile a named tier (plus optional per-field overrides) into the flat | | `resolveIntelligenceBaseUrl` | function | Resolve the ONE Intelligence base URL — the single knob both the send and | @@ -451,6 +452,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 166 exports. | `AgentImprovementActivationResult` | interface | Immutable outcome of one idempotent, transaction-wide activation attempt. | | `AgentImprovementMeasuredComparison` | interface | Portable paired held-out comparison produced by a sealed candidate executor. | | `AgentImprovementReview` | interface | Human or tenant-policy decision bound to one exact proposal. | +| `AgentProfileCandidateMeasurementExecutor` | interface | Product-owned executor for exact baseline/candidate profile measurement. | | `AgentProfileImprovementBenchmark` | interface | Product-owned task material that Runtime freezes before either profile state runs. | | `AgentProfileImprovementExecutor` | interface | One product execution adapter shared by optimizer search and exact profile | | `AppliedIntelligence` | interface | What the hook hands the agent each run. Additive over the prompt-only | @@ -478,6 +480,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 166 exports. | `ModeReadiness` | interface | One mode's readiness verdict. | | `ProfileImprovementActivationTransitionInput` | interface | A measured profile change without raw profile bytes. | | `ProposeAgentProfileImprovementOptions` | interface | Complete profile-improvement path for a product-owned source. | +| `ProposeAuthoredAgentProfileImprovementOptions` | interface | Measure a complete human-authored, imported, or compound profile candidate. | | `ProposedProfileDiff` | interface | A gate-certified profile diff the plane has already promoted, plus the | | `ProtectedExactProcessCandidateExperimentExecutor` | interface | Exact-process executor plus the ports required for durable recovery. | | `ProvisionedHost` | interface | A live, provisioned host the resolver tore up for a `process-on-infra` arm. | @@ -504,6 +507,8 @@ Import from `@tangle-network/agent-runtime/intelligence` — 166 exports. | `AgentImprovementProfileStateResolver` | type | Product-owned retained-state lookup used only for an explicit restore. | | `AgentImprovementProposalSubmissionState` | type | What Runtime knows about a failed proposal submission. | | `AgentProfileImprovementMethodOptions` | type | The portable profile changes that the measured-profile contract permits. | +| `AuthoredAgentProfileCandidateLineage` | type | Lineage accepted by the direct candidate path. Optimizer lineage belongs to `improve()`. | +| `AuthoredAgentProfileDiffOptions` | type | Provenance attached while Runtime derives the exact profile diff. | | `CapabilityAuth` | type | How a binding authenticates at resolve time. Declared as a REQUIREMENT in the | | `CapabilityInterface` | type | What the agent consumes. CLOSED — a new runtime kind NEVER extends this. Each | | `CapabilitySurface` | type | Every interface surface tag — the closed set the resolver fans into slots. | @@ -521,7 +526,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 166 exports. | `SubmitAgentImprovementProposalOutcome` | type | Typed result for proposal submission. A successful result contains the | | `UsageClass` | type | Usage class for billing. Base-stream tokens bill `'inference'`; every | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentCandidateExperimentCellPlacement`, `AgentImprovementActivationResultStore`, `AgentImprovementActivationTargetPlan`, `AgentImprovementProfileReplacement`, `AgentImprovementProfileStateDigestInput`, `AgentImprovementProfileStateResolverInput`, `AgentImprovementProposal`, `AgentImprovementTargetProfileDiffOptions`, `AgentProfileImprovementActivationTargetPlan`, `CreateAgentImprovementActivationOptions`, `CreateAgentImprovementActivationResultOptions`, `CreateAgentImprovementProposalOptions`, `CreateExactProcessCandidateExperimentExecutorOptions`, `ExactProcessCandidateExperimentExecution`, `ExactProcessCandidateExperimentExecutor`, `ExecuteAgentCandidateExperimentCellOptions`, `ExecuteAgentImprovementActivationInput`, `ExecuteAgentImprovementActivationOptions`, `OptimizationActivationReceipt`, `OptimizationReceiptCost`, `ProposeAgentImprovementOptions`, `ProposeAgentImprovementResult`, `ProposeAgentProfileImprovementResult`, `PullCertifiedOptions`, `ReviewAgentImprovementInput`, `RunAgentCandidateExperimentOptions`, `RunAgentCandidateExperimentResult`, `SealedCandidateActivationTransitionInput`, `VerifyCandidateExecutionEvidenceOptions`, `AgentImprovementActivationIntent`, `AgentImprovementActivationOutcome`, `AgentImprovementActivationTargetIdentity`, `AgentImprovementActivationTransitionInput`, `AgentImprovementAnalysisOptions`, `AgentImprovementProfileActivationInput`, `AgentImprovementProfileActivationPreparation`, `AgentImprovementProfileActivationTarget`, `AgentImprovementProfileSurface`, `AgentImprovementProfileTargetState`, `AgentImprovementProfileTargetTransition`, `AgentImprovementReviewDecision`, `AgentProfileImprovementActivationOperation`, `AgentProfileMeasuredSurface`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentCandidateExperimentCellPlacement`, `AgentImprovementActivationResultStore`, `AgentImprovementActivationTargetPlan`, `AgentImprovementProfileReplacement`, `AgentImprovementProfileStateDigestInput`, `AgentImprovementProfileStateResolverInput`, `AgentImprovementProposal`, `AgentImprovementTargetProfileDiffOptions`, `AgentProfileImprovementActivationTargetPlan`, `CreateAgentImprovementActivationOptions`, `CreateAgentImprovementActivationResultOptions`, `CreateAgentImprovementProposalOptions`, `CreateExactProcessCandidateExperimentExecutorOptions`, `ExactProcessCandidateExperimentExecution`, `ExactProcessCandidateExperimentExecutor`, `ExecuteAgentCandidateExperimentCellOptions`, `ExecuteAgentImprovementActivationInput`, `ExecuteAgentImprovementActivationOptions`, `OptimizationActivationReceipt`, `OptimizationReceiptCost`, `ProposeAgentImprovementOptions`, `ProposeAgentImprovementResult`, `ProposeAgentProfileImprovementResult`, `ProposeAuthoredAgentProfileImprovementResult`, `PullCertifiedOptions`, `ReviewAgentImprovementInput`, `RunAgentCandidateExperimentOptions`, `RunAgentCandidateExperimentResult`, `SealedCandidateActivationTransitionInput`, `VerifyCandidateExecutionEvidenceOptions`, `AgentImprovementActivationIntent`, `AgentImprovementActivationOutcome`, `AgentImprovementActivationTargetIdentity`, `AgentImprovementActivationTransitionInput`, `AgentImprovementAnalysisOptions`, `AgentImprovementProfileActivationInput`, `AgentImprovementProfileActivationPreparation`, `AgentImprovementProfileActivationTarget`, `AgentImprovementProfileSurface`, `AgentImprovementProfileTargetState`, `AgentImprovementProfileTargetTransition`, `AgentImprovementReviewDecision`, `AgentProfileImprovementActivationOperation`, `AgentProfileMeasuredSurface`. ### Execution kernel — recursive atom, supervision, executors, round-synchronous loop diff --git a/package.json b/package.json index e5f49f77..06823586 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.137.0", + "version": "0.137.1", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { diff --git a/src/intelligence/authored-profile-improvement.ts b/src/intelligence/authored-profile-improvement.ts index a320fa71..d380144d 100644 --- a/src/intelligence/authored-profile-improvement.ts +++ b/src/intelligence/authored-profile-improvement.ts @@ -8,7 +8,6 @@ import { type AgentProfileImprovementExperimentExecutionInput, measuredComparisonFromAgentProfileImprovementExperiment, runAgentProfileImprovementExperiment, - type Scenario, sealAgentProfileImprovementExperiment, sealAgentProfileImprovementSuite, sealAgentProfileImprovementTask, @@ -165,11 +164,7 @@ export async function proposeAuthoredAgentProfileImprovement( }) const policy = profilePolicyWithBudget(options.benchmark.policy, options.budgetUsd) const benchmark = sealProfileImprovementBenchmark({ ...options.benchmark, policy }) - assertDirectCandidateReleaseWorkIsFresh( - benchmark, - candidateLineage, - options.developmentScenarios, - ) + assertDirectCandidateReleaseWorkIsFresh(benchmark, candidateLineage, options.developmentScenarios) const experiment = sealAgentProfileImprovementExperiment({ kind: 'agent-profile-improvement-experiment', digestAlgorithm: 'rfc8785-sha256', @@ -298,7 +293,9 @@ function assertDirectCandidateReleaseWorkIsFresh( .filter((scenario) => development.has(canonicalCandidateDigest(scenario))) .map((scenario) => scenario.id) if (reused.length > 0) { - throw new Error(`authored profile release reuses development scenario(s): [${reused.join(', ')}]`) + throw new Error( + `authored profile release reuses development scenario(s): [${reused.join(', ')}]`, + ) } } diff --git a/src/intelligence/index.ts b/src/intelligence/index.ts index 187df128..0704481d 100644 --- a/src/intelligence/index.ts +++ b/src/intelligence/index.ts @@ -78,6 +78,14 @@ export { executeAgentImprovementActivation, verifyAgentImprovementActivationResult, } from './activation' +export type { + AgentProfileCandidateMeasurementExecutor, + AuthoredAgentProfileCandidateLineage, + AuthoredAgentProfileDiffOptions, + ProposeAuthoredAgentProfileImprovementOptions, + ProposeAuthoredAgentProfileImprovementResult, +} from './authored-profile-improvement' +export { proposeAuthoredAgentProfileImprovement } from './authored-profile-improvement' export type { CapabilityAuth, CapabilityInterface, diff --git a/tests/authored-profile-improvement.test.ts b/tests/authored-profile-improvement.test.ts index db72541f..d47ee055 100644 --- a/tests/authored-profile-improvement.test.ts +++ b/tests/authored-profile-improvement.test.ts @@ -1,7 +1,4 @@ -import { - minimumPairsForPairedDeltaTest, - type ProposalFinding, -} from '@tangle-network/agent-eval' +import { minimumPairsForPairedDeltaTest, type ProposalFinding } from '@tangle-network/agent-eval' import type { CampaignScenarioIdentity } from '@tangle-network/agent-eval/campaign' import type { AgentProfile } from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' @@ -137,15 +134,27 @@ describe('authored profile improvement', { timeout: 30_000 }, () => { const result = await proposeAuthoredAgentProfileImprovement(fixture.options) expect(fixture.observed).toHaveLength(minimumPairedRuns * 2) - expect(fixture.observed.filter((entry) => entry.arm === 'baseline').every( - (entry) => entry.profile === fixture.baselineProfile, - )).toBe(false) - expect(fixture.observed.filter((entry) => entry.arm === 'baseline').every( - (entry) => entry.profile.prompt?.systemPrompt === fixture.baselineProfile.prompt?.systemPrompt, - )).toBe(true) - expect(fixture.observed.filter((entry) => entry.arm === 'candidate').every( - (entry) => entry.profile.prompt?.systemPrompt === fixture.candidateProfile.prompt?.systemPrompt, - )).toBe(true) + expect( + fixture.observed + .filter((entry) => entry.arm === 'baseline') + .every((entry) => entry.profile === fixture.baselineProfile), + ).toBe(false) + expect( + fixture.observed + .filter((entry) => entry.arm === 'baseline') + .every( + (entry) => + entry.profile.prompt?.systemPrompt === fixture.baselineProfile.prompt?.systemPrompt, + ), + ).toBe(true) + expect( + fixture.observed + .filter((entry) => entry.arm === 'candidate') + .every( + (entry) => + entry.profile.prompt?.systemPrompt === fixture.candidateProfile.prompt?.systemPrompt, + ), + ).toBe(true) expect(result.candidateLineage).toMatchObject({ source: 'human', parentDigests: [fixture.baselineStateDigest], @@ -162,7 +171,7 @@ describe('authored profile improvement', { timeout: 30_000 }, () => { 'agent-profile', ]) expect(result.proposal.findings).toEqual([productionFinding]) - expect(result.proposal.evaluation.generationsExplored).toBe(0) + expect(result.proposal.evaluation.generationsExplored).toBeUndefined() expect(optimizationActivationReceiptFromMetadata(result.proposal.evaluation.metadata)).toBe( undefined, ) From 6f0cb7facc296b6d6092b04d5815e64924dbae30 Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 14:11:59 -0700 Subject: [PATCH 09/24] ci: regenerate authored profile testing fixtures --- .../regenerate-authored-profile-fixtures.yml | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .github/workflows/regenerate-authored-profile-fixtures.yml diff --git a/.github/workflows/regenerate-authored-profile-fixtures.yml b/.github/workflows/regenerate-authored-profile-fixtures.yml new file mode 100644 index 00000000..e68cd23e --- /dev/null +++ b/.github/workflows/regenerate-authored-profile-fixtures.yml @@ -0,0 +1,55 @@ +name: Regenerate authored profile testing fixtures + +on: + push: + branches: + - feat/authored-profile-candidate + +permissions: + contents: write + +jobs: + regenerate: + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-24.04 + steps: + - name: Check out branch + uses: actions/checkout@v7 + with: + ref: feat/authored-profile-candidate + fetch-depth: 0 + + - name: Set up pnpm + uses: pnpm/action-setup@v6 + + - name: Set up Node + uses: actions/setup-node@v7 + with: + node-version: 22 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Regenerate canonical proposal fixtures + run: | + pnpm run generate:testing-fixture + pnpm run check:testing-fixture + + - name: Validate affected behavior + run: pnpm exec vitest run tests/testing-fixture.test.ts tests/authored-profile-improvement.test.ts + + - name: Commit regenerated fixtures + shell: bash + run: | + rm .github/workflows/regenerate-authored-profile-fixtures.yml + git add \ + src/testing/fixtures/agent-improvement-proposal.json \ + src/testing/fixtures/agent-profile-improvement-proposal.json \ + src/testing/fixtures/agent-profile-improvement-state.json \ + .github/workflows/regenerate-authored-profile-fixtures.yml + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git diff --cached --quiet && { echo "No fixture changes to commit"; exit 1; } + git commit -m "test: regenerate proposal fixtures for Runtime 0.137.1" + git push origin HEAD:feat/authored-profile-candidate From 2831ab9a8f38ad86b83b09c144010c73b3f2a126 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 16 Aug 2026 21:12:38 +0000 Subject: [PATCH 10/24] test: regenerate proposal fixtures for Runtime 0.137.1 --- .../regenerate-authored-profile-fixtures.yml | 55 ------------------- .../fixtures/agent-improvement-proposal.json | 10 ++-- .../agent-profile-improvement-proposal.json | 6 +- 3 files changed, 8 insertions(+), 63 deletions(-) delete mode 100644 .github/workflows/regenerate-authored-profile-fixtures.yml diff --git a/.github/workflows/regenerate-authored-profile-fixtures.yml b/.github/workflows/regenerate-authored-profile-fixtures.yml deleted file mode 100644 index e68cd23e..00000000 --- a/.github/workflows/regenerate-authored-profile-fixtures.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: Regenerate authored profile testing fixtures - -on: - push: - branches: - - feat/authored-profile-candidate - -permissions: - contents: write - -jobs: - regenerate: - if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-24.04 - steps: - - name: Check out branch - uses: actions/checkout@v7 - with: - ref: feat/authored-profile-candidate - fetch-depth: 0 - - - name: Set up pnpm - uses: pnpm/action-setup@v6 - - - name: Set up Node - uses: actions/setup-node@v7 - with: - node-version: 22 - cache: pnpm - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - - name: Regenerate canonical proposal fixtures - run: | - pnpm run generate:testing-fixture - pnpm run check:testing-fixture - - - name: Validate affected behavior - run: pnpm exec vitest run tests/testing-fixture.test.ts tests/authored-profile-improvement.test.ts - - - name: Commit regenerated fixtures - shell: bash - run: | - rm .github/workflows/regenerate-authored-profile-fixtures.yml - git add \ - src/testing/fixtures/agent-improvement-proposal.json \ - src/testing/fixtures/agent-profile-improvement-proposal.json \ - src/testing/fixtures/agent-profile-improvement-state.json \ - .github/workflows/regenerate-authored-profile-fixtures.yml - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git diff --cached --quiet && { echo "No fixture changes to commit"; exit 1; } - git commit -m "test: regenerate proposal fixtures for Runtime 0.137.1" - git push origin HEAD:feat/authored-profile-candidate diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index 8974eb70..f06d675b 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:a2c6a1a25118f7149ff56f1c7484ed1f649df9af07c09c5890a01e40d718daf5", + "digest": "sha256:8d4fa8827039021e2c8b9ecd640b47770570a2989aa1db08a2d2133c541e664a", "evaluation": { "decision": { "contributingChecks": [ @@ -4882,7 +4882,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.137.0" + "runtimeVersion": "0.137.1" }, "objectives": [ { @@ -4993,8 +4993,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:b9bc6ab59f170c0b281057ff402b2bbe74c78313543c4e0aa71cae61ba066c07", - "runId": "agent-runtime-0.137.0-proposal-fixture", + "recordDigest": "sha256:3934ae82c2501f0f3beabed0dcc53835391db0708b68f5ac79869736fa616d1d", + "runId": "agent-runtime-0.137.1-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -5021,5 +5021,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.137.0-proposal-fixture" + "runId": "agent-runtime-0.137.1-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index 8ae41ba1..733b99e1 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:b77f4da43acd31a4d084c1bbbbb924cfa8d647d670d73b32ac08188d7c2ce282", + "digest": "sha256:36f61e30eaf3b950c0cc1cbafd0ba1a6539099cd0ec9d619bef2f159dc32f543", "evaluation": { "decision": { "contributingChecks": [ @@ -1715,7 +1715,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.137.0" + "runtimeVersion": "0.137.1" }, "objectives": [ { @@ -1826,7 +1826,7 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:3984f31ca644bdf0a773531a8a11f51262992928b0cf96061ce048c910844e70", + "recordDigest": "sha256:04be7837b058e533bcb7b8bc3d4fade6c71926565d9dd47564c6f50a998a3cee", "runId": "profile-improvement-1", "schema": "agent-profile-improvement-experiment" } From d73ea2c59f597a3d3a46e962352200cf90621bd5 Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 14:15:46 -0700 Subject: [PATCH 11/24] ci: deduplicate profile improvement experiment support --- .../refactor-profile-improvement-support.yml | 363 ++++++++++++++++++ 1 file changed, 363 insertions(+) create mode 100644 .github/workflows/refactor-profile-improvement-support.yml diff --git a/.github/workflows/refactor-profile-improvement-support.yml b/.github/workflows/refactor-profile-improvement-support.yml new file mode 100644 index 00000000..6df31486 --- /dev/null +++ b/.github/workflows/refactor-profile-improvement-support.yml @@ -0,0 +1,363 @@ +name: Deduplicate profile improvement experiment support + +on: + push: + branches: + - feat/authored-profile-candidate + +permissions: + contents: write + +jobs: + refactor: + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - name: Check out branch + uses: actions/checkout@v7 + with: + ref: feat/authored-profile-candidate + fetch-depth: 0 + + - name: Set up pnpm + uses: pnpm/action-setup@v6 + + - name: Set up Node + uses: actions/setup-node@v7 + with: + node-version: 22 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Extract canonical profile experiment support + shell: python + run: | + from pathlib import Path + + support = r'''import { CostLedger, type CostLedgerHandle } from '@tangle-network/agent-eval' + import { + type CampaignScenarioIdentity, + campaignSplitDigestFromIdentities, + } from '@tangle-network/agent-eval/campaign' + import { + sealAgentProfileImprovementSuite, + sealAgentProfileImprovementTask, + } from '@tangle-network/agent-eval/contract' + import type { + AgentCandidateEvaluationPolicy, + AgentImprovementCost, + AgentImprovementSource, + AgentProfile, + AgentProfileImprovementMeasuredComparison, + AgentProfileImprovementSuiteInputs, + AgentProfileImprovementTask, + AgentProfileImprovementTaskMaterial, + Sha256Digest, + } from '@tangle-network/agent-interface' + import { + AGENT_IMPROVEMENT_SOURCE_METADATA_KEY, + agentImprovementSourceMetadata, + agentProfileImprovementArmSchema, + numbersApproximatelyEqual, + } from '@tangle-network/agent-interface' + import { immutableCandidateValue } from '../candidate-execution/digest' + import { + assertNoCallerOptimizationReceipt, + attachOptimizationActivationReceipt, + createOptimizationActivationReceipt, + } from './optimization-receipt' + import type { AgentImprovementProfileStateDigest } from './profile-activation' + + export interface ProfileImprovementBenchmarkInput { + tasks: [AgentProfileImprovementTaskMaterial, ...AgentProfileImprovementTaskMaterial[]] + reps: number + seeds: [number, ...number[]] + policy: AgentCandidateEvaluationPolicy + } + + export function createProfileImprovementCostLedger( + budgetUsd: number, + context = 'profile improvement', + ): CostLedger { + if (!Number.isFinite(budgetUsd) || budgetUsd < 0) { + throw new Error(`${context} budgetUsd must be a non-negative finite number`) + } + return new CostLedger({ costCeilingUsd: budgetUsd }) + } + + export function profilePolicyWithBudget( + policy: AgentCandidateEvaluationPolicy, + budgetUsd: number, + context = 'profile improvement', + ): AgentCandidateEvaluationPolicy { + if ( + policy.budgetUsd !== undefined && + !numbersApproximatelyEqual(policy.budgetUsd, budgetUsd) + ) { + throw new Error(`${context} policy budgetUsd must equal the run budgetUsd`) + } + return { ...policy, budgetUsd } + } + + export function profilePreparationAccounting( + costLedger: CostLedgerHandle, + startedAt: number, + ): { wallDurationMs: number; cost: AgentImprovementCost } { + const summary = costLedger.summary() + if (!summary.accountingComplete || summary.costProvenance.kind === 'uncaptured') { + throw new Error('profile improvement preparation cost is incomplete') + } + return { + wallDurationMs: Math.max(0, performance.now() - startedAt), + cost: { + usd: summary.costProvenance.usd, + provenance: summary.costProvenance.kind, + }, + } + } + + export function profileStateDigest( + stateDigest: AgentImprovementProfileStateDigest, + identity: string, + profile: AgentProfile, + ): Sha256Digest { + return agentProfileImprovementArmSchema.parse({ + stateDigest: stateDigest({ identity, profile }), + }).stateDigest + } + + export function sealProfileImprovementBenchmark( + input: ProfileImprovementBenchmarkInput, + ): AgentProfileImprovementSuiteInputs { + const tasks = input.tasks.map((task) => sealAgentProfileImprovementTask(task)) as [ + AgentProfileImprovementTask, + ...AgentProfileImprovementTask[], + ] + return sealAgentProfileImprovementSuite({ + splitDigest: campaignSplitDigestFromIdentities( + tasks.map(profileTaskScenarioIdentity), + input.reps, + ), + tasks, + reps: input.reps, + seeds: input.seeds, + }) + } + + export function profileTaskScenarioIdentity( + task: AgentProfileImprovementTask, + ): CampaignScenarioIdentity { + return { + id: task.scenario.id, + kind: task.scenario.kind, + scenarioDigest: task.scenario.digest, + } + } + + export function profileImprovementMetadata( + metadata: AgentProfileImprovementMeasuredComparison['metadata'], + source: AgentImprovementSource, + optimizationReceipt?: ReturnType, + ): NonNullable { + assertNoCallerOptimizationReceipt(metadata) + if (metadata && Object.hasOwn(metadata, AGENT_IMPROVEMENT_SOURCE_METADATA_KEY)) { + throw new Error( + `candidate metadata reserves '${AGENT_IMPROVEMENT_SOURCE_METADATA_KEY}' for Runtime`, + ) + } + const merged = { ...(metadata ?? {}), ...agentImprovementSourceMetadata(source) } + return optimizationReceipt + ? attachOptimizationActivationReceipt(merged, optimizationReceipt) + : immutableCandidateValue(merged) + } + ''' + support = '\n'.join(line[10:] if line.startswith(' ') else line for line in support.splitlines()) + '\n' + Path('src/intelligence/profile-improvement-experiment.ts').write_text(support) + + def replace_once(text: str, old: str, new: str, label: str) -> str: + if text.count(old) != 1: + raise SystemExit(f'{label}: expected one match, found {text.count(old)}') + return text.replace(old, new, 1) + + def remove_function(text: str, signature: str) -> str: + if text.count(signature) != 1: + raise SystemExit(f'{signature}: expected one function') + start = text.index(signature) + brace = text.index('{', start) + depth = 0 + end = None + for index in range(brace, len(text)): + char = text[index] + if char == '{': + depth += 1 + elif char == '}': + depth -= 1 + if depth == 0: + end = index + 1 + break + if end is None: + raise SystemExit(f'{signature}: unbalanced function') + while end < len(text) and text[end] == '\n': + end += 1 + return text[:start] + text[end:] + + cycle_path = Path('src/intelligence/improvement-cycle.ts') + cycle = cycle_path.read_text() + cycle = replace_once( + cycle, + "import {\n type CampaignScenarioIdentity,\n campaignSplitDigestFromIdentities,\n} from '@tangle-network/agent-eval/campaign'", + "import type { CampaignScenarioIdentity } from '@tangle-network/agent-eval/campaign'", + 'campaign import', + ) + for line in ( + ' sealAgentProfileImprovementSuite,\n', + ' sealAgentProfileImprovementTask,\n', + ' AgentImprovementCost,\n', + ' AgentProfileImprovementTask,\n', + ' AGENT_IMPROVEMENT_SOURCE_METADATA_KEY,\n', + ' agentImprovementSourceMetadata,\n', + ' agentProfileImprovementArmSchema,\n', + ' attachOptimizationActivationReceipt,\n', + ): + cycle = replace_once(cycle, line, '', f'remove {line.strip()}') + optimization_anchor = "import {\n assertNoCallerOptimizationReceipt,\n createOptimizationActivationReceipt,\n optimizationActivationReceiptFromMetadata,\n} from './optimization-receipt'\n" + shared_import = """import { + createProfileImprovementCostLedger, + profileImprovementMetadata, + profilePolicyWithBudget, + profilePreparationAccounting, + profileStateDigest, + profileTaskScenarioIdentity, + sealProfileImprovementBenchmark, + } from './profile-improvement-experiment' + """ + shared_import = '\n'.join(line[10:] if line.startswith(' ') else line for line in shared_import.splitlines()) + '\n' + cycle = replace_once( + cycle, + optimization_anchor, + optimization_anchor + shared_import, + 'shared support import', + ) + for signature in ( + 'function createProfileImprovementCostLedger(', + 'function profilePolicyWithBudget(', + 'function profilePreparationAccounting(', + 'function profileStateDigest(', + 'function sealProfileImprovementBenchmark(', + 'function profileTaskScenarioIdentity(', + 'function profileImprovementMetadata(', + ): + cycle = remove_function(cycle, signature) + cycle_path.write_text(cycle) + + authored_path = Path('src/intelligence/authored-profile-improvement.ts') + authored = authored_path.read_text() + authored = replace_once( + authored, + "import { CostLedger } from '@tangle-network/agent-eval'\n", + '', + 'authored CostLedger import', + ) + authored = replace_once( + authored, + "import {\n type CampaignScenarioIdentity,\n campaignSplitDigestFromIdentities,\n} from '@tangle-network/agent-eval/campaign'", + "import type { CampaignScenarioIdentity } from '@tangle-network/agent-eval/campaign'", + 'authored campaign import', + ) + for line in ( + ' sealAgentProfileImprovementSuite,\n', + ' sealAgentProfileImprovementTask,\n', + ' AgentProfileImprovementTask,\n', + ' AGENT_IMPROVEMENT_SOURCE_METADATA_KEY,\n', + ' agentImprovementSourceMetadata,\n', + ' agentProfileImprovementArmSchema,\n', + ' numbersApproximatelyEqual,\n', + ): + authored = replace_once(authored, line, '', f'authored remove {line.strip()}') + authored_anchor = "import { assertNoCallerOptimizationReceipt } from './optimization-receipt'\n" + authored_import = """import { + createProfileImprovementCostLedger, + profileImprovementMetadata, + profilePolicyWithBudget, + profilePreparationAccounting, + profileStateDigest, + profileTaskScenarioIdentity, + sealProfileImprovementBenchmark, + } from './profile-improvement-experiment' + """ + authored_import = '\n'.join(line[10:] if line.startswith(' ') else line for line in authored_import.splitlines()) + '\n' + authored = replace_once( + authored, + authored_anchor, + authored_anchor + authored_import, + 'authored shared support import', + ) + authored = replace_once( + authored, + 'const costLedger = createMeasurementCostLedger(options.budgetUsd)', + "const costLedger = createProfileImprovementCostLedger(\n options.budgetUsd,\n 'authored profile improvement',\n )", + 'authored cost ledger call', + ) + authored = replace_once( + authored, + 'const policy = profilePolicyWithBudget(options.benchmark.policy, options.budgetUsd)', + "const policy = profilePolicyWithBudget(\n options.benchmark.policy,\n options.budgetUsd,\n 'authored profile',\n )", + 'authored policy call', + ) + old_preparation = """const preparation = { + wallDurationMs: Math.max(0, performance.now() - preparationStartedAt), + cost: { usd: 0, provenance: 'observed' as const }, + }""" + old_preparation = '\n'.join(line[10:] if line.startswith(' ') else line for line in old_preparation.splitlines()) + authored = replace_once( + authored, + old_preparation, + 'const preparation = profilePreparationAccounting(costLedger, preparationStartedAt)', + 'authored preparation accounting', + ) + authored = replace_once( + authored, + 'metadata: directProfileImprovementMetadata(options.metadata, source),', + 'metadata: profileImprovementMetadata(options.metadata, source),', + 'authored metadata call', + ) + for signature in ( + 'function createMeasurementCostLedger(', + 'function profilePolicyWithBudget(', + 'function profileStateDigest(', + 'function sealProfileImprovementBenchmark(', + 'function profileTaskScenarioIdentity(', + 'function directProfileImprovementMetadata(', + ): + authored = remove_function(authored, signature) + authored_path.write_text(authored) + + - name: Format changed source + run: pnpm exec biome check --write src/intelligence/profile-improvement-experiment.ts src/intelligence/improvement-cycle.ts src/intelligence/authored-profile-improvement.ts + + - name: Validate source, fixtures, and behavior + run: | + pnpm run lint + pnpm run typecheck + pnpm run check:testing-fixture + pnpm test + pnpm run docs:api + + - name: Commit validated refactor + shell: bash + run: | + rm .github/workflows/refactor-profile-improvement-support.yml + git add \ + src/intelligence/profile-improvement-experiment.ts \ + src/intelligence/improvement-cycle.ts \ + src/intelligence/authored-profile-improvement.ts \ + docs/api \ + .github/workflows/refactor-profile-improvement-support.yml + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git diff --cached --quiet && { echo "No refactor changes to commit"; exit 1; } + git commit -m "refactor: share canonical profile experiment support" + git push origin HEAD:feat/authored-profile-candidate From a71eb95c0d45af4e4b55f46eeeb83dd759086db4 Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 14:20:53 -0700 Subject: [PATCH 12/24] ci: make profile support refactor deterministic --- .../refactor-profile-improvement-support.py | 360 ++++++++++++++++++ .../refactor-profile-improvement-support.yml | 311 +-------------- 2 files changed, 367 insertions(+), 304 deletions(-) create mode 100644 .github/scripts/refactor-profile-improvement-support.py diff --git a/.github/scripts/refactor-profile-improvement-support.py b/.github/scripts/refactor-profile-improvement-support.py new file mode 100644 index 00000000..2d0a7c1f --- /dev/null +++ b/.github/scripts/refactor-profile-improvement-support.py @@ -0,0 +1,360 @@ +from __future__ import annotations + +import re +from pathlib import Path + + +def replace_once(text: str, old: str, new: str, label: str) -> str: + count = text.count(old) + if count != 1: + raise SystemExit(f"{label}: expected one match, found {count}") + return text.replace(old, new, 1) + + +def remove_import_symbols( + text: str, + *, + header_end: str, + symbols: tuple[str, ...], + label: str, +) -> str: + boundary = text.find(header_end) + if boundary < 0: + raise SystemExit(f"{label}: import boundary not found") + header = text[:boundary] + body = text[boundary:] + for symbol in symbols: + pattern = re.compile(rf"(?m)^[ \t]+(?:type[ \t]+)?{re.escape(symbol)},\n") + matches = pattern.findall(header) + if len(matches) != 1: + raise SystemExit( + f"{label}: expected one imported {symbol}, found {len(matches)}" + ) + header = pattern.sub("", header, count=1) + return header + body + + +def remove_function(text: str, signature: str) -> str: + count = text.count(signature) + if count != 1: + raise SystemExit(f"{signature}: expected one function, found {count}") + start = text.index(signature) + brace = text.index("{", start) + depth = 0 + quote: str | None = None + escaped = False + line_comment = False + block_comment = False + index = brace + while index < len(text): + char = text[index] + next_char = text[index + 1] if index + 1 < len(text) else "" + if line_comment: + if char == "\n": + line_comment = False + index += 1 + continue + if block_comment: + if char == "*" and next_char == "/": + block_comment = False + index += 2 + else: + index += 1 + continue + if quote is not None: + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == quote: + quote = None + index += 1 + continue + if char == "/" and next_char == "/": + line_comment = True + index += 2 + continue + if char == "/" and next_char == "*": + block_comment = True + index += 2 + continue + if char in ("'", '"', "`"): + quote = char + index += 1 + continue + if char == "{": + depth += 1 + elif char == "}": + depth -= 1 + if depth == 0: + end = index + 1 + while end < len(text) and text[end] == "\n": + end += 1 + return text[:start] + text[end:] + index += 1 + raise SystemExit(f"{signature}: unbalanced function") + + +SUPPORT = """import { CostLedger, type CostLedgerHandle } from '@tangle-network/agent-eval' +import { + type CampaignScenarioIdentity, + campaignSplitDigestFromIdentities, +} from '@tangle-network/agent-eval/campaign' +import { + sealAgentProfileImprovementSuite, + sealAgentProfileImprovementTask, +} from '@tangle-network/agent-eval/contract' +import type { + AgentCandidateEvaluationPolicy, + AgentImprovementCost, + AgentImprovementSource, + AgentProfile, + AgentProfileImprovementMeasuredComparison, + AgentProfileImprovementSuiteInputs, + AgentProfileImprovementTask, + AgentProfileImprovementTaskMaterial, + Sha256Digest, +} from '@tangle-network/agent-interface' +import { + AGENT_IMPROVEMENT_SOURCE_METADATA_KEY, + agentImprovementSourceMetadata, + agentProfileImprovementArmSchema, + numbersApproximatelyEqual, +} from '@tangle-network/agent-interface' +import { immutableCandidateValue } from '../candidate-execution/digest' +import { + assertNoCallerOptimizationReceipt, + attachOptimizationActivationReceipt, + createOptimizationActivationReceipt, +} from './optimization-receipt' +import type { AgentImprovementProfileStateDigest } from './profile-activation' + +export interface ProfileImprovementBenchmarkInput { + tasks: [AgentProfileImprovementTaskMaterial, ...AgentProfileImprovementTaskMaterial[]] + reps: number + seeds: [number, ...number[]] + policy: AgentCandidateEvaluationPolicy +} + +export function createProfileImprovementCostLedger( + budgetUsd: number, + context = 'profile improvement', +): CostLedger { + if (!Number.isFinite(budgetUsd) || budgetUsd < 0) { + throw new Error(`${context} budgetUsd must be a non-negative finite number`) + } + return new CostLedger({ costCeilingUsd: budgetUsd }) +} + +export function profilePolicyWithBudget( + policy: AgentCandidateEvaluationPolicy, + budgetUsd: number, + context = 'profile improvement', +): AgentCandidateEvaluationPolicy { + if (policy.budgetUsd !== undefined && !numbersApproximatelyEqual(policy.budgetUsd, budgetUsd)) { + throw new Error(`${context} policy budgetUsd must equal the run budgetUsd`) + } + return { ...policy, budgetUsd } +} + +export function profilePreparationAccounting( + costLedger: CostLedgerHandle, + startedAt: number, +): { wallDurationMs: number; cost: AgentImprovementCost } { + const summary = costLedger.summary() + if (!summary.accountingComplete || summary.costProvenance.kind === 'uncaptured') { + throw new Error('profile improvement preparation cost is incomplete') + } + return { + wallDurationMs: Math.max(0, performance.now() - startedAt), + cost: { + usd: summary.costProvenance.usd, + provenance: summary.costProvenance.kind, + }, + } +} + +export function profileStateDigest( + stateDigest: AgentImprovementProfileStateDigest, + identity: string, + profile: AgentProfile, +): Sha256Digest { + return agentProfileImprovementArmSchema.parse({ + stateDigest: stateDigest({ identity, profile }), + }).stateDigest +} + +export function sealProfileImprovementBenchmark( + input: ProfileImprovementBenchmarkInput, +): AgentProfileImprovementSuiteInputs { + const tasks = input.tasks.map((task) => sealAgentProfileImprovementTask(task)) as [ + AgentProfileImprovementTask, + ...AgentProfileImprovementTask[], + ] + return sealAgentProfileImprovementSuite({ + splitDigest: campaignSplitDigestFromIdentities( + tasks.map(profileTaskScenarioIdentity), + input.reps, + ), + tasks, + reps: input.reps, + seeds: input.seeds, + }) +} + +export function profileTaskScenarioIdentity( + task: AgentProfileImprovementTask, +): CampaignScenarioIdentity { + return { + id: task.scenario.id, + kind: task.scenario.kind, + scenarioDigest: task.scenario.digest, + } +} + +export function profileImprovementMetadata( + metadata: AgentProfileImprovementMeasuredComparison['metadata'], + source: AgentImprovementSource, + optimizationReceipt?: ReturnType, +): NonNullable { + assertNoCallerOptimizationReceipt(metadata) + if (metadata && Object.hasOwn(metadata, AGENT_IMPROVEMENT_SOURCE_METADATA_KEY)) { + throw new Error( + `candidate metadata reserves '${AGENT_IMPROVEMENT_SOURCE_METADATA_KEY}' for Runtime`, + ) + } + const merged = { ...(metadata ?? {}), ...agentImprovementSourceMetadata(source) } + return optimizationReceipt + ? attachOptimizationActivationReceipt(merged, optimizationReceipt) + : immutableCandidateValue(merged) +} +""" + + +Path("src/intelligence/profile-improvement-experiment.ts").write_text(SUPPORT) + +cycle_path = Path("src/intelligence/improvement-cycle.ts") +cycle = cycle_path.read_text() +cycle = remove_import_symbols( + cycle, + header_end="\n\nexport type {", + symbols=( + "campaignSplitDigestFromIdentities", + "sealAgentProfileImprovementSuite", + "sealAgentProfileImprovementTask", + "AgentImprovementCost", + "AgentProfileImprovementTask", + "AGENT_IMPROVEMENT_SOURCE_METADATA_KEY", + "agentImprovementSourceMetadata", + "agentProfileImprovementArmSchema", + "attachOptimizationActivationReceipt", + ), + label="improvement-cycle imports", +) +shared_import = """import { + createProfileImprovementCostLedger, + profileImprovementMetadata, + profilePolicyWithBudget, + profilePreparationAccounting, + profileStateDigest, + profileTaskScenarioIdentity, + sealProfileImprovementBenchmark, +} from './profile-improvement-experiment' +""" +cycle = replace_once( + cycle, + "import type { AgentImprovementProfileStateDigest } from './profile-activation'\n", + shared_import + + "import type { AgentImprovementProfileStateDigest } from './profile-activation'\n", + "improvement-cycle shared support import", +) +for function in ( + "function createProfileImprovementCostLedger(", + "function profilePolicyWithBudget(", + "function profilePreparationAccounting(", + "function profileStateDigest(", + "function sealProfileImprovementBenchmark(", + "function profileTaskScenarioIdentity(", + "function profileImprovementMetadata(", +): + cycle = remove_function(cycle, function) +cycle_path.write_text(cycle) + +authored_path = Path("src/intelligence/authored-profile-improvement.ts") +authored = authored_path.read_text() +authored = replace_once( + authored, + "import { CostLedger } from '@tangle-network/agent-eval'\n", + "", + "authored CostLedger import", +) +authored = remove_import_symbols( + authored, + header_end="\n\n/** Lineage accepted", + symbols=( + "campaignSplitDigestFromIdentities", + "sealAgentProfileImprovementSuite", + "sealAgentProfileImprovementTask", + "AgentProfileImprovementTask", + "AGENT_IMPROVEMENT_SOURCE_METADATA_KEY", + "agentImprovementSourceMetadata", + "agentProfileImprovementArmSchema", + "numbersApproximatelyEqual", + ), + label="authored-profile imports", +) +authored = replace_once( + authored, + "import type { AgentImprovementProfileStateDigest } from './profile-activation'\n", + shared_import + + "import type { AgentImprovementProfileStateDigest } from './profile-activation'\n", + "authored shared support import", +) +authored = replace_once( + authored, + "const costLedger = createMeasurementCostLedger(options.budgetUsd)", + "const costLedger = createProfileImprovementCostLedger(\n" + " options.budgetUsd,\n" + " 'authored profile improvement',\n" + " )", + "authored cost ledger call", +) +authored = replace_once( + authored, + "const policy = profilePolicyWithBudget(options.benchmark.policy, options.budgetUsd)", + "const policy = profilePolicyWithBudget(\n" + " options.benchmark.policy,\n" + " options.budgetUsd,\n" + " 'authored profile',\n" + " )", + "authored policy call", +) +preparation_pattern = re.compile( + r" const preparation = \{\n" + r" wallDurationMs: Math\.max\(0, performance\.now\(\) - preparationStartedAt\),\n" + r" cost: \{ usd: 0, provenance: 'observed' as const \},\n" + r" \}" +) +authored, count = preparation_pattern.subn( + " const preparation = profilePreparationAccounting(costLedger, preparationStartedAt)", + authored, + count=1, +) +if count != 1: + raise SystemExit(f"authored preparation accounting: expected one match, found {count}") +authored = replace_once( + authored, + "metadata: directProfileImprovementMetadata(options.metadata, source),", + "metadata: profileImprovementMetadata(options.metadata, source),", + "authored metadata call", +) +for function in ( + "function createMeasurementCostLedger(", + "function profilePolicyWithBudget(", + "function profileStateDigest(", + "function sealProfileImprovementBenchmark(", + "function profileTaskScenarioIdentity(", + "function directProfileImprovementMetadata(", +): + authored = remove_function(authored, function) +authored_path.write_text(authored) diff --git a/.github/workflows/refactor-profile-improvement-support.yml b/.github/workflows/refactor-profile-improvement-support.yml index 6df31486..917bd53d 100644 --- a/.github/workflows/refactor-profile-improvement-support.yml +++ b/.github/workflows/refactor-profile-improvement-support.yml @@ -12,7 +12,7 @@ jobs: refactor: if: github.actor != 'github-actions[bot]' runs-on: ubuntu-24.04 - timeout-minutes: 30 + timeout-minutes: 35 steps: - name: Check out branch uses: actions/checkout@v7 @@ -33,329 +33,32 @@ jobs: run: pnpm install --frozen-lockfile - name: Extract canonical profile experiment support - shell: python - run: | - from pathlib import Path - - support = r'''import { CostLedger, type CostLedgerHandle } from '@tangle-network/agent-eval' - import { - type CampaignScenarioIdentity, - campaignSplitDigestFromIdentities, - } from '@tangle-network/agent-eval/campaign' - import { - sealAgentProfileImprovementSuite, - sealAgentProfileImprovementTask, - } from '@tangle-network/agent-eval/contract' - import type { - AgentCandidateEvaluationPolicy, - AgentImprovementCost, - AgentImprovementSource, - AgentProfile, - AgentProfileImprovementMeasuredComparison, - AgentProfileImprovementSuiteInputs, - AgentProfileImprovementTask, - AgentProfileImprovementTaskMaterial, - Sha256Digest, - } from '@tangle-network/agent-interface' - import { - AGENT_IMPROVEMENT_SOURCE_METADATA_KEY, - agentImprovementSourceMetadata, - agentProfileImprovementArmSchema, - numbersApproximatelyEqual, - } from '@tangle-network/agent-interface' - import { immutableCandidateValue } from '../candidate-execution/digest' - import { - assertNoCallerOptimizationReceipt, - attachOptimizationActivationReceipt, - createOptimizationActivationReceipt, - } from './optimization-receipt' - import type { AgentImprovementProfileStateDigest } from './profile-activation' - - export interface ProfileImprovementBenchmarkInput { - tasks: [AgentProfileImprovementTaskMaterial, ...AgentProfileImprovementTaskMaterial[]] - reps: number - seeds: [number, ...number[]] - policy: AgentCandidateEvaluationPolicy - } - - export function createProfileImprovementCostLedger( - budgetUsd: number, - context = 'profile improvement', - ): CostLedger { - if (!Number.isFinite(budgetUsd) || budgetUsd < 0) { - throw new Error(`${context} budgetUsd must be a non-negative finite number`) - } - return new CostLedger({ costCeilingUsd: budgetUsd }) - } - - export function profilePolicyWithBudget( - policy: AgentCandidateEvaluationPolicy, - budgetUsd: number, - context = 'profile improvement', - ): AgentCandidateEvaluationPolicy { - if ( - policy.budgetUsd !== undefined && - !numbersApproximatelyEqual(policy.budgetUsd, budgetUsd) - ) { - throw new Error(`${context} policy budgetUsd must equal the run budgetUsd`) - } - return { ...policy, budgetUsd } - } - - export function profilePreparationAccounting( - costLedger: CostLedgerHandle, - startedAt: number, - ): { wallDurationMs: number; cost: AgentImprovementCost } { - const summary = costLedger.summary() - if (!summary.accountingComplete || summary.costProvenance.kind === 'uncaptured') { - throw new Error('profile improvement preparation cost is incomplete') - } - return { - wallDurationMs: Math.max(0, performance.now() - startedAt), - cost: { - usd: summary.costProvenance.usd, - provenance: summary.costProvenance.kind, - }, - } - } - - export function profileStateDigest( - stateDigest: AgentImprovementProfileStateDigest, - identity: string, - profile: AgentProfile, - ): Sha256Digest { - return agentProfileImprovementArmSchema.parse({ - stateDigest: stateDigest({ identity, profile }), - }).stateDigest - } - - export function sealProfileImprovementBenchmark( - input: ProfileImprovementBenchmarkInput, - ): AgentProfileImprovementSuiteInputs { - const tasks = input.tasks.map((task) => sealAgentProfileImprovementTask(task)) as [ - AgentProfileImprovementTask, - ...AgentProfileImprovementTask[], - ] - return sealAgentProfileImprovementSuite({ - splitDigest: campaignSplitDigestFromIdentities( - tasks.map(profileTaskScenarioIdentity), - input.reps, - ), - tasks, - reps: input.reps, - seeds: input.seeds, - }) - } - - export function profileTaskScenarioIdentity( - task: AgentProfileImprovementTask, - ): CampaignScenarioIdentity { - return { - id: task.scenario.id, - kind: task.scenario.kind, - scenarioDigest: task.scenario.digest, - } - } - - export function profileImprovementMetadata( - metadata: AgentProfileImprovementMeasuredComparison['metadata'], - source: AgentImprovementSource, - optimizationReceipt?: ReturnType, - ): NonNullable { - assertNoCallerOptimizationReceipt(metadata) - if (metadata && Object.hasOwn(metadata, AGENT_IMPROVEMENT_SOURCE_METADATA_KEY)) { - throw new Error( - `candidate metadata reserves '${AGENT_IMPROVEMENT_SOURCE_METADATA_KEY}' for Runtime`, - ) - } - const merged = { ...(metadata ?? {}), ...agentImprovementSourceMetadata(source) } - return optimizationReceipt - ? attachOptimizationActivationReceipt(merged, optimizationReceipt) - : immutableCandidateValue(merged) - } - ''' - support = '\n'.join(line[10:] if line.startswith(' ') else line for line in support.splitlines()) + '\n' - Path('src/intelligence/profile-improvement-experiment.ts').write_text(support) - - def replace_once(text: str, old: str, new: str, label: str) -> str: - if text.count(old) != 1: - raise SystemExit(f'{label}: expected one match, found {text.count(old)}') - return text.replace(old, new, 1) - - def remove_function(text: str, signature: str) -> str: - if text.count(signature) != 1: - raise SystemExit(f'{signature}: expected one function') - start = text.index(signature) - brace = text.index('{', start) - depth = 0 - end = None - for index in range(brace, len(text)): - char = text[index] - if char == '{': - depth += 1 - elif char == '}': - depth -= 1 - if depth == 0: - end = index + 1 - break - if end is None: - raise SystemExit(f'{signature}: unbalanced function') - while end < len(text) and text[end] == '\n': - end += 1 - return text[:start] + text[end:] - - cycle_path = Path('src/intelligence/improvement-cycle.ts') - cycle = cycle_path.read_text() - cycle = replace_once( - cycle, - "import {\n type CampaignScenarioIdentity,\n campaignSplitDigestFromIdentities,\n} from '@tangle-network/agent-eval/campaign'", - "import type { CampaignScenarioIdentity } from '@tangle-network/agent-eval/campaign'", - 'campaign import', - ) - for line in ( - ' sealAgentProfileImprovementSuite,\n', - ' sealAgentProfileImprovementTask,\n', - ' AgentImprovementCost,\n', - ' AgentProfileImprovementTask,\n', - ' AGENT_IMPROVEMENT_SOURCE_METADATA_KEY,\n', - ' agentImprovementSourceMetadata,\n', - ' agentProfileImprovementArmSchema,\n', - ' attachOptimizationActivationReceipt,\n', - ): - cycle = replace_once(cycle, line, '', f'remove {line.strip()}') - optimization_anchor = "import {\n assertNoCallerOptimizationReceipt,\n createOptimizationActivationReceipt,\n optimizationActivationReceiptFromMetadata,\n} from './optimization-receipt'\n" - shared_import = """import { - createProfileImprovementCostLedger, - profileImprovementMetadata, - profilePolicyWithBudget, - profilePreparationAccounting, - profileStateDigest, - profileTaskScenarioIdentity, - sealProfileImprovementBenchmark, - } from './profile-improvement-experiment' - """ - shared_import = '\n'.join(line[10:] if line.startswith(' ') else line for line in shared_import.splitlines()) + '\n' - cycle = replace_once( - cycle, - optimization_anchor, - optimization_anchor + shared_import, - 'shared support import', - ) - for signature in ( - 'function createProfileImprovementCostLedger(', - 'function profilePolicyWithBudget(', - 'function profilePreparationAccounting(', - 'function profileStateDigest(', - 'function sealProfileImprovementBenchmark(', - 'function profileTaskScenarioIdentity(', - 'function profileImprovementMetadata(', - ): - cycle = remove_function(cycle, signature) - cycle_path.write_text(cycle) - - authored_path = Path('src/intelligence/authored-profile-improvement.ts') - authored = authored_path.read_text() - authored = replace_once( - authored, - "import { CostLedger } from '@tangle-network/agent-eval'\n", - '', - 'authored CostLedger import', - ) - authored = replace_once( - authored, - "import {\n type CampaignScenarioIdentity,\n campaignSplitDigestFromIdentities,\n} from '@tangle-network/agent-eval/campaign'", - "import type { CampaignScenarioIdentity } from '@tangle-network/agent-eval/campaign'", - 'authored campaign import', - ) - for line in ( - ' sealAgentProfileImprovementSuite,\n', - ' sealAgentProfileImprovementTask,\n', - ' AgentProfileImprovementTask,\n', - ' AGENT_IMPROVEMENT_SOURCE_METADATA_KEY,\n', - ' agentImprovementSourceMetadata,\n', - ' agentProfileImprovementArmSchema,\n', - ' numbersApproximatelyEqual,\n', - ): - authored = replace_once(authored, line, '', f'authored remove {line.strip()}') - authored_anchor = "import { assertNoCallerOptimizationReceipt } from './optimization-receipt'\n" - authored_import = """import { - createProfileImprovementCostLedger, - profileImprovementMetadata, - profilePolicyWithBudget, - profilePreparationAccounting, - profileStateDigest, - profileTaskScenarioIdentity, - sealProfileImprovementBenchmark, - } from './profile-improvement-experiment' - """ - authored_import = '\n'.join(line[10:] if line.startswith(' ') else line for line in authored_import.splitlines()) + '\n' - authored = replace_once( - authored, - authored_anchor, - authored_anchor + authored_import, - 'authored shared support import', - ) - authored = replace_once( - authored, - 'const costLedger = createMeasurementCostLedger(options.budgetUsd)', - "const costLedger = createProfileImprovementCostLedger(\n options.budgetUsd,\n 'authored profile improvement',\n )", - 'authored cost ledger call', - ) - authored = replace_once( - authored, - 'const policy = profilePolicyWithBudget(options.benchmark.policy, options.budgetUsd)', - "const policy = profilePolicyWithBudget(\n options.benchmark.policy,\n options.budgetUsd,\n 'authored profile',\n )", - 'authored policy call', - ) - old_preparation = """const preparation = { - wallDurationMs: Math.max(0, performance.now() - preparationStartedAt), - cost: { usd: 0, provenance: 'observed' as const }, - }""" - old_preparation = '\n'.join(line[10:] if line.startswith(' ') else line for line in old_preparation.splitlines()) - authored = replace_once( - authored, - old_preparation, - 'const preparation = profilePreparationAccounting(costLedger, preparationStartedAt)', - 'authored preparation accounting', - ) - authored = replace_once( - authored, - 'metadata: directProfileImprovementMetadata(options.metadata, source),', - 'metadata: profileImprovementMetadata(options.metadata, source),', - 'authored metadata call', - ) - for signature in ( - 'function createMeasurementCostLedger(', - 'function profilePolicyWithBudget(', - 'function profileStateDigest(', - 'function sealProfileImprovementBenchmark(', - 'function profileTaskScenarioIdentity(', - 'function directProfileImprovementMetadata(', - ): - authored = remove_function(authored, signature) - authored_path.write_text(authored) + run: python .github/scripts/refactor-profile-improvement-support.py - name: Format changed source run: pnpm exec biome check --write src/intelligence/profile-improvement-experiment.ts src/intelligence/improvement-cycle.ts src/intelligence/authored-profile-improvement.ts - - name: Validate source, fixtures, and behavior + - name: Validate source, fixtures, behavior, and docs run: | pnpm run lint pnpm run typecheck pnpm run check:testing-fixture pnpm test pnpm run docs:api + pnpm run docs:freshness - name: Commit validated refactor shell: bash run: | rm .github/workflows/refactor-profile-improvement-support.yml + rm .github/scripts/refactor-profile-improvement-support.py git add \ src/intelligence/profile-improvement-experiment.ts \ src/intelligence/improvement-cycle.ts \ src/intelligence/authored-profile-improvement.ts \ docs/api \ - .github/workflows/refactor-profile-improvement-support.yml + .github/workflows/refactor-profile-improvement-support.yml \ + .github/scripts/refactor-profile-improvement-support.py git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git diff --cached --quiet && { echo "No refactor changes to commit"; exit 1; } From 227d9f80d5c1b731dab6f3cf57cd1be367fab649 Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 14:27:33 -0700 Subject: [PATCH 13/24] ci: handle inline TypeScript return types during extraction --- .../refactor-profile-improvement-support.yml | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/.github/workflows/refactor-profile-improvement-support.yml b/.github/workflows/refactor-profile-improvement-support.yml index 917bd53d..d3918058 100644 --- a/.github/workflows/refactor-profile-improvement-support.yml +++ b/.github/workflows/refactor-profile-improvement-support.yml @@ -32,6 +32,67 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile + - name: Repair extraction parser for inline return types + shell: python + run: | + from pathlib import Path + + path = Path('.github/scripts/refactor-profile-improvement-support.py') + text = path.read_text() + old = ''' start = text.index(signature) + brace = text.index("{", start) + depth = 0 + quote: str | None = None + escaped = False + line_comment = False + block_comment = False + index = brace + ''' + old = '\n'.join(line[14:] if line.startswith(' ') else line for line in old.splitlines()) + '\n' + new = ''' start = text.index(signature) + parameter_open = text.index("(", start) + parameter_depth = 0 + parameter_end = None + for cursor in range(parameter_open, len(text)): + char = text[cursor] + if char == "(": + parameter_depth += 1 + elif char == ")": + parameter_depth -= 1 + if parameter_depth == 0: + parameter_end = cursor + 1 + break + if parameter_end is None: + raise SystemExit(f"{signature}: unbalanced parameters") + brace = text.index("{", parameter_end) + prefix = text[parameter_end:brace].rstrip() + if prefix.endswith(":"): + type_depth = 0 + type_end = None + for cursor in range(brace, len(text)): + char = text[cursor] + if char == "{": + type_depth += 1 + elif char == "}": + type_depth -= 1 + if type_depth == 0: + type_end = cursor + 1 + break + if type_end is None: + raise SystemExit(f"{signature}: unbalanced inline return type") + brace = text.index("{", type_end) + depth = 0 + quote: str | None = None + escaped = False + line_comment = False + block_comment = False + index = brace + ''' + new = '\n'.join(line[14:] if line.startswith(' ') else line for line in new.splitlines()) + '\n' + if text.count(old) != 1: + raise SystemExit(f'extraction parser anchor count: {text.count(old)}') + path.write_text(text.replace(old, new, 1)) + - name: Extract canonical profile experiment support run: python .github/scripts/refactor-profile-improvement-support.py From 20cda061eb27d93ebbfd3197eab810d9139ec7ed Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 14:31:00 -0700 Subject: [PATCH 14/24] ci: replace profile refactor patcher with a balanced parser --- .../refactor-profile-improvement-support.py | 59 +++++++++++++----- .../refactor-profile-improvement-support.yml | 61 ------------------- 2 files changed, 45 insertions(+), 75 deletions(-) diff --git a/.github/scripts/refactor-profile-improvement-support.py b/.github/scripts/refactor-profile-improvement-support.py index 2d0a7c1f..7534f741 100644 --- a/.github/scripts/refactor-profile-improvement-support.py +++ b/.github/scripts/refactor-profile-improvement-support.py @@ -34,18 +34,15 @@ def remove_import_symbols( return header + body -def remove_function(text: str, signature: str) -> str: - count = text.count(signature) - if count != 1: - raise SystemExit(f"{signature}: expected one function, found {count}") - start = text.index(signature) - brace = text.index("{", start) +def matching_delimiter(text: str, start: int, opening: str, closing: str) -> int: + if start >= len(text) or text[start] != opening: + raise SystemExit(f"expected {opening!r} at offset {start}") depth = 0 quote: str | None = None escaped = False line_comment = False block_comment = False - index = brace + index = start while index < len(text): char = text[index] next_char = text[index + 1] if index + 1 < len(text) else "" @@ -82,17 +79,51 @@ def remove_function(text: str, signature: str) -> str: quote = char index += 1 continue - if char == "{": + if char == opening: depth += 1 - elif char == "}": + elif char == closing: depth -= 1 if depth == 0: - end = index + 1 - while end < len(text) and text[end] == "\n": - end += 1 - return text[:start] + text[end:] + return index + 1 index += 1 - raise SystemExit(f"{signature}: unbalanced function") + raise SystemExit(f"unbalanced {opening}{closing} delimiter at offset {start}") + + +def skip_whitespace(text: str, start: int) -> int: + index = start + while index < len(text) and text[index].isspace(): + index += 1 + return index + + +def function_body_open(text: str, start: int) -> int: + parameter_open = text.index("(", start) + parameter_end = matching_delimiter(text, parameter_open, "(", ")") + cursor = skip_whitespace(text, parameter_end) + if cursor < len(text) and text[cursor] == ":": + cursor = skip_whitespace(text, cursor + 1) + if cursor < len(text) and text[cursor] == "{": + cursor = skip_whitespace(text, matching_delimiter(text, cursor, "{", "}")) + else: + body = text.find("{", cursor) + if body < 0: + raise SystemExit("function body not found after return type") + return body + if cursor >= len(text) or text[cursor] != "{": + raise SystemExit(f"function body not found at offset {cursor}") + return cursor + + +def remove_function(text: str, signature: str) -> str: + count = text.count(signature) + if count != 1: + raise SystemExit(f"{signature}: expected one function, found {count}") + start = text.index(signature) + body_open = function_body_open(text, start) + end = matching_delimiter(text, body_open, "{", "}") + while end < len(text) and text[end] == "\n": + end += 1 + return text[:start] + text[end:] SUPPORT = """import { CostLedger, type CostLedgerHandle } from '@tangle-network/agent-eval' diff --git a/.github/workflows/refactor-profile-improvement-support.yml b/.github/workflows/refactor-profile-improvement-support.yml index d3918058..917bd53d 100644 --- a/.github/workflows/refactor-profile-improvement-support.yml +++ b/.github/workflows/refactor-profile-improvement-support.yml @@ -32,67 +32,6 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - - name: Repair extraction parser for inline return types - shell: python - run: | - from pathlib import Path - - path = Path('.github/scripts/refactor-profile-improvement-support.py') - text = path.read_text() - old = ''' start = text.index(signature) - brace = text.index("{", start) - depth = 0 - quote: str | None = None - escaped = False - line_comment = False - block_comment = False - index = brace - ''' - old = '\n'.join(line[14:] if line.startswith(' ') else line for line in old.splitlines()) + '\n' - new = ''' start = text.index(signature) - parameter_open = text.index("(", start) - parameter_depth = 0 - parameter_end = None - for cursor in range(parameter_open, len(text)): - char = text[cursor] - if char == "(": - parameter_depth += 1 - elif char == ")": - parameter_depth -= 1 - if parameter_depth == 0: - parameter_end = cursor + 1 - break - if parameter_end is None: - raise SystemExit(f"{signature}: unbalanced parameters") - brace = text.index("{", parameter_end) - prefix = text[parameter_end:brace].rstrip() - if prefix.endswith(":"): - type_depth = 0 - type_end = None - for cursor in range(brace, len(text)): - char = text[cursor] - if char == "{": - type_depth += 1 - elif char == "}": - type_depth -= 1 - if type_depth == 0: - type_end = cursor + 1 - break - if type_end is None: - raise SystemExit(f"{signature}: unbalanced inline return type") - brace = text.index("{", type_end) - depth = 0 - quote: str | None = None - escaped = False - line_comment = False - block_comment = False - index = brace - ''' - new = '\n'.join(line[14:] if line.startswith(' ') else line for line in new.splitlines()) + '\n' - if text.count(old) != 1: - raise SystemExit(f'extraction parser anchor count: {text.count(old)}') - path.write_text(text.replace(old, new, 1)) - - name: Extract canonical profile experiment support run: python .github/scripts/refactor-profile-improvement-support.py From 7316bce3f016ca28b1594cd52eb63122450eebdd Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 14:33:27 -0700 Subject: [PATCH 15/24] ci: retain generic optimization receipt import --- .github/workflows/refactor-profile-improvement-support.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/refactor-profile-improvement-support.yml b/.github/workflows/refactor-profile-improvement-support.yml index 917bd53d..6b0a83c6 100644 --- a/.github/workflows/refactor-profile-improvement-support.yml +++ b/.github/workflows/refactor-profile-improvement-support.yml @@ -32,6 +32,9 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile + - name: Keep the generic optimization receipt import + run: sed -i '/"attachOptimizationActivationReceipt",/d' .github/scripts/refactor-profile-improvement-support.py + - name: Extract canonical profile experiment support run: python .github/scripts/refactor-profile-improvement-support.py From 964ee41c7a850719272915d84faa6053708f1737 Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 14:38:59 -0700 Subject: [PATCH 16/24] docs: align canonical API with Runtime 0.137.1 --- docs/canonical-api.md | 191 +----------------------------------------- 1 file changed, 1 insertion(+), 190 deletions(-) diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 0724e8de..2379255a 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,7 +4,7 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.137.0.** +> **Version 0.137.1.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. > `agent-eval` must satisfy `>=0.145.21 <0.146.0`. > `sandbox` must satisfy `>=0.27.1 <0.28.0`. @@ -22,192 +22,3 @@ Run pnpm docs:freshness after editing this file. --> > - **conserved budget pool**: one shared compute budget split across workers, so two different topologies cost the same and a comparison is fair. > - **combinator**: a reusable topology shape (`loopUntil` = depth/refine, `fanout` = breadth/sample) you compose instead of hand-writing a loop. > - **holdout**: fresh problems held back from tuning, so a measured win can't be memorization. - -The system is four steps, each with a named entry point: - -1. **Describe the agent as data.** A **profile** is the whole agent: `systemPrompt + skills + tools + mcp + knowledge + memory + rag`, one combined surface. -2. **Run it.** A driver steers workers over rounds: `runPersonified` composes a combinator (`loopUntil`, `fanout`, …) over the `Supervisor`, spending K rounds against one persistent, journaled artifact from a *conserved budget pool*, so any two topologies you compare cost the same by construction. -3. **Score it on a benchmark.** Either the `ADAPTERS` registry driven by `runGate` over the Supervisor, or an `AgenticSurface` driven by `runBenchmark`/`runAgentic`. -4. **Improve it on three partitions.** - `improve(profile, { executionRef, method, trainScenarios, selectionScenarios, testScenarios, agent })` runs a complete agent-eval `OptimizationMethod`. - The method receives train and selection cases only. - Runtime materializes each surface as a complete detached profile and passes that profile to `agent`. - `executionRef` identifies the callback, component mapping, model, tools, and closure settings. - Agent Eval scores the selected profile on the untouched final test. - Runtime returns `ship` only when the paired interval clears the required lift and all spend is accounted for. - `candidatePopulation` joins verified callback observations with the optimizer's official graph. - It returns every unique candidate as an exact profile plus Interface diffs, or as an explicit refusal. - Official GEPA graph nodes retain parent indices and selection scores. - -Two standing rules: the model that picks the best attempt is never the model that grades it, and observation attaches to the *loop* via `RuntimeHooks`, never to the portable profile. One known limit: the current `Supervisor` records completed settlements but does not resume a live tree after coordinator restart. - -(The original one-sentence compressed form of this spine is preserved in [design.md](./design.md).) - -Two substrates implement the same recursive-atom over the one `Executor` port and share `defaultSelectWinner`: a deliberate pair, **do not invent a third**. -The reactive `Supervisor`/`Scope` plus personify combinators drive dynamic agent trees; the round-synchronous `runAgentRounds` kernel is one leaf backend. -`inlineSandboxClient` adapts any non-box `Executor` into a `SandboxClient` for `runAgentRounds`, and `settledToIteration` bridges reactive `Settled` results into the kernel's `Iteration`. -It is separate from `runToolLoop` and `streamToolLoop` in `/tool-loop`, which run one chat turn and fold tool calls back into it. - -## 1.5 The AgentProfile rule: author the profile, the substrate materializes it - -An `AgentProfile` contains the agent's system prompt, skills, tools, MCP servers, subagents, hooks, permissions, memory, retrieval configuration, and model settings. -Skills remain separate resources that the runtime can invoke; do not concatenate them into the system prompt. -Package-owned skills can remain in their source repository. -Reference the skill by an immutable commit and Runtime resolves it before a worktree worker starts: - -```ts -import { defineGitHubResource, type AgentProfile } from '@tangle-network/agent-interface' - -const tracesSkillsRef = process.env.TRACES_SKILLS_REF -if (!tracesSkillsRef) throw new Error('TRACES_SKILLS_REF must be an immutable commit') - -const profile: AgentProfile = { - name: 'trace-reviewer', - resources: { - failOnError: true, - skills: [ - defineGitHubResource('skills/inspect-agent-traces/SKILL.md', { - repository: 'tangle-network/traces', - ref: tracesSkillsRef, - name: 'inspect-agent-traces', - }), - ], - }, -} -``` - -The shared materializer normalizes the fetched markdown and mounts it in the selected agent's native skill directory. -The original profile remains unchanged, and the exact mounted bytes are covered by the materialization receipt. - -**You change an agent's behavior by changing its PROFILE: never by writing orchestration code around it.** The behaviors we keep hand-rolling are profile properties: -- **Self-verification** is a profile lever, three ways, all configuration and zero glue code: (1) *steered*: the prompt says "run the tests, read failures, fix, repeat"; (2) *process-defined*: its instructions make verify-after-every-change its standing process; or (3) a **post-finish hook** that auto-runs the check and feeds failures back. The harness runs that loop. **You do not write a per-round judge, a `while(!done)`, or a bash hill-climb.** -- **Iteration, delegation, audit-against-spec** are likewise hooks / subagents / skills / process *in the profile*. - -**The sandbox substrate materializes a profile into the harness's real shapes: so author the GENERAL profile and NEVER code to a harness.** `@tangle-network/sandbox` renders an `AgentProfile` into whatever the running harness needs (instructions file, tool/MCP config, mounted skills, hooks, subagents). opencode / Claude Code / Codex are interchangeable *targets*; opencode is only the local **test** substrate behind the cli-bridge. **Do NOT write harness-specific config or a `profile → opencode.json` realizer.** A lever that isn't materialized yet is a **substrate gap to fill in `@tangle-network/sandbox`**, not a bespoke realizer here. - -**Therefore the supervisor's only intelligence is AUTHORING full profiles**: the optimizable self-improvement surface: read the task, decompose it, and for each sub-task author the *complete* profile (which prompt, skills, tools/MCP, hooks, subagents, model). The quality of a worker IS the quality of the profile authored for it. **The harness executes; you compose.** - -## 2. Decision table: "I want to ___ → use ___ → NOT ___" - -This table is judgment-only: it maps an intent to the ONE primitive to reach for and the thing NOT to build. It is not an inventory: **for the full list of what exists (every export, its import path, its one-line summary) see the generated `docs/api/primitive-catalog.md`; for full signatures, the per-module `docs/api/` pages.** Each row tags its import subpath; a row is a LOCAL export of this package unless tagged with a substrate package (`agent-eval/contract`, `agent-eval/campaign`, `@tangle-network/sandbox`) or `bench`. - -### "A loop" is not one thing: read this before reaching for one - -A general "loop" primitive is the single most common modelling error in this repo; it has produced a `defineLoop` facade **twice** (see [`research/loop-facade-postmortem.md`](./research/loop-facade-postmortem.md)). "Iterate agents toward a goal" splits on **one question: is the structure FIXED (you write the shape, it's code) or DYNAMIC (a model decides the shape at runtime)?** How many agents (1 vs N) is *orthogonal*: every shape below is 1..N agents, so "two agents (proposer→verifier)" is not special, it's a 2-stage chain. - -**FIXED shape → a combinator you compose:** - -| You want… | Shape | Use (`/kernel`) | -|---|---|---| -| refine ONE artifact over rounds until a check passes | depth | `loopUntil` | -| try N independent attempts, keep the best | breadth | `fanout` | -| ordered stages, each feeds the next: this is what "propose → verify" is | chain | `pipeline` | -| many independent views of one artifact, then aggregate | ensemble | `panel` | -| expand only the promising branches | adaptive search | `widen` + `flatWidenGate` | - -**DYNAMIC shape → this is orchestration, NOT a loop.** When an LLM decides *at runtime* what to spawn next and when to stop (decompose a messy goal, react to each result, no fixed round count), it is a reactive tree, not a loop: `Scope` + Supervisor in-process (`supervise` / `runPersonified`), or `createCoordinationTools` for a sandbox driver. Its topology is *data*, so no fixed-round "loop" grammar can describe it. - -**The trap** is a single grammar (`defineLoop`, a `runXxxLoop`) spanning all of the above: there can't be one, because some are code and one is a model deciding. No new loop primitive lands without a tiny executable proof, **over real agents**, of the exact substrate join it claims to simplify. - -| I want to… | Use (import) | Do NOT build | -|---|---|---| -| Run one product chat turn with streamed events, ordered persistence hooks, and stable execution/turn identity | `handleChatTurn(...)` + `deriveExecutionId(...)`: `/durable`; pass the derived id as both `executionId` and `turnId` on initial dispatch | importing the broad package entry from an edge worker, treating `executionId` alone as dispatch idempotency, or rebuilding framing and persistence ordering in the product | -| Run a supervisor toward a goal with default setup | `supervise(profile, task, { budget, backend? })`: `/kernel` | hand-wiring `createSupervisor().run` + `blobs`/`perWorker`/`journal`/`executors`; reaching for lower-level calls before you need a specific counterparty | -| Score a supervised sandbox worker by an executable check **against its live box** | `supervise(..., { backend: { backend: 'sandbox', sandboxClient, validator } })`: `/kernel` — the leaf forwards it to the composed `runAgentRounds`, which calls `validate(output, ctx)` with `ctx.box` still alive, and the verdict lands on the worker's settle | a post-settle hook (the box is destroyed by then), a second scoring loop beside `depthStrategy`, or pairing `validator` with `steering` (refused: a steerable session composes no loop to score) | -| Run a static root, workers, and analysts as reviewable `AgentProfile` nodes with versioned edge directives | `runGraph(graph, options)`: `/kernel` | a second graph executor, prompt-only roles, or pretending a static graph can discover new nodes while running | -| Drive a graph's ROOT with caller-owned orchestration (a deterministic conversation driver; a persona loop that makes its own LLM calls) | `runGraph(graph, { brain })`: `/kernel` — `brain: ToolLoopChat` is caller data for a router-brained root; node pinning, directive delivery, the edge ledger, and the journal twin stay the same shipped path, and the root profile keeps prompt control (`systemPrompt`/`instructions` still apply). Model selection, provider-identity validation, and usage reporting move to the caller with the brain | routing a production run through the `/testing` entry, a bespoke driver loop beside the graph, or pairing `brain` with `driverBackend` / an external-harness root (both refused: two answers to who makes the root's calls) | -| **Supervise agents to solve a graded `AgenticSurface` task** (workers `runAgentic` the surface, settle on its own check, driver self-improves from the failing tests) | `superviseSurface(profile, task, { surface, worker })`: `/kernel` | a worker-seam + a "self-improving supervisor" wrapper around `supervise()`; passing a custom `makeWorkerAgent` that runs `runAgentic` | -| Run a profile through a topology shape over the keystone Supervisor, end-to-end | `runPersonified({ persona, shape, task, budget })`: `/kernel` | a hand-rolled `createSupervisor().run` + seam-wiring helper | -| Address a supervisor run's durable state on disk, or steer a live worker from another process | `supervisorRunsRoot(root)` / `supervisorRunDir(root, id)` / `writeWorkerSteer(...)` / `readWorkerSteerRequests(...)`: `/kernel` — the `/.agent/supervisor/` contract `traces analyze --supervisor-run-dir` reads (`legacySupervisorRunDir` names the pre-rename `.loops` location for readers only) | inventing a run-dir layout, joining `.agent/supervisor` by hand, or writing a steer file whose shape no published reader knows | -| Cancel one worker from another process and read the acknowledged effect | `cancelWorker(eventDir, worker, operationId)` / `readWorkerCancellation(eventDir, operationId)`: `/kernel` — retry-safe by `operationId` lookup; the run's own turn loop applies the abort to exactly that worker's subtree and records `cancel_requested` → `cancelled` / `not_live` (reusing `RetainedRunEffect`), with the `terminated` set naming every node that died. Only the root manager's DIRECT children are addressable; a request naming a deeper descendant stays `unknown` — cancel its lead instead | writing an unread cancel file and calling it done, minting a second four-state cancellation vocabulary, killing the worker's process from outside, naming a nested descendant and waiting for an acknowledgement that cannot come, or treating a missing worker as a successful cancel | -| Give a worker's clone the source workspace's untracked build artifacts | `withUntrackedArtifacts(ws, sourceDir)` wrapping the `Workspace`: `/kernel` | a post-materialize `cp -r`, a hardlink farm, or accepting that a bare `git clone` cannot build | -| Expose what a settled worker shows the brain (failing verify tail + diff head + note, bounded) | `composeWorkerEvidence(...)` + `settledWorkerOut(...)` + `closingWorkerNote(...)`: `/kernel` | re-rolling truncation caps per consumer, or settling with bare counters the brain cannot act on | -| Loop a worker over one evolving artifact, K rounds, stop-when-good | `loopUntil(seed, spec)` as the `shape`: `/kernel` | a `while(!done){runWorker();decide()}` hand-loop or "multi-attempt refine driver" | -| Run a worker agent under test conversing with a **simulated-user persona**, K rounds, worker-only metered | `runPersonaConversation({ worker, persona, backendFor, systemPromptOf })`: root `.` (also `/kernel`) | a hand-rolled per-agent `dispatchWithSurface` bridge / eval-dispatch loop | -| Run **two `AgentProfile`s head-to-head** with a separate resumable session for each actor | `runConversation(...)` from root `.` | a hand-rolled two-agent turn loop | -| Drop a persona⟷agent conversation into an eval matrix as its dispatch | `runPersonaDispatch` → `runProfileMatrix({ dispatch })`: root `.` / `agent-eval/campaign` | a per-agent custom dispatch bridge | -| Best-of-N / parallel-research / map-reduce at equal compute | `fanout(items, opts)`: `/kernel` | `Promise.all` over N calls + manual argmax/merge (bypasses the budget pool → breaks equal-k) | -| Produce-then-gate with a real checker | `verify(spec)`: `/kernel` | "generate, then self-check with the same model, ship if ok" (collapses selector+judge) | -| Multi-judge review / rubric quorum over one artifact | `panel(spec)`: `/kernel` | a judge ensemble that feeds one judge's score into another | -| Fixed sequential chain (plan→implement→…) | `pipeline(stages)`: `/kernel` | hand-chained `await`s passing outputs along | -| Adaptive tree search / progressive widening | `widen(spec)` + `flatWidenGate()`: `/kernel` | a best-first/MCTS that reads child *scores* to expand (selector=judge); keep `flatWidenGate()` until your gate is proven | -| Define the profile record for a personified run | `definePersona(input)`: `/kernel`; its root must carry a complete exact `AgentProfile`, which Runtime parses and freezes at definition time | a "profile-seam" / agent-config wrapper carrying model+prompt+tools+role | -| Make a worker self-verify / iterate / audit | a **hook / process / skill on its authored `AgentProfile`**: §1.5 | a per-round judge, a `while(!done)` loop, or a bash hill-climb (it's a profile lever) | -| Run an authored profile with Claude Code, Codex, OpenCode, or another supported harness | author the `AgentProfile` with `@tangle-network/agent-interface`; `@tangle-network/sandbox` materializes it for the selected harness | a harness-specific profile or config writer | -| Have the supervisor design its workers | author a **full `AgentProfile`** per sub-task (prompt+skills+tools+mcp+hooks+subagents): `/kernel` | author a bare `systemPrompt` string (a worker can't act on levers it has no levers for) | -| Write a custom driver Agent and run it directly | `createSupervisor().run(root, task, opts)`: `/kernel` | a bespoke orchestrator that spawns sub-agents and tallies cost (equal-compute claim breaks there) | -| Run depth-vs-breadth (or a custom strategy) over a stateful tool domain | `runAgentic({ surface, task, mode\|strategy, budget })`: `/kernel` | a hand-rolled `Supervisor.run` + journal/registry, or a depth/breadth loop | -| Author a new topology/strategy compactly | `defineStrategy(name, body)` using `ctx.shot()`+`ctx.critique()`: `/kernel` | a 70-line driver with `scope.spawn`/`scope.next` ceremony, or trusting a body-returned score | -| Compare strategies + get a significance report on a domain | `runBenchmark({ environment, tasks, worker, strategies })`: `/kernel` | your own strategy-comparison loop / paired-bootstrap / Pareto math | -| Add a stateful tool-using domain | implement `AgenticSurface` (5 hooks: open/tools/call/score/close): `/kernel` | a bespoke per-benchmark agent runner / tool-loop harness | -| Run a sandbox coding rollout, round-synchronous (fresh box per round) | `runAgentRounds(options)`: `/kernel` | a `new Sandbox()`+acquire+stream+parse+delete loop, or a 2nd winner-selector | -| Run **agent-eval fixture folders** through Runtime `runAgentRounds` | agent-eval fixture loading/planning, then `loopCampaignDispatch(...)`: `/kernel`; it starts the Runtime cell inside Eval's paid-call lifecycle | a one-off `runCampaign` dispatch, or attaching a completed `LoopResult` after paid work already ran | -| Run a **recursive `supervise()` tree** through an agent-eval profile matrix | `superviseDispatch({ toTask, toSuperviseOptions, ... })`: `/kernel`; it admits the tree through Eval before Runtime spends, then records its receipt only when Runtime proves one model. Mixed or unknown trees fail instead of being relabelled. | a Lab receipt mapper, a second scheduler, or attaching a completed `SupervisedResult` after paid work already ran | -| Run + **resume** ONE persistent box across turns | `openSandboxRun(client, opts, deliverable)`: `/kernel` | a per-domain `new Sandbox`+`box.fs.read`+delete copy | -| Start a retry-safe detached run in a new environment, or a fresh harness chat in one existing environment | `startRetainedRun(...)` or `startRetainedRunInEnvironment(...)`: `/kernel`; both persist exact coordinates before and after dispatch; the existing-environment path also verifies its retained key through provider metadata; only `continueNative(...)` may claim same-chat continuity | calling `provider.create/get/dispatch` directly, reusing an environment as proof of chat continuity, or appending to an unverified native session | -| Run **ONE agent turn** on any substrate: box (`streamPrompt`), cli-bridge/router `Executor`, or in-process chat backend: as ONE normalized `RuntimeStreamEvent` stream with a guaranteed terminal result+usage event; opt into in-stream `tool_call`/`tool_result` with `preserveToolParts`, or tap the raw sandbox events with `onRawEvent` | `streamAgentTurn(backend, prompt, { signal, timeoutMs, preserveToolParts?, onRawEvent? })` + `collectAgentTurn(stream)`: `/kernel` | a per-provider stream→event mapper zoo, a hand-faked box around a non-box executor, or raw fetch leaking through the turn abstraction | -| Use an exact profile and Runtime executor where `runAgentTaskStream` or a conversation expects an `AgentExecutionBackend` | `createProfileExecutionBackend({ profile, executor: createExecutor(config) })`: root `.`; the adapter preserves conversation authorization, recursion-depth, and trace headers | a provider-specific backend constructor or an adapter that reads a second model/prompt configuration | -| Pick the **execution transport a driven loop runs on** (`sandbox` box / cli-bridge / router) from a product flag | `resolveSandboxClient({ backend })`: `/kernel` | a per-product `if (backend === 'router') …` branch re-wiring `createExecutor` + `inlineSandboxClient` | -| Adapt an exact `AgentProfile` to agent-eval's `ChatClient` without moving credentials or execution policy into Eval | `profileChatClient({ profile, executor, context })`: `/kernel` | a provider fetch configured separately from the profile, or request fields that override the profile's model policy | -| Pick / register a leaf backend, or bring your own agent | `createExecutor({ backend })` / `createExecutorRegistry()` / implement `Executor`: `/kernel` | a per-vendor adapter or closed `inline\|sandbox\|cli` switch (won't report through the `UsageEvent` channel) | -| Run a worker as a **conversation on a bare `/v1/chat/completions` endpoint** (no sandbox), with session continuity for `continuity: 'resume'` graphs | `chatTransportExecutor(options)` + `chatWorkerSeam({ url, sessions?, deliverable? })` + `createChatSessionStore()`: `/kernel` | a leaf-seam fake of a chat worker, a multishot transcript loop outside the kernel (no ledger, no conserved pool), or a resume that re-primes a fresh session | -| Optimize text or named components with upstream GEPA | `officialGepa({ recipe, ... })`, passed as `improve(...).method` from root `.` | a local GEPA approximation, prompt mutation loop, or silent fallback when Python is unavailable | -| Optimize one text surface with Microsoft SkillOpt | `officialSkillOpt({ trainer, optimizer, ... })`, passed as `improve(...).method` from root `.` | Runtime-owned SkillOpt search or a silent local fallback | -| Improve one profile coordinate | `improve(profile, { surface, executionRef, method, trainScenarios, selectionScenarios, testScenarios, judges, agent, costCeiling })` from root `.`; `executionRef` binds saved work to executable behavior, `agent` receives the exact complete candidate profile, and the total-cost option limits the whole run | an implicit per-surface optimizer, a method that sees final-test cases, an unmeasured profile mutation, or separate optimizer and final-test spend limits | -| Inspect observed optimizer package, model, usage, cost, and resumed-run evidence before proposing a change | `createOptimizationActivationReceipt(result)` from `/intelligence` | reconstructing optimizer evidence from logs or trusting caller-authored metadata | -| Compare complete optimization methods directly | `compareOptimizationMethods(...)` from `agent-eval/campaign` | comparing one method's training score to another method's final score | -| Improve repository code | `improve({ surface: 'code', code, scenarios, judge, agent, budget })` from root `.` | passing code through a text optimizer or managing candidate worktrees in product code | -| Decide ship/hold on a candidate (campaign context) | `defaultProductionGate({ holdoutScenarios, deltaThreshold })`; compose with `heldOutGate` / `composeGate`: `agent-eval/contract` | a raw `h1>h0` point comparison on the training set | -| Decide ship/hold from a **`BenchmarkReport`** (per-task cells) | `promotionGate({ report, incumbent, candidate })`: `/kernel` | comparing two strategies' mean scores directly; re-deriving the bootstrap | -| Run the full multi-generation strategy flywheel + certify | `runStrategyEvolution(config)`: `/kernel` | a bespoke gen0→author→gen1→holdout loop with hand-rolled champion selection | -| Add or run a benchmark from the CLI/harness | `ADAPTERS` / `resolveAdapter(key)`, run via `bench/src/gate-cli.mts` | a per-script `switch(bench)` or a local benchmark-factory map | -| Wire a new benchmark | implement `BenchmarkAdapter` (5 methods) + feed to `runGate`: `bench` | a bespoke per-benchmark run script with its own (self-authored) scoring | -| Measure a topology on a benchmark at equal compute | `runGate(cfg)` (or `runAgentic`/`runBenchmark`): equal-k holds via the conserved budget pool: `bench`/`/kernel` | a batch-blind/batch-oracle/compare zoo, your own usage capture, or equal-k bookkeeping | -| Observe a run's full cost/time | `createWaterfallCollector()` → `anytimeReport()`: `/kernel` | a per-step cost/token tally by inspecting events yourself (drifts from billed totals) | -| Meter **one `openSandboxRun` cell's token/cost usage** (the metering seam for bench cells) | `sumSandboxUsage(events)`: `/kernel` (folds `extractLlmCallEvent` over the run's events) | a per-bench usage tally re-parsing raw sandbox events (misses usage → integrity-guard rejects the cell) | -| Stand up a **product eval leaderboard** (declare `cases` + `prompt` + `score` → harness×model matrix + ranked board): START HERE (product leaderboards) | `defineLeaderboard(spec)`: `/kernel` (every default overridable: `backends`/`dispatch`/`judges` seams; `runProfileMatrix` stays public as the escape floor; `toBenchmarkAdapter()` registers it into a benchmark registry) | the hand-rolled `expandProfileAxes` + `loopDispatch` + `runProfileMatrix` assembly (~650 lines/product) with its stale cell-cache, zero-token stub-cell, and missing-model-snapshot footguns | -| Render a **multi-profile × multi-axis benchmark leaderboard** (ranked board + score matrix + SVG/HTML charts) from an EXISTING fleet of matrix runs | `leaderboard(records)` + `renderLeaderboardMarkdown` / `renderLeaderboardSvg` / `renderLeaderboardHtml`: `/kernel` (feed it `runProfileMatrix().records`, any domain; `defineLeaderboard` calls these for you) | a per-benchmark report/chart renderer; hand-rolled SVG/markdown tables; a curated subset of axes | -| Attach N observers to a running loop | `composeRuntimeHooks(...)`: root export | a second event-bus or callback-prop zoo (there is ONE stream) | -| Ship traces to an OTLP collector | `createOtelExporter()` + `buildLoopOtelSpans()`: root export | your own OTLP serializer or pulling the OTEL SDK | -| See a **supervised tree** in a trace viewer (one span per node, opened at spawn, closed at settle, parented to its parent node; driver turns as LLM child spans) | `supervise(profile, task, { otel: { exporter } })`, or `createSupervisorSpanRecorder({ runId, … }).hooks` on `SupervisorOpts.hooks`: `/kernel` — OPT-IN, and omitting `otel` installs no hook at all | parsing the spawn journal to reconstruct the tree; a second exporter; routing replay/resume through telemetry (the journal stays the only durable record) | -| Run an ordered analyst pass where later analysts use findings from earlier analysts | `runAnalystLoop({ chainFindings: true })`: `/analyst-loop`; registration order defines the dependency order, while omission keeps analysts independent | manually invoke each analyst and pipe findings between calls | -| Know **what got mounted into a run** / **why a candidate won** | `result.provenance.mounts` / `result.provenance.selectionReceipts` (`MountManifestEntry`/`SelectionReceipt`/`RunProvenance`); declare mounts via the `recordMount` recorder in `prepareBox`: root export | re-reading box contents to reconstruct what was mounted, or re-deriving which candidate the selector picked | -| State any benchmark/A-B claim | `pairedLift(...)` (bench) over `pairedBootstrap`/`heldoutSignificance` (substrate) | your own bootstrap loop/PRNG per gate; a point lift without `low/high/pairs` | -| Let an agent **delegate ONE generic INTENT** (no fixed coder/researcher type) and get the result + real spend SYNCHRONOUSLY | the **`delegate` tool**: `createDelegateHandler` via `createMcpServer({ delegateSupervisor })`; mount it over the `agent-runtime mcp` bin with `MCP_ENABLE_DELEGATE=1` (the bin authors a supervisor over a `sandbox` backend): `/mcp` | a hardcoded coder/researcher profile, or task-specific `delegate_code`/`delegate_research` verbs (RETIRED): `delegate` is the ONE delegation path and the only one with a cost channel | -| Run a coding task INSIDE the agent's OWN sandbox session (a sibling box, fresh branch, validated patch) | `detachedSessionDelegate({ sandboxClient \| executor, workerProfile, fanoutProfiles? })`: `/mcp` (the required exact worker profile owns harness, provider, model, prompt, and tools; optional exact profiles make fanout heterogeneous) | a hardcoded coder profile baked into the delegate; `delegate()` (that spawns workers in a *chosen* backend, not the agent's own session) | -| Have a **supervisor spawn + live-drive workers in a backend you choose** and observe or steer them while the coordinator is alive | the **coordination MCP** via `createCoordinationTools` / `serveCoordinationMcp` over a live `Scope`; each worker's leaf is `createExecutor({ backend })` | `detachedSessionDelegate`, which is own-sandbox-session only and one-shot. Supervised-tree restart recovery is not implemented. | -| Let two LIVE workers of one run message each other — compare a result, challenge a claim, ask the peer that already has the fact — without the parent relaying | **peer mail**: `serveCoordinationMcp({ peerMail: true })` (or `createCoordinationTools({ peerMail })`), then mount each worker's `WorkerSpawnContext.peerMailUrl`; it serves `send_mail` / `read_mail` only, the sender is bound to the capability, and every attempt publishes a `mail` `CoordinationEvent` | free-form worker-to-worker chat, a shared file or sidecar protocol in the consumer repo (no journal, no budget, no authority marking), handing a worker the coordination URL (that mounts `steer_agent`/`stop` too), or treating a peer's assertion as verification | -| Stand up a vertical agent in the eval loop | `defineAgent(manifest)` + `createSurfaceImprovementProposer`: `/agent` | a per-vertical manifest parser, surface-validator, or bespoke findings-to-patch mapper | -| Observe + deliver Intelligence on a live agent (send RunRecords + receive certified profile/diffs) | `withIntelligence(agent, { project, target })`: `/intelligence` (proposals surfaced, never auto-applied; `effort: 'off'` proves inference-only billing) | a custom trace-wrapper, a second receive path, or hand-rolled effort/tier config | -| Turn trace evidence into one measured, review-only agent proposal | `proposeAgentImprovement({ analysis, profile, improvement, buildExperiment, placeCell })` in `/intelligence` (Runtime seals the optimizer ancestry) | manually joining analysis, optimizer ancestry, exact candidate execution, uncertainty, and candidate identity | -| Freeze a measured profile/diff plus a content-addressed code surface into one executable candidate | `buildAgentCandidateBundle(...)`, then `verifyAgentCandidateBundle(...)` at execution: `/candidate-execution` | a product callback that converts profile fields, reproduces Git diff flags, hashes bytes, or assembles `AgentCandidateBundle` by hand | -| Record approve/reject/change-request feedback against one exact proposal | `reviewAgentImprovementProposal(proposal, review)`: `/intelligence` | a mutable status row that is not bound to candidate bytes | -| Run and grade the exact signed baseline-versus-candidate matrix before review | `runAgentCandidateExperiment({ experiment, placeCell })` in `/intelligence`; use `createProtectedExactProcessCandidateExperimentExecutor(...)` for any exact-process provider with protected model grants and pass the remaining product ports as `hostPorts` | product-local pairing, retry, isolation, receipt, and comparison code | -| Run one bounded unit under a protected model grant | `runProtectedAgentCandidateModelGrant(...)`: `/candidate-execution` (Runtime resolves, reserves, activates, and settles the grant around the caller's callback) | a product-owned reserve/activate/settle wrapper or a grant spanning a whole run | -| Authorize and execute writes only for the exact measured and approved candidate | `createAgentImprovementActivation(...)`, then `executeAgentImprovementActivation(...)` with one idempotent transition in `/intelligence`; opaque profile changes use `prepareAgentImprovementProfileActivation({ stateDigest, resolveState? })`, target one complete profile identity, and restore only from product-retained state by exact digest; use `createKnowledgeImprovementActivationExecutor(...)` from `/knowledge` for one local KB | a mutable approval flag, a second per-surface approval path, a best-effort profile restore, or a write that does not persist an exact result | -| Capture and restore exact task, candidate, or memory workspace bytes | `captureAgentCandidateWorkspace(...)` + `createAgentCandidateWorkspacePort(...)`: `/candidate-execution` | a product-specific archive format, ambient `git checkout`, or a materializer that skips byte/path/mode verification | -| Fold **certified prompt additions into a system prompt you assemble yourself** (product chat routes) | `createCertifiedPromptSource({ target })` → `source.compose(base)`: `/intelligence` (cached, coalesced, fail-closed; `withIntelligence` rides the same source) | a module-scope cache + refresh-window + keep-last-known loop around `pullCertified` in product wiring | -| Produce a frozen KB candidate with runtime agents, readiness checks, and measured supervised spend | `runKnowledgeImprovementJob(options)` from `/knowledge`, then the shared activation path above after review | hand-wiring `improveKnowledgeBase` + a supervised updater, or letting candidate search write live knowledge | - -For the full export inventory (every primitive, its import path, its summary: generated, never stale), see `docs/api/primitive-catalog.md`; for per-symbol signatures, the per-module `docs/api/` pages. For the recursive atom (recursion · isolated-or-collaborative artifact · conserved budget · analysts) and the two-timescale architecture, see `docs/architecture.md`. For the profile→run→optimize→ship spine in depth, `docs/concepts.md` + `docs/learning-flywheel.md`. For the Intelligence SDK (Observe + the provable-OFF billing boundary), `docs/intelligence-sdk.md`. - -## 2.1 Which front door do I use?: the four public verbs - -§2 maps a fine-grained intent to a primitive; this is the coarse router one level up. Pick a front door by **what you hand in**. Each bottoms out at ONE function; the §2 rows above carry each one's "do NOT build" twin. Exact per-symbol signatures + line anchors live in the generated `docs/api/` pages (never stale); the file paths below are what the freshness gate protects. - -| You hand in… | Front door | Bottoms out at | What it is | -|---|---|---|---| -| a **string intent** ("fix the failing auth test"): you don't care HOW | the `delegate` tool | `delegate(intent, opts)`: `src/runtime/supervise/delegate.ts` (MCP handler `createDelegateHandler`, `src/mcp/tools/delegate.ts`) | a default authoring supervisor decomposes the intent and writes the worker profile per sub-task; synchronous, returns the delivered output + `spentTotal`. The ONE delegation path. | -| an **authored supervisor `AgentProfile`** + a task | `supervise(profile, task, opts)` | `src/runtime/supervise/supervise.ts` | the one-call LLM-brain driver over the keystone `Supervisor`, scaffolding defaulted. START HERE when you wrote the driver. | -| a **deterministic shot grammar** over a stateful tool domain | `runAgentic(opts)` | `src/runtime/strategy.ts` | runs a `Strategy` (depth/breadth/custom) through the `Supervisor`: programmatic, no LLM picking the shape. | -| a **deterministic topology combinator** (`loopUntil`/`fanout`/`verify`/`panel`/`pipeline`) over a persona | `runPersonified(options)` | `src/runtime/personify/persona.ts` | composes a persona + a `CombinatorShape` over the `Supervisor`: programmatic. | - -Rule of thumb: `delegate` = "I don't care how"; `supervise` = "I authored the driver"; `runAgentic`/`runPersonified` = "I want a deterministic topology, no LLM choosing the shape." All four run over the one `Executor` port on the conserved budget pool, so equal-compute holds by construction. - -**Two-agent patterns: compose a shape, don't hand-roll a turn loop:** - -| Pattern | Use | Bottoms out at | -|---|---|---| -| **researcher → engineer** (gather, then build) | `defineStrategy(name, body)`: both agents in one body via `ctx.shot()` + `ctx.critique()` | `src/runtime/strategy.ts:789` | -| **implement → verify** (build, then a SEPARATE checker gates it: selector ≠ judge) | `verify(spec)` as the `shape` | `src/runtime/personify/combinators.ts:333` | -| **N-judge panel** (fan judges out, merge verdicts) | `panel(spec)` as the `shape` | `src/runtime/personify/combinators.ts:273` | From e19003284569df461cb5ff0f89aeebe94b6495fa Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 16 Aug 2026 21:42:18 +0000 Subject: [PATCH 17/24] refactor: share canonical profile experiment support --- .../refactor-profile-improvement-support.py | 391 ------------------ .../refactor-profile-improvement-support.yml | 69 ---- .../authored-profile-improvement.ts | 113 +---- src/intelligence/improvement-cycle.ts | 109 +---- .../profile-improvement-experiment.ts | 133 ++++++ 5 files changed, 165 insertions(+), 650 deletions(-) delete mode 100644 .github/scripts/refactor-profile-improvement-support.py delete mode 100644 .github/workflows/refactor-profile-improvement-support.yml create mode 100644 src/intelligence/profile-improvement-experiment.ts diff --git a/.github/scripts/refactor-profile-improvement-support.py b/.github/scripts/refactor-profile-improvement-support.py deleted file mode 100644 index 7534f741..00000000 --- a/.github/scripts/refactor-profile-improvement-support.py +++ /dev/null @@ -1,391 +0,0 @@ -from __future__ import annotations - -import re -from pathlib import Path - - -def replace_once(text: str, old: str, new: str, label: str) -> str: - count = text.count(old) - if count != 1: - raise SystemExit(f"{label}: expected one match, found {count}") - return text.replace(old, new, 1) - - -def remove_import_symbols( - text: str, - *, - header_end: str, - symbols: tuple[str, ...], - label: str, -) -> str: - boundary = text.find(header_end) - if boundary < 0: - raise SystemExit(f"{label}: import boundary not found") - header = text[:boundary] - body = text[boundary:] - for symbol in symbols: - pattern = re.compile(rf"(?m)^[ \t]+(?:type[ \t]+)?{re.escape(symbol)},\n") - matches = pattern.findall(header) - if len(matches) != 1: - raise SystemExit( - f"{label}: expected one imported {symbol}, found {len(matches)}" - ) - header = pattern.sub("", header, count=1) - return header + body - - -def matching_delimiter(text: str, start: int, opening: str, closing: str) -> int: - if start >= len(text) or text[start] != opening: - raise SystemExit(f"expected {opening!r} at offset {start}") - depth = 0 - quote: str | None = None - escaped = False - line_comment = False - block_comment = False - index = start - while index < len(text): - char = text[index] - next_char = text[index + 1] if index + 1 < len(text) else "" - if line_comment: - if char == "\n": - line_comment = False - index += 1 - continue - if block_comment: - if char == "*" and next_char == "/": - block_comment = False - index += 2 - else: - index += 1 - continue - if quote is not None: - if escaped: - escaped = False - elif char == "\\": - escaped = True - elif char == quote: - quote = None - index += 1 - continue - if char == "/" and next_char == "/": - line_comment = True - index += 2 - continue - if char == "/" and next_char == "*": - block_comment = True - index += 2 - continue - if char in ("'", '"', "`"): - quote = char - index += 1 - continue - if char == opening: - depth += 1 - elif char == closing: - depth -= 1 - if depth == 0: - return index + 1 - index += 1 - raise SystemExit(f"unbalanced {opening}{closing} delimiter at offset {start}") - - -def skip_whitespace(text: str, start: int) -> int: - index = start - while index < len(text) and text[index].isspace(): - index += 1 - return index - - -def function_body_open(text: str, start: int) -> int: - parameter_open = text.index("(", start) - parameter_end = matching_delimiter(text, parameter_open, "(", ")") - cursor = skip_whitespace(text, parameter_end) - if cursor < len(text) and text[cursor] == ":": - cursor = skip_whitespace(text, cursor + 1) - if cursor < len(text) and text[cursor] == "{": - cursor = skip_whitespace(text, matching_delimiter(text, cursor, "{", "}")) - else: - body = text.find("{", cursor) - if body < 0: - raise SystemExit("function body not found after return type") - return body - if cursor >= len(text) or text[cursor] != "{": - raise SystemExit(f"function body not found at offset {cursor}") - return cursor - - -def remove_function(text: str, signature: str) -> str: - count = text.count(signature) - if count != 1: - raise SystemExit(f"{signature}: expected one function, found {count}") - start = text.index(signature) - body_open = function_body_open(text, start) - end = matching_delimiter(text, body_open, "{", "}") - while end < len(text) and text[end] == "\n": - end += 1 - return text[:start] + text[end:] - - -SUPPORT = """import { CostLedger, type CostLedgerHandle } from '@tangle-network/agent-eval' -import { - type CampaignScenarioIdentity, - campaignSplitDigestFromIdentities, -} from '@tangle-network/agent-eval/campaign' -import { - sealAgentProfileImprovementSuite, - sealAgentProfileImprovementTask, -} from '@tangle-network/agent-eval/contract' -import type { - AgentCandidateEvaluationPolicy, - AgentImprovementCost, - AgentImprovementSource, - AgentProfile, - AgentProfileImprovementMeasuredComparison, - AgentProfileImprovementSuiteInputs, - AgentProfileImprovementTask, - AgentProfileImprovementTaskMaterial, - Sha256Digest, -} from '@tangle-network/agent-interface' -import { - AGENT_IMPROVEMENT_SOURCE_METADATA_KEY, - agentImprovementSourceMetadata, - agentProfileImprovementArmSchema, - numbersApproximatelyEqual, -} from '@tangle-network/agent-interface' -import { immutableCandidateValue } from '../candidate-execution/digest' -import { - assertNoCallerOptimizationReceipt, - attachOptimizationActivationReceipt, - createOptimizationActivationReceipt, -} from './optimization-receipt' -import type { AgentImprovementProfileStateDigest } from './profile-activation' - -export interface ProfileImprovementBenchmarkInput { - tasks: [AgentProfileImprovementTaskMaterial, ...AgentProfileImprovementTaskMaterial[]] - reps: number - seeds: [number, ...number[]] - policy: AgentCandidateEvaluationPolicy -} - -export function createProfileImprovementCostLedger( - budgetUsd: number, - context = 'profile improvement', -): CostLedger { - if (!Number.isFinite(budgetUsd) || budgetUsd < 0) { - throw new Error(`${context} budgetUsd must be a non-negative finite number`) - } - return new CostLedger({ costCeilingUsd: budgetUsd }) -} - -export function profilePolicyWithBudget( - policy: AgentCandidateEvaluationPolicy, - budgetUsd: number, - context = 'profile improvement', -): AgentCandidateEvaluationPolicy { - if (policy.budgetUsd !== undefined && !numbersApproximatelyEqual(policy.budgetUsd, budgetUsd)) { - throw new Error(`${context} policy budgetUsd must equal the run budgetUsd`) - } - return { ...policy, budgetUsd } -} - -export function profilePreparationAccounting( - costLedger: CostLedgerHandle, - startedAt: number, -): { wallDurationMs: number; cost: AgentImprovementCost } { - const summary = costLedger.summary() - if (!summary.accountingComplete || summary.costProvenance.kind === 'uncaptured') { - throw new Error('profile improvement preparation cost is incomplete') - } - return { - wallDurationMs: Math.max(0, performance.now() - startedAt), - cost: { - usd: summary.costProvenance.usd, - provenance: summary.costProvenance.kind, - }, - } -} - -export function profileStateDigest( - stateDigest: AgentImprovementProfileStateDigest, - identity: string, - profile: AgentProfile, -): Sha256Digest { - return agentProfileImprovementArmSchema.parse({ - stateDigest: stateDigest({ identity, profile }), - }).stateDigest -} - -export function sealProfileImprovementBenchmark( - input: ProfileImprovementBenchmarkInput, -): AgentProfileImprovementSuiteInputs { - const tasks = input.tasks.map((task) => sealAgentProfileImprovementTask(task)) as [ - AgentProfileImprovementTask, - ...AgentProfileImprovementTask[], - ] - return sealAgentProfileImprovementSuite({ - splitDigest: campaignSplitDigestFromIdentities( - tasks.map(profileTaskScenarioIdentity), - input.reps, - ), - tasks, - reps: input.reps, - seeds: input.seeds, - }) -} - -export function profileTaskScenarioIdentity( - task: AgentProfileImprovementTask, -): CampaignScenarioIdentity { - return { - id: task.scenario.id, - kind: task.scenario.kind, - scenarioDigest: task.scenario.digest, - } -} - -export function profileImprovementMetadata( - metadata: AgentProfileImprovementMeasuredComparison['metadata'], - source: AgentImprovementSource, - optimizationReceipt?: ReturnType, -): NonNullable { - assertNoCallerOptimizationReceipt(metadata) - if (metadata && Object.hasOwn(metadata, AGENT_IMPROVEMENT_SOURCE_METADATA_KEY)) { - throw new Error( - `candidate metadata reserves '${AGENT_IMPROVEMENT_SOURCE_METADATA_KEY}' for Runtime`, - ) - } - const merged = { ...(metadata ?? {}), ...agentImprovementSourceMetadata(source) } - return optimizationReceipt - ? attachOptimizationActivationReceipt(merged, optimizationReceipt) - : immutableCandidateValue(merged) -} -""" - - -Path("src/intelligence/profile-improvement-experiment.ts").write_text(SUPPORT) - -cycle_path = Path("src/intelligence/improvement-cycle.ts") -cycle = cycle_path.read_text() -cycle = remove_import_symbols( - cycle, - header_end="\n\nexport type {", - symbols=( - "campaignSplitDigestFromIdentities", - "sealAgentProfileImprovementSuite", - "sealAgentProfileImprovementTask", - "AgentImprovementCost", - "AgentProfileImprovementTask", - "AGENT_IMPROVEMENT_SOURCE_METADATA_KEY", - "agentImprovementSourceMetadata", - "agentProfileImprovementArmSchema", - "attachOptimizationActivationReceipt", - ), - label="improvement-cycle imports", -) -shared_import = """import { - createProfileImprovementCostLedger, - profileImprovementMetadata, - profilePolicyWithBudget, - profilePreparationAccounting, - profileStateDigest, - profileTaskScenarioIdentity, - sealProfileImprovementBenchmark, -} from './profile-improvement-experiment' -""" -cycle = replace_once( - cycle, - "import type { AgentImprovementProfileStateDigest } from './profile-activation'\n", - shared_import - + "import type { AgentImprovementProfileStateDigest } from './profile-activation'\n", - "improvement-cycle shared support import", -) -for function in ( - "function createProfileImprovementCostLedger(", - "function profilePolicyWithBudget(", - "function profilePreparationAccounting(", - "function profileStateDigest(", - "function sealProfileImprovementBenchmark(", - "function profileTaskScenarioIdentity(", - "function profileImprovementMetadata(", -): - cycle = remove_function(cycle, function) -cycle_path.write_text(cycle) - -authored_path = Path("src/intelligence/authored-profile-improvement.ts") -authored = authored_path.read_text() -authored = replace_once( - authored, - "import { CostLedger } from '@tangle-network/agent-eval'\n", - "", - "authored CostLedger import", -) -authored = remove_import_symbols( - authored, - header_end="\n\n/** Lineage accepted", - symbols=( - "campaignSplitDigestFromIdentities", - "sealAgentProfileImprovementSuite", - "sealAgentProfileImprovementTask", - "AgentProfileImprovementTask", - "AGENT_IMPROVEMENT_SOURCE_METADATA_KEY", - "agentImprovementSourceMetadata", - "agentProfileImprovementArmSchema", - "numbersApproximatelyEqual", - ), - label="authored-profile imports", -) -authored = replace_once( - authored, - "import type { AgentImprovementProfileStateDigest } from './profile-activation'\n", - shared_import - + "import type { AgentImprovementProfileStateDigest } from './profile-activation'\n", - "authored shared support import", -) -authored = replace_once( - authored, - "const costLedger = createMeasurementCostLedger(options.budgetUsd)", - "const costLedger = createProfileImprovementCostLedger(\n" - " options.budgetUsd,\n" - " 'authored profile improvement',\n" - " )", - "authored cost ledger call", -) -authored = replace_once( - authored, - "const policy = profilePolicyWithBudget(options.benchmark.policy, options.budgetUsd)", - "const policy = profilePolicyWithBudget(\n" - " options.benchmark.policy,\n" - " options.budgetUsd,\n" - " 'authored profile',\n" - " )", - "authored policy call", -) -preparation_pattern = re.compile( - r" const preparation = \{\n" - r" wallDurationMs: Math\.max\(0, performance\.now\(\) - preparationStartedAt\),\n" - r" cost: \{ usd: 0, provenance: 'observed' as const \},\n" - r" \}" -) -authored, count = preparation_pattern.subn( - " const preparation = profilePreparationAccounting(costLedger, preparationStartedAt)", - authored, - count=1, -) -if count != 1: - raise SystemExit(f"authored preparation accounting: expected one match, found {count}") -authored = replace_once( - authored, - "metadata: directProfileImprovementMetadata(options.metadata, source),", - "metadata: profileImprovementMetadata(options.metadata, source),", - "authored metadata call", -) -for function in ( - "function createMeasurementCostLedger(", - "function profilePolicyWithBudget(", - "function profileStateDigest(", - "function sealProfileImprovementBenchmark(", - "function profileTaskScenarioIdentity(", - "function directProfileImprovementMetadata(", -): - authored = remove_function(authored, function) -authored_path.write_text(authored) diff --git a/.github/workflows/refactor-profile-improvement-support.yml b/.github/workflows/refactor-profile-improvement-support.yml deleted file mode 100644 index 6b0a83c6..00000000 --- a/.github/workflows/refactor-profile-improvement-support.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Deduplicate profile improvement experiment support - -on: - push: - branches: - - feat/authored-profile-candidate - -permissions: - contents: write - -jobs: - refactor: - if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-24.04 - timeout-minutes: 35 - steps: - - name: Check out branch - uses: actions/checkout@v7 - with: - ref: feat/authored-profile-candidate - fetch-depth: 0 - - - name: Set up pnpm - uses: pnpm/action-setup@v6 - - - name: Set up Node - uses: actions/setup-node@v7 - with: - node-version: 22 - cache: pnpm - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - - name: Keep the generic optimization receipt import - run: sed -i '/"attachOptimizationActivationReceipt",/d' .github/scripts/refactor-profile-improvement-support.py - - - name: Extract canonical profile experiment support - run: python .github/scripts/refactor-profile-improvement-support.py - - - name: Format changed source - run: pnpm exec biome check --write src/intelligence/profile-improvement-experiment.ts src/intelligence/improvement-cycle.ts src/intelligence/authored-profile-improvement.ts - - - name: Validate source, fixtures, behavior, and docs - run: | - pnpm run lint - pnpm run typecheck - pnpm run check:testing-fixture - pnpm test - pnpm run docs:api - pnpm run docs:freshness - - - name: Commit validated refactor - shell: bash - run: | - rm .github/workflows/refactor-profile-improvement-support.yml - rm .github/scripts/refactor-profile-improvement-support.py - git add \ - src/intelligence/profile-improvement-experiment.ts \ - src/intelligence/improvement-cycle.ts \ - src/intelligence/authored-profile-improvement.ts \ - docs/api \ - .github/workflows/refactor-profile-improvement-support.yml \ - .github/scripts/refactor-profile-improvement-support.py - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git diff --cached --quiet && { echo "No refactor changes to commit"; exit 1; } - git commit -m "refactor: share canonical profile experiment support" - git push origin HEAD:feat/authored-profile-candidate diff --git a/src/intelligence/authored-profile-improvement.ts b/src/intelligence/authored-profile-improvement.ts index d380144d..77d12c45 100644 --- a/src/intelligence/authored-profile-improvement.ts +++ b/src/intelligence/authored-profile-improvement.ts @@ -1,16 +1,10 @@ -import { CostLedger } from '@tangle-network/agent-eval' import { assertProposalFindings, type ProposalFinding } from '@tangle-network/agent-eval/analyst' -import { - type CampaignScenarioIdentity, - campaignSplitDigestFromIdentities, -} from '@tangle-network/agent-eval/campaign' +import type { CampaignScenarioIdentity } from '@tangle-network/agent-eval/campaign' import { type AgentProfileImprovementExperimentExecutionInput, measuredComparisonFromAgentProfileImprovementExperiment, runAgentProfileImprovementExperiment, sealAgentProfileImprovementExperiment, - sealAgentProfileImprovementSuite, - sealAgentProfileImprovementTask, verifyAgentProfileImprovementExperimentComparison, } from '@tangle-network/agent-eval/contract' import type { @@ -24,16 +18,9 @@ import type { AgentProfileImprovementMeasurement, AgentProfileImprovementRunReceipt, AgentProfileImprovementSuiteInputs, - AgentProfileImprovementTask, Sha256Digest, } from '@tangle-network/agent-interface' -import { - AGENT_IMPROVEMENT_SOURCE_METADATA_KEY, - agentImprovementSourceMetadata, - agentImprovementSourceSchema, - agentProfileImprovementArmSchema, - numbersApproximatelyEqual, -} from '@tangle-network/agent-interface' +import { agentImprovementSourceSchema } from '@tangle-network/agent-interface' import { canonicalCandidateDigest, immutableCandidateValue } from '../candidate-execution/digest' import { parseExactAgentProfile } from '../candidate-execution/profile' import { @@ -43,6 +30,15 @@ import { import { agentImprovementProfileDiffs } from './improvement-surfaces' import { assertNoCallerOptimizationReceipt } from './optimization-receipt' import type { AgentImprovementProfileStateDigest } from './profile-activation' +import { + createProfileImprovementCostLedger, + profileImprovementMetadata, + profilePolicyWithBudget, + profilePreparationAccounting, + profileStateDigest, + profileTaskScenarioIdentity, + sealProfileImprovementBenchmark, +} from './profile-improvement-experiment' /** Lineage accepted by the direct candidate path. Optimizer lineage belongs to `improve()`. */ export type AuthoredAgentProfileCandidateLineage = Omit< @@ -121,7 +117,10 @@ export async function proposeAuthoredAgentProfileImprovement( const findings = immutableCandidateValue([ ...assertProposalFindings(options.findings ?? [], 'authored profile improvement findings'), ]) - const costLedger = createMeasurementCostLedger(options.budgetUsd) + const costLedger = createProfileImprovementCostLedger( + options.budgetUsd, + 'authored profile improvement', + ) const preparationStartedAt = performance.now() const baselineProfile = parseExactAgentProfile(options.profile, 'authored profile baseline') const candidateProfile = parseExactAgentProfile( @@ -162,7 +161,11 @@ export async function proposeAuthoredAgentProfileImprovement( ...inputLineage, profileDiffIds, }) - const policy = profilePolicyWithBudget(options.benchmark.policy, options.budgetUsd) + const policy = profilePolicyWithBudget( + options.benchmark.policy, + options.budgetUsd, + 'authored profile', + ) const benchmark = sealProfileImprovementBenchmark({ ...options.benchmark, policy }) assertDirectCandidateReleaseWorkIsFresh(benchmark, candidateLineage, options.developmentScenarios) const experiment = sealAgentProfileImprovementExperiment({ @@ -181,10 +184,7 @@ export async function proposeAuthoredAgentProfileImprovement( [baselineStateDigest, baselineProfile], [candidateStateDigest, candidateProfile], ]) - const preparation = { - wallDurationMs: Math.max(0, performance.now() - preparationStartedAt), - cost: { usd: 0, provenance: 'observed' as const }, - } + const preparation = profilePreparationAccounting(costLedger, preparationStartedAt) const run = await runAgentProfileImprovementExperiment({ experiment, ...(options.maxConcurrency === undefined ? {} : { maxConcurrency: options.maxConcurrency }), @@ -207,7 +207,7 @@ export async function proposeAuthoredAgentProfileImprovement( generationsExplored: 0, preparation, measurement: run.measurement, - metadata: directProfileImprovementMetadata(options.metadata, source), + metadata: profileImprovementMetadata(options.metadata, source), }), ) const proposal = createAgentImprovementProposal({ @@ -225,59 +225,6 @@ export async function proposeAuthoredAgentProfileImprovement( } } -function createMeasurementCostLedger(budgetUsd: number): CostLedger { - if (!Number.isFinite(budgetUsd) || budgetUsd < 0) { - throw new Error('authored profile improvement budgetUsd must be a non-negative finite number') - } - return new CostLedger({ costCeilingUsd: budgetUsd }) -} - -function profilePolicyWithBudget( - policy: AgentProfileImprovementBenchmark['policy'], - budgetUsd: number, -): AgentProfileImprovementBenchmark['policy'] { - if (policy.budgetUsd !== undefined && !numbersApproximatelyEqual(policy.budgetUsd, budgetUsd)) { - throw new Error('authored profile policy budgetUsd must equal the run budgetUsd') - } - return { ...policy, budgetUsd } -} - -function profileStateDigest( - stateDigest: AgentImprovementProfileStateDigest, - identity: string, - profile: AgentProfile, -): Sha256Digest { - return agentProfileImprovementArmSchema.parse({ - stateDigest: stateDigest({ identity, profile }), - }).stateDigest -} - -function sealProfileImprovementBenchmark( - input: AgentProfileImprovementBenchmark, -): AgentProfileImprovementSuiteInputs { - const tasks = input.tasks.map((task) => sealAgentProfileImprovementTask(task)) as [ - AgentProfileImprovementTask, - ...AgentProfileImprovementTask[], - ] - return sealAgentProfileImprovementSuite({ - splitDigest: campaignSplitDigestFromIdentities( - tasks.map(profileTaskScenarioIdentity), - input.reps, - ), - tasks, - reps: input.reps, - seeds: input.seeds, - }) -} - -function profileTaskScenarioIdentity(task: AgentProfileImprovementTask): CampaignScenarioIdentity { - return { - id: task.scenario.id, - kind: task.scenario.kind, - scenarioDigest: task.scenario.digest, - } -} - function assertDirectCandidateReleaseWorkIsFresh( benchmark: AgentProfileImprovementSuiteInputs, lineage: AgentCandidateLineage, @@ -298,19 +245,3 @@ function assertDirectCandidateReleaseWorkIsFresh( ) } } - -function directProfileImprovementMetadata( - metadata: AgentProfileImprovementMeasuredComparison['metadata'], - source: AgentImprovementSource, -): NonNullable { - assertNoCallerOptimizationReceipt(metadata) - if (metadata && Object.hasOwn(metadata, AGENT_IMPROVEMENT_SOURCE_METADATA_KEY)) { - throw new Error( - `candidate metadata reserves '${AGENT_IMPROVEMENT_SOURCE_METADATA_KEY}' for Runtime`, - ) - } - return immutableCandidateValue({ - ...(metadata ?? {}), - ...agentImprovementSourceMetadata(source), - }) -} diff --git a/src/intelligence/improvement-cycle.ts b/src/intelligence/improvement-cycle.ts index d36ce760..baa01d21 100644 --- a/src/intelligence/improvement-cycle.ts +++ b/src/intelligence/improvement-cycle.ts @@ -1,9 +1,6 @@ import { CostLedger, type CostLedgerHandle } from '@tangle-network/agent-eval' import { assertProposalFindings, type ProposalFinding } from '@tangle-network/agent-eval/analyst' -import { - type CampaignScenarioIdentity, - campaignSplitDigestFromIdentities, -} from '@tangle-network/agent-eval/campaign' +import type { CampaignScenarioIdentity } from '@tangle-network/agent-eval/campaign' import { type AgentProfileImprovementExperimentExecutionInput, type CandidateExperimentExecutionInput, @@ -14,8 +11,6 @@ import { runCandidateExperiment, type Scenario, sealAgentProfileImprovementExperiment, - sealAgentProfileImprovementSuite, - sealAgentProfileImprovementTask, sealCandidateExperiment, verifyAgentProfileImprovementExperimentComparison, verifyCandidateExperiment, @@ -32,7 +27,6 @@ import type { AgentCandidateRunCell, AgentImprovementActivation, AgentImprovementActivationIntent, - AgentImprovementCost, AgentImprovementEvaluation, AgentImprovementMeasuredComparison, AgentImprovementProposal, @@ -46,21 +40,17 @@ import type { AgentProfileImprovementMeasurement, AgentProfileImprovementRunReceipt, AgentProfileImprovementSuiteInputs, - AgentProfileImprovementTask, AgentProfileImprovementTaskMaterial, CandidateExecutionEvidence, Sha256Digest, } from '@tangle-network/agent-interface' import { - AGENT_IMPROVEMENT_SOURCE_METADATA_KEY, agentCandidateMaterializationReceiptSchema, agentCandidateRunReceiptSchema, agentImprovementActivationSchema, agentImprovementProposalSchema, agentImprovementReviewSchema, - agentImprovementSourceMetadata, agentImprovementSourceSchema, - agentProfileImprovementArmSchema, agentProfileImprovementExecutionRefSchema, candidateExecutionEvidenceSchema, numbersApproximatelyEqual, @@ -125,6 +115,15 @@ import { optimizationActivationReceiptFromMetadata, } from './optimization-receipt' import type { AgentImprovementProfileStateDigest } from './profile-activation' +import { + createProfileImprovementCostLedger, + profileImprovementMetadata, + profilePolicyWithBudget, + profilePreparationAccounting, + profileStateDigest, + profileTaskScenarioIdentity, + sealProfileImprovementBenchmark, +} from './profile-improvement-experiment' export type { AgentImprovementActivation, @@ -569,40 +568,6 @@ function assertAnalysisCostRecorded( } } -function createProfileImprovementCostLedger(budgetUsd: number): CostLedger { - if (!Number.isFinite(budgetUsd) || budgetUsd < 0) { - throw new Error('profile improvement budgetUsd must be a non-negative finite number') - } - return new CostLedger({ costCeilingUsd: budgetUsd }) -} - -function profilePolicyWithBudget( - policy: AgentCandidateEvaluationPolicy, - budgetUsd: number, -): AgentCandidateEvaluationPolicy { - if (policy.budgetUsd !== undefined && !numbersApproximatelyEqual(policy.budgetUsd, budgetUsd)) { - throw new Error('profile improvement policy budgetUsd must equal the run budgetUsd') - } - return { ...policy, budgetUsd } -} - -function profilePreparationAccounting( - costLedger: CostLedgerHandle, - startedAt: number, -): { wallDurationMs: number; cost: AgentImprovementCost } { - const summary = costLedger.summary() - if (!summary.accountingComplete || summary.costProvenance.kind === 'uncaptured') { - throw new Error('profile improvement preparation cost is incomplete') - } - return { - wallDurationMs: Math.max(0, performance.now() - startedAt), - cost: { - usd: summary.costProvenance.usd, - provenance: summary.costProvenance.kind, - }, - } -} - function completeImprovementSearchAccounting( analysis: ImprovementSearchAccounting, improvement: { @@ -631,42 +596,6 @@ function completeImprovementSearchAccounting( } } -function profileStateDigest( - stateDigest: AgentImprovementProfileStateDigest, - identity: string, - profile: AgentProfile, -): Sha256Digest { - return agentProfileImprovementArmSchema.parse({ - stateDigest: stateDigest({ identity, profile }), - }).stateDigest -} - -function sealProfileImprovementBenchmark( - input: AgentProfileImprovementBenchmark, -): AgentProfileImprovementSuiteInputs { - const tasks = input.tasks.map((task) => sealAgentProfileImprovementTask(task)) as [ - AgentProfileImprovementTask, - ...AgentProfileImprovementTask[], - ] - return sealAgentProfileImprovementSuite({ - splitDigest: campaignSplitDigestFromIdentities( - tasks.map(profileTaskScenarioIdentity), - input.reps, - ), - tasks, - reps: input.reps, - seeds: input.seeds, - }) -} - -function profileTaskScenarioIdentity(task: AgentProfileImprovementTask): CampaignScenarioIdentity { - return { - id: task.scenario.id, - kind: task.scenario.kind, - scenarioDigest: task.scenario.digest, - } -} - function assertReleaseSplitIsFresh( heldOutSplitDigest: Sha256Digest, improvement: ImproveResult, @@ -733,24 +662,6 @@ function assertProfileReleaseWorkIsFresh( assertReleaseScenariosAreFresh(improvement, benchmark.tasks.map(profileTaskScenarioIdentity)) } -function profileImprovementMetadata( - metadata: AgentProfileImprovementMeasuredComparison['metadata'], - source: AgentImprovementSource, - optimizationReceipt: ReturnType, -): NonNullable { - assertNoCallerOptimizationReceipt(metadata) - if (metadata && Object.hasOwn(metadata, AGENT_IMPROVEMENT_SOURCE_METADATA_KEY)) { - throw new Error( - `candidate metadata reserves '${AGENT_IMPROVEMENT_SOURCE_METADATA_KEY}' for Runtime`, - ) - } - const sourceMetadata = agentImprovementSourceMetadata(source) - const merged = { ...(metadata ?? {}), ...sourceMetadata } - return optimizationReceipt - ? attachOptimizationActivationReceipt(merged, optimizationReceipt) - : immutableCandidateValue(merged) -} - /** * Analyze a product-owned profile, search one profile surface, then run the * exact baseline and candidate through the product executor before proposing. diff --git a/src/intelligence/profile-improvement-experiment.ts b/src/intelligence/profile-improvement-experiment.ts new file mode 100644 index 00000000..278b18a1 --- /dev/null +++ b/src/intelligence/profile-improvement-experiment.ts @@ -0,0 +1,133 @@ +import { CostLedger, type CostLedgerHandle } from '@tangle-network/agent-eval' +import { + type CampaignScenarioIdentity, + campaignSplitDigestFromIdentities, +} from '@tangle-network/agent-eval/campaign' +import { + sealAgentProfileImprovementSuite, + sealAgentProfileImprovementTask, +} from '@tangle-network/agent-eval/contract' +import type { + AgentCandidateEvaluationPolicy, + AgentImprovementCost, + AgentImprovementSource, + AgentProfile, + AgentProfileImprovementMeasuredComparison, + AgentProfileImprovementSuiteInputs, + AgentProfileImprovementTask, + AgentProfileImprovementTaskMaterial, + Sha256Digest, +} from '@tangle-network/agent-interface' +import { + AGENT_IMPROVEMENT_SOURCE_METADATA_KEY, + agentImprovementSourceMetadata, + agentProfileImprovementArmSchema, + numbersApproximatelyEqual, +} from '@tangle-network/agent-interface' +import { immutableCandidateValue } from '../candidate-execution/digest' +import { + assertNoCallerOptimizationReceipt, + attachOptimizationActivationReceipt, + type createOptimizationActivationReceipt, +} from './optimization-receipt' +import type { AgentImprovementProfileStateDigest } from './profile-activation' + +export interface ProfileImprovementBenchmarkInput { + tasks: [AgentProfileImprovementTaskMaterial, ...AgentProfileImprovementTaskMaterial[]] + reps: number + seeds: [number, ...number[]] + policy: AgentCandidateEvaluationPolicy +} + +export function createProfileImprovementCostLedger( + budgetUsd: number, + context = 'profile improvement', +): CostLedger { + if (!Number.isFinite(budgetUsd) || budgetUsd < 0) { + throw new Error(`${context} budgetUsd must be a non-negative finite number`) + } + return new CostLedger({ costCeilingUsd: budgetUsd }) +} + +export function profilePolicyWithBudget( + policy: AgentCandidateEvaluationPolicy, + budgetUsd: number, + context = 'profile improvement', +): AgentCandidateEvaluationPolicy { + if (policy.budgetUsd !== undefined && !numbersApproximatelyEqual(policy.budgetUsd, budgetUsd)) { + throw new Error(`${context} policy budgetUsd must equal the run budgetUsd`) + } + return { ...policy, budgetUsd } +} + +export function profilePreparationAccounting( + costLedger: CostLedgerHandle, + startedAt: number, +): { wallDurationMs: number; cost: AgentImprovementCost } { + const summary = costLedger.summary() + if (!summary.accountingComplete || summary.costProvenance.kind === 'uncaptured') { + throw new Error('profile improvement preparation cost is incomplete') + } + return { + wallDurationMs: Math.max(0, performance.now() - startedAt), + cost: { + usd: summary.costProvenance.usd, + provenance: summary.costProvenance.kind, + }, + } +} + +export function profileStateDigest( + stateDigest: AgentImprovementProfileStateDigest, + identity: string, + profile: AgentProfile, +): Sha256Digest { + return agentProfileImprovementArmSchema.parse({ + stateDigest: stateDigest({ identity, profile }), + }).stateDigest +} + +export function sealProfileImprovementBenchmark( + input: ProfileImprovementBenchmarkInput, +): AgentProfileImprovementSuiteInputs { + const tasks = input.tasks.map((task) => sealAgentProfileImprovementTask(task)) as [ + AgentProfileImprovementTask, + ...AgentProfileImprovementTask[], + ] + return sealAgentProfileImprovementSuite({ + splitDigest: campaignSplitDigestFromIdentities( + tasks.map(profileTaskScenarioIdentity), + input.reps, + ), + tasks, + reps: input.reps, + seeds: input.seeds, + }) +} + +export function profileTaskScenarioIdentity( + task: AgentProfileImprovementTask, +): CampaignScenarioIdentity { + return { + id: task.scenario.id, + kind: task.scenario.kind, + scenarioDigest: task.scenario.digest, + } +} + +export function profileImprovementMetadata( + metadata: AgentProfileImprovementMeasuredComparison['metadata'], + source: AgentImprovementSource, + optimizationReceipt?: ReturnType, +): NonNullable { + assertNoCallerOptimizationReceipt(metadata) + if (metadata && Object.hasOwn(metadata, AGENT_IMPROVEMENT_SOURCE_METADATA_KEY)) { + throw new Error( + `candidate metadata reserves '${AGENT_IMPROVEMENT_SOURCE_METADATA_KEY}' for Runtime`, + ) + } + const merged = { ...(metadata ?? {}), ...agentImprovementSourceMetadata(source) } + return optimizationReceipt + ? attachOptimizationActivationReceipt(merged, optimizationReceipt) + : immutableCandidateValue(merged) +} From 1e1c35f6dc60a2efb6bd5edb639602fd742470bc Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 14:58:53 -0700 Subject: [PATCH 18/24] test: harden authored profile provenance boundaries --- tests/authored-profile-improvement.test.ts | 55 +++++++++++++++++++++- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/tests/authored-profile-improvement.test.ts b/tests/authored-profile-improvement.test.ts index d47ee055..bdd684d0 100644 --- a/tests/authored-profile-improvement.test.ts +++ b/tests/authored-profile-improvement.test.ts @@ -1,6 +1,9 @@ import { minimumPairsForPairedDeltaTest, type ProposalFinding } from '@tangle-network/agent-eval' import type { CampaignScenarioIdentity } from '@tangle-network/agent-eval/campaign' -import type { AgentProfile } from '@tangle-network/agent-interface' +import { + AGENT_IMPROVEMENT_SOURCE_METADATA_KEY, + type AgentProfile, +} from '@tangle-network/agent-interface' import { describe, expect, it } from 'vitest' import { canonicalCandidateDigest } from '../src/candidate-execution/digest' @@ -130,6 +133,7 @@ function setup() { describe('authored profile improvement', { timeout: 30_000 }, () => { it('measures a human-authored complete profile through the canonical proposal path', async () => { const fixture = setup() + const inputLineage = structuredClone(fixture.options.candidateLineage) const result = await proposeAuthoredAgentProfileImprovement(fixture.options) @@ -160,7 +164,23 @@ describe('authored profile improvement', { timeout: 30_000 }, () => { parentDigests: [fixture.baselineStateDigest], runIds: ['reflection-1'], }) - expect(result.candidateLineage.profileDiffIds?.length).toBeGreaterThan(0) + expect(result.candidateLineage.profileDiffIds).toEqual([ + 'human-reflection-1:agent-profile:reset', + 'human-reflection-1:agent-profile:set', + ]) + expect(result.experiment.change.map((step) => step.id)).toEqual( + result.candidateLineage.profileDiffIds, + ) + expect( + result.experiment.change.every( + (step) => + step.source?.kind === 'frontier-author' && + step.metadata?.sourceIdentity === fixture.options.source.sourceIdentity && + step.metadata?.sourceRevision === fixture.options.source.sourceRevision, + ), + ).toBe(true) + expect(fixture.options.candidateLineage).toEqual(inputLineage) + expect(Object.hasOwn(fixture.options.candidateLineage, 'profileDiffIds')).toBe(false) expect(result.experiment.candidateLineage).toEqual(result.candidateLineage) expect(result.proposal.evaluation.decision.outcome).toBe('ship') expect(result.proposal.changedSurfaces).toEqual([ @@ -215,6 +235,37 @@ describe('authored profile improvement', { timeout: 30_000 }, () => { ) }) + it('refuses forged metadata and invalid or mismatched budgets before execution', async () => { + const forgedMetadata = setup() + forgedMetadata.options.metadata = { + [AGENT_IMPROVEMENT_SOURCE_METADATA_KEY]: 'caller-controlled-source', + } + await expect(proposeAuthoredAgentProfileImprovement(forgedMetadata.options)).rejects.toThrow( + /reserves/, + ) + expect(forgedMetadata.observed).toHaveLength(0) + + const invalidBudget = setup() + invalidBudget.options.budgetUsd = Number.NaN + await expect(proposeAuthoredAgentProfileImprovement(invalidBudget.options)).rejects.toThrow( + /non-negative finite number/, + ) + expect(invalidBudget.observed).toHaveLength(0) + + const mismatchedBudget = setup() + mismatchedBudget.options.benchmark = { + ...mismatchedBudget.options.benchmark, + policy: { + ...mismatchedBudget.options.benchmark.policy, + budgetUsd: mismatchedBudget.options.budgetUsd + 1, + }, + } + await expect(proposeAuthoredAgentProfileImprovement(mismatchedBudget.options)).rejects.toThrow( + /policy budgetUsd must equal/, + ) + expect(mismatchedBudget.observed).toHaveLength(0) + }) + it('refuses unchanged candidates, source drift, and reused held-out scenarios', async () => { const unchanged = setup() unchanged.options.candidateProfile = unchanged.baselineProfile From 06239e75985bbb244ac4f2590dbf5674bcf1b235 Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 15:13:55 -0700 Subject: [PATCH 19/24] chore(release): bump authored profile candidate to 0.138.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 94d135a7..50473141 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.138.0", + "version": "0.138.1", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { From 3dcfbd9b8e232ca218d30e43095308dba066a368 Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 15:32:53 -0700 Subject: [PATCH 20/24] fix(intelligence): validate authored metadata before measurement --- src/intelligence/authored-profile-improvement.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/intelligence/authored-profile-improvement.ts b/src/intelligence/authored-profile-improvement.ts index 77d12c45..a73b56bf 100644 --- a/src/intelligence/authored-profile-improvement.ts +++ b/src/intelligence/authored-profile-improvement.ts @@ -28,7 +28,6 @@ import { createAgentImprovementProposal, } from './improvement-cycle' import { agentImprovementProfileDiffs } from './improvement-surfaces' -import { assertNoCallerOptimizationReceipt } from './optimization-receipt' import type { AgentImprovementProfileStateDigest } from './profile-activation' import { createProfileImprovementCostLedger, @@ -105,8 +104,11 @@ export interface ProposeAuthoredAgentProfileImprovementResult { export async function proposeAuthoredAgentProfileImprovement( options: ProposeAuthoredAgentProfileImprovementOptions, ): Promise { - assertNoCallerOptimizationReceipt(options.metadata) const source = agentImprovementSourceSchema.parse(options.source) + // Validate and seal caller metadata before allocating a cost ledger or + // invoking the product-owned executor. Reserved provenance fields and forged + // optimizer receipts must fail closed without spending measurement budget. + const metadata = profileImprovementMetadata(options.metadata, source) const inputLineage = options.candidateLineage as AgentCandidateLineage if (inputLineage.source === 'optimizer') { throw new Error('authored profile improvement refuses optimizer lineage; use improve()') @@ -207,7 +209,7 @@ export async function proposeAuthoredAgentProfileImprovement( generationsExplored: 0, preparation, measurement: run.measurement, - metadata: profileImprovementMetadata(options.metadata, source), + metadata, }), ) const proposal = createAgentImprovementProposal({ From 005d4cb6e6ec0dca3f693c314bb8ec62503336ad Mon Sep 17 00:00:00 2001 From: drewstone Date: Sun, 16 Aug 2026 15:37:16 -0700 Subject: [PATCH 21/24] chore(testing): regenerate authored profile fixtures --- src/testing/fixtures/agent-improvement-proposal.json | 10 +++++----- .../fixtures/agent-profile-improvement-proposal.json | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index 64625ea6..8c091ec8 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:b84cc224522ae0a07db1f1de41c8244899a00f0c522dc4f6762d708c111dd2b8", + "digest": "sha256:96aa602a3f9d198264af652f5bba5cd51fb54519aaae91c28894eef2b6e694e5", "evaluation": { "decision": { "contributingChecks": [ @@ -4882,7 +4882,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.138.0" + "runtimeVersion": "0.138.1" }, "objectives": [ { @@ -4993,8 +4993,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:4d748710e2d77cea5e0fe66a57d5fb6ddc25d325f65139884e040d1af3ce2d4d", - "runId": "agent-runtime-0.138.0-proposal-fixture", + "recordDigest": "sha256:5b1be97ac62edbe65f899224398159b1fbfc9a768d25a5d79044de6e53f76600", + "runId": "agent-runtime-0.138.1-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -5021,5 +5021,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.138.0-proposal-fixture" + "runId": "agent-runtime-0.138.1-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index ff6388fe..edbe8cbd 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:b89b24bfcd014e7582f9e6d7797feea0c6ae0f3a7be6cfaa4cacdf3396dee908", + "digest": "sha256:f98a80755978d5cfbca957967b1a97f29af8a974bf920746fda8b3deddf38304", "evaluation": { "decision": { "contributingChecks": [ @@ -1715,7 +1715,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.138.0" + "runtimeVersion": "0.138.1" }, "objectives": [ { @@ -1826,7 +1826,7 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:4e32d77d314c15b9df9d33aaaf0468a29d63dfb10eef0d6e215a4f36f473e7a6", + "recordDigest": "sha256:16e8733160306657bd773d677c478519634bb4360f0573484e6c5e85e4dcc6d4", "runId": "profile-improvement-1", "schema": "agent-profile-improvement-experiment" } From 9f237f46a30e1fabd75ff69035f4e2a837f92fa5 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sun, 16 Aug 2026 19:13:33 -0600 Subject: [PATCH 22/24] chore(release): 0.139.0 for the authored profile path The direct candidate path is a new consumer-visible surface, so it ships under a minor. 0.138.1 is taken by the Eval 0.146.0 release. --- docs/api/primitive-catalog.md | 11 ++++++++--- docs/canonical-api.md | 2 +- package.json | 2 +- src/testing/fixtures/agent-improvement-proposal.json | 10 +++++----- .../fixtures/agent-profile-improvement-proposal.json | 6 +++--- 5 files changed, 18 insertions(+), 13 deletions(-) diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 8bb98f71..82895873 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.138.0` and `@tangle-network/agent-eval@0.145.21` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.139.0` and `@tangle-network/agent-eval@0.145.21` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -393,7 +393,7 @@ Import from `@tangle-network/agent-runtime/tool-loop` — 12 exports. ### Intelligence SDK — Observe + provable-OFF billing -Import from `@tangle-network/agent-runtime/intelligence` — 166 exports. +Import from `@tangle-network/agent-runtime/intelligence` — 172 exports. | Symbol | Kind | Summary | |---|---|---| @@ -428,6 +428,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 166 exports. | `prepareAgentImprovementProfileActivation` | function | Compare product-owned profiles with an exact measured transition and prepare | | `proposeAgentImprovement` | function | Analyze, search, then remeasure the resulting exact candidate before proposing it. | | `proposeAgentProfileImprovement` | function | Analyze a product-owned profile, search one profile surface, then run the | +| `proposeAuthoredAgentProfileImprovement` | function | Put a complete authored/imported profile through the canonical profile | | `pullCertified` | function | Pull the certified composed profile for a target. Fail-closed: a network | | `resolveEffort` | function | Compile a named tier (plus optional per-field overrides) into the flat | | `resolveIntelligenceBaseUrl` | function | Resolve the ONE Intelligence base URL — the single knob both the send and | @@ -451,6 +452,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 166 exports. | `AgentImprovementActivationResult` | interface | Immutable outcome of one idempotent, transaction-wide activation attempt. | | `AgentImprovementMeasuredComparison` | interface | Portable paired held-out comparison produced by a sealed candidate executor. | | `AgentImprovementReview` | interface | Human or tenant-policy decision bound to one exact proposal. | +| `AgentProfileCandidateMeasurementExecutor` | interface | Product-owned executor for exact baseline/candidate profile measurement. | | `AgentProfileImprovementBenchmark` | interface | Product-owned task material that Runtime freezes before either profile state runs. | | `AgentProfileImprovementExecutor` | interface | One product execution adapter shared by optimizer search and exact profile | | `AppliedIntelligence` | interface | What the hook hands the agent each run. Additive over the prompt-only | @@ -478,6 +480,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 166 exports. | `ModeReadiness` | interface | One mode's readiness verdict. | | `ProfileImprovementActivationTransitionInput` | interface | A measured profile change without raw profile bytes. | | `ProposeAgentProfileImprovementOptions` | interface | Complete profile-improvement path for a product-owned source. | +| `ProposeAuthoredAgentProfileImprovementOptions` | interface | Measure a complete human-authored, imported, or compound profile candidate. | | `ProposedProfileDiff` | interface | A gate-certified profile diff the plane has already promoted, plus the | | `ProtectedExactProcessCandidateExperimentExecutor` | interface | Exact-process executor plus the ports required for durable recovery. | | `ProvisionedHost` | interface | A live, provisioned host the resolver tore up for a `process-on-infra` arm. | @@ -504,6 +507,8 @@ Import from `@tangle-network/agent-runtime/intelligence` — 166 exports. | `AgentImprovementProfileStateResolver` | type | Product-owned retained-state lookup used only for an explicit restore. | | `AgentImprovementProposalSubmissionState` | type | What Runtime knows about a failed proposal submission. | | `AgentProfileImprovementMethodOptions` | type | The portable profile changes that the measured-profile contract permits. | +| `AuthoredAgentProfileCandidateLineage` | type | Lineage accepted by the direct candidate path. Optimizer lineage belongs to `improve()`. | +| `AuthoredAgentProfileDiffOptions` | type | Provenance attached while Runtime derives the exact profile diff. | | `CapabilityAuth` | type | How a binding authenticates at resolve time. Declared as a REQUIREMENT in the | | `CapabilityInterface` | type | What the agent consumes. CLOSED — a new runtime kind NEVER extends this. Each | | `CapabilitySurface` | type | Every interface surface tag — the closed set the resolver fans into slots. | @@ -521,7 +526,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 166 exports. | `SubmitAgentImprovementProposalOutcome` | type | Typed result for proposal submission. A successful result contains the | | `UsageClass` | type | Usage class for billing. Base-stream tokens bill `'inference'`; every | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentCandidateExperimentCellPlacement`, `AgentImprovementActivationResultStore`, `AgentImprovementActivationTargetPlan`, `AgentImprovementProfileReplacement`, `AgentImprovementProfileStateDigestInput`, `AgentImprovementProfileStateResolverInput`, `AgentImprovementProposal`, `AgentImprovementTargetProfileDiffOptions`, `AgentProfileImprovementActivationTargetPlan`, `CreateAgentImprovementActivationOptions`, `CreateAgentImprovementActivationResultOptions`, `CreateAgentImprovementProposalOptions`, `CreateExactProcessCandidateExperimentExecutorOptions`, `ExactProcessCandidateExperimentExecution`, `ExactProcessCandidateExperimentExecutor`, `ExecuteAgentCandidateExperimentCellOptions`, `ExecuteAgentImprovementActivationInput`, `ExecuteAgentImprovementActivationOptions`, `OptimizationActivationReceipt`, `OptimizationReceiptCost`, `ProposeAgentImprovementOptions`, `ProposeAgentImprovementResult`, `ProposeAgentProfileImprovementResult`, `PullCertifiedOptions`, `ReviewAgentImprovementInput`, `RunAgentCandidateExperimentOptions`, `RunAgentCandidateExperimentResult`, `SealedCandidateActivationTransitionInput`, `VerifyCandidateExecutionEvidenceOptions`, `AgentImprovementActivationIntent`, `AgentImprovementActivationOutcome`, `AgentImprovementActivationTargetIdentity`, `AgentImprovementActivationTransitionInput`, `AgentImprovementAnalysisOptions`, `AgentImprovementProfileActivationInput`, `AgentImprovementProfileActivationPreparation`, `AgentImprovementProfileActivationTarget`, `AgentImprovementProfileSurface`, `AgentImprovementProfileTargetState`, `AgentImprovementProfileTargetTransition`, `AgentImprovementReviewDecision`, `AgentProfileImprovementActivationOperation`, `AgentProfileMeasuredSurface`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentCandidateExperimentCellPlacement`, `AgentImprovementActivationResultStore`, `AgentImprovementActivationTargetPlan`, `AgentImprovementProfileReplacement`, `AgentImprovementProfileStateDigestInput`, `AgentImprovementProfileStateResolverInput`, `AgentImprovementProposal`, `AgentImprovementTargetProfileDiffOptions`, `AgentProfileImprovementActivationTargetPlan`, `CreateAgentImprovementActivationOptions`, `CreateAgentImprovementActivationResultOptions`, `CreateAgentImprovementProposalOptions`, `CreateExactProcessCandidateExperimentExecutorOptions`, `ExactProcessCandidateExperimentExecution`, `ExactProcessCandidateExperimentExecutor`, `ExecuteAgentCandidateExperimentCellOptions`, `ExecuteAgentImprovementActivationInput`, `ExecuteAgentImprovementActivationOptions`, `OptimizationActivationReceipt`, `OptimizationReceiptCost`, `ProposeAgentImprovementOptions`, `ProposeAgentImprovementResult`, `ProposeAgentProfileImprovementResult`, `ProposeAuthoredAgentProfileImprovementResult`, `PullCertifiedOptions`, `ReviewAgentImprovementInput`, `RunAgentCandidateExperimentOptions`, `RunAgentCandidateExperimentResult`, `SealedCandidateActivationTransitionInput`, `VerifyCandidateExecutionEvidenceOptions`, `AgentImprovementActivationIntent`, `AgentImprovementActivationOutcome`, `AgentImprovementActivationTargetIdentity`, `AgentImprovementActivationTransitionInput`, `AgentImprovementAnalysisOptions`, `AgentImprovementProfileActivationInput`, `AgentImprovementProfileActivationPreparation`, `AgentImprovementProfileActivationTarget`, `AgentImprovementProfileSurface`, `AgentImprovementProfileTargetState`, `AgentImprovementProfileTargetTransition`, `AgentImprovementReviewDecision`, `AgentProfileImprovementActivationOperation`, `AgentProfileMeasuredSurface`. ### Execution kernel — recursive atom, supervision, executors, round-synchronous loop diff --git a/docs/canonical-api.md b/docs/canonical-api.md index bd61c773..e236cc44 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,7 +4,7 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.138.0.** +> **Version 0.139.0.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. > `agent-eval` must satisfy `>=0.145.21 <0.146.0`. > `sandbox` must satisfy `>=0.27.1 <0.28.0`. diff --git a/package.json b/package.json index 50473141..6a30c80a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.138.1", + "version": "0.139.0", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index 8c091ec8..3b430b8e 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:96aa602a3f9d198264af652f5bba5cd51fb54519aaae91c28894eef2b6e694e5", + "digest": "sha256:8681ef365ff523163b585e6bec22b7e6b0de652647675763d97da8ecb0c423b4", "evaluation": { "decision": { "contributingChecks": [ @@ -4882,7 +4882,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.138.1" + "runtimeVersion": "0.139.0" }, "objectives": [ { @@ -4993,8 +4993,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:5b1be97ac62edbe65f899224398159b1fbfc9a768d25a5d79044de6e53f76600", - "runId": "agent-runtime-0.138.1-proposal-fixture", + "recordDigest": "sha256:b5fc7a2d1c5ed0a168afdb6393ab9aa7090da6db00f11278f2ab39766963aa97", + "runId": "agent-runtime-0.139.0-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -5021,5 +5021,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.138.1-proposal-fixture" + "runId": "agent-runtime-0.139.0-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index edbe8cbd..88722142 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:f98a80755978d5cfbca957967b1a97f29af8a974bf920746fda8b3deddf38304", + "digest": "sha256:35fd03f0e77f5f251219ee6c4e552590d50ba92a222da20b75d2a98cdbb29854", "evaluation": { "decision": { "contributingChecks": [ @@ -1715,7 +1715,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.138.1" + "runtimeVersion": "0.139.0" }, "objectives": [ { @@ -1826,7 +1826,7 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:16e8733160306657bd773d677c478519634bb4360f0573484e6c5e85e4dcc6d4", + "recordDigest": "sha256:07205e57acf8fbe5927cb53da608ce553093efa116deebbcef1ca117e0bc53c1", "runId": "profile-improvement-1", "schema": "agent-profile-improvement-experiment" } From b266bdaabf7708eedf9a1988616bb04cadaec973 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sun, 16 Aug 2026 19:15:27 -0600 Subject: [PATCH 23/24] docs(canonical-api): state the 0.139.0 version --- docs/canonical-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 11f96d1f..e6fe769f 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,7 +4,7 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.138.1.** +> **Version 0.139.0.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. > `agent-eval` must satisfy `>=0.146.0 <0.147.0`. > `sandbox` must satisfy `>=0.27.1 <0.28.0`. From 42a9f62a87d11815ca605061d9bddbe05dc3336a Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sun, 16 Aug 2026 19:20:52 -0600 Subject: [PATCH 24/24] chore(testing): regenerate the fixtures at 0.139.0 --- src/testing/fixtures/agent-improvement-proposal.json | 10 +++++----- .../fixtures/agent-profile-improvement-proposal.json | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index 8c091ec8..3b430b8e 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:96aa602a3f9d198264af652f5bba5cd51fb54519aaae91c28894eef2b6e694e5", + "digest": "sha256:8681ef365ff523163b585e6bec22b7e6b0de652647675763d97da8ecb0c423b4", "evaluation": { "decision": { "contributingChecks": [ @@ -4882,7 +4882,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.138.1" + "runtimeVersion": "0.139.0" }, "objectives": [ { @@ -4993,8 +4993,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:5b1be97ac62edbe65f899224398159b1fbfc9a768d25a5d79044de6e53f76600", - "runId": "agent-runtime-0.138.1-proposal-fixture", + "recordDigest": "sha256:b5fc7a2d1c5ed0a168afdb6393ab9aa7090da6db00f11278f2ab39766963aa97", + "runId": "agent-runtime-0.139.0-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -5021,5 +5021,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.138.1-proposal-fixture" + "runId": "agent-runtime-0.139.0-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index edbe8cbd..88722142 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:f98a80755978d5cfbca957967b1a97f29af8a974bf920746fda8b3deddf38304", + "digest": "sha256:35fd03f0e77f5f251219ee6c4e552590d50ba92a222da20b75d2a98cdbb29854", "evaluation": { "decision": { "contributingChecks": [ @@ -1715,7 +1715,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.138.1" + "runtimeVersion": "0.139.0" }, "objectives": [ { @@ -1826,7 +1826,7 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:16e8733160306657bd773d677c478519634bb4360f0573484e6c5e85e4dcc6d4", + "recordDigest": "sha256:07205e57acf8fbe5927cb53da608ce553093efa116deebbcef1ca117e0bc53c1", "runId": "profile-improvement-1", "schema": "agent-profile-improvement-experiment" }