diff --git a/docs/api/durable.md b/docs/api/durable.md index a93e3bd9..b9ab24e7 100644 --- a/docs/api/durable.md +++ b/docs/api/durable.md @@ -269,6 +269,61 @@ NDJSON body to return as the platform `Response` body. Content type for the response. +*** + +### DurableCoordinationStreamIdentity + +#### Properties + +##### runId + +> `readonly` **runId**: `string` + +##### ownerIds + +> `readonly` **ownerIds**: readonly `string`[] + +Exact owner ids present in the side-log, sorted for deterministic display. + +##### unscopedRecords + +> `readonly` **unscopedRecords**: `number` + +Records written before owner-scoped coordination identities were introduced. + +##### recordCount + +> `readonly` **recordCount**: `number` + +*** + +### DurableSupervisionDiscovery + +Identities discoverable from one `supervise({ runDir })` directory without +already knowing the root node or coordination run id stored inside it. + +#### Properties + +##### runDir + +> `readonly` **runDir**: `string` + +##### spawnJournalPath + +> `readonly` **spawnJournalPath**: `string` + +##### coordinationLogPath + +> `readonly` **coordinationLogPath**: `string` + +##### roots + +> `readonly` **roots**: readonly `string`[] + +##### coordinationStreams + +> `readonly` **coordinationStreams**: readonly [`DurableCoordinationStreamIdentity`](#durablecoordinationstreamidentity)[] + ## Functions ### handleChatTurn() @@ -342,3 +397,28 @@ Wire integration: #### Throws `RangeError` when `turnIndex` is invalid or the result exceeds 256 bytes. + +*** + +### discoverDurableSupervisionRun() + +> **discoverDurableSupervisionRun**(`runDir`): `Promise`\<[`DurableSupervisionDiscovery`](#durablesupervisiondiscovery)\> + +Discover the stable identities recorded by Runtime's durable supervision +files. This is the developer-facing first step before calling +`FileSpawnJournal.loadTree(root)`, `loadSpawnForest(journal, root)`, or +`FileCoordinationLog.load(runId, ownerId)`. + +Missing files produce empty collections. A malformed committed JSONL record +still fails loud through the same parser used by the runtime; a torn final +append is ignored because it was never acknowledged as committed. + +#### Parameters + +##### runDir + +`string` + +#### Returns + +`Promise`\<[`DurableSupervisionDiscovery`](#durablesupervisiondiscovery)\> diff --git a/docs/api/index.md b/docs/api/index.md index e100e2e4..7a4c6121 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -6144,6 +6144,105 @@ Exact materialized profile presented for validation before any candidate run. *** +### CreateProfileImprovementHarnessOptions + +#### Type Parameters + +##### TScenario + +`TScenario` *extends* `Scenario` + +##### TArtifact + +`TArtifact` + +#### Properties + +##### profile + +> **profile**: `AgentProfile` + +Exact baseline profile. It is parsed, detached, and frozen at construction. + +##### executionRef + +> **executionRef**: `` `sha256:${string}` `` + +Immutable identity of the bound executor, models, tools, component mapping, +and every closure or external setting that can change measured behavior. + +##### agent + +> **agent**: [`ImproveProfileAgent`](#improveprofileagent)\<`TScenario`, `TArtifact`\> + +Execute one exact materialized profile on one scenario. + +##### validateCandidate? + +> `optional` **validateCandidate?**: [`ImproveCandidateValidator`](#improvecandidatevalidator) + +Optional validator shared by every run from this harness. + +*** + +### ProfileImprovementHarness + +A small, reusable front door over `improve(profile, options)`. + +The harness freezes the baseline and binds execution identity once, which +removes the two easiest sources of accidental experiment drift when a +developer runs several methods, surfaces, or held-out suites against the +same agent. It does not replace or narrow `improve`; callers retain every +method option and may still use the lower-level API directly. + +#### Type Parameters + +##### TScenario + +`TScenario` *extends* `Scenario` + +##### TArtifact + +`TArtifact` + +#### Properties + +##### profile + +> `readonly` **profile**: `object` + +Detached immutable baseline actually used by every run. + +##### profileDigest + +> `readonly` **profileDigest**: `` `sha256:${string}` `` + +Canonical digest of the bound baseline profile. + +##### executionRef + +> `readonly` **executionRef**: `` `sha256:${string}` `` + +Exact execution identity bound at construction. + +#### Methods + +##### run() + +> **run**(`options`): `Promise`\<[`ImproveMethodResult`](#improvemethodresult)\> + +###### Parameters + +###### options + +[`ProfileImprovementHarnessRunOptions`](#profileimprovementharnessrunoptions)\<`TScenario`, `TArtifact`\> + +###### Returns + +`Promise`\<[`ImproveMethodResult`](#improvemethodresult)\> + +*** + ### RawTraceDistillerOptions #### Properties @@ -11364,6 +11463,30 @@ Official SkillOpt configuration plus bounded Runtime findings context. *** +### ProfileImprovementHarnessRunOptions + +> **ProfileImprovementHarnessRunOptions**\<`TScenario`, `TArtifact`\> = `Omit`\<[`ImproveMethodOptions`](#improvemethodoptions)\<`TScenario`, `TArtifact`\>, `"executionRef"` \| `"agent"` \| `"validateCandidate"`\> & `object` + +#### Type Declaration + +##### validateCandidate? + +> `optional` **validateCandidate?**: [`ImproveCandidateValidator`](#improvecandidatevalidator) + +Override the harness-level validator for this run. + +#### Type Parameters + +##### TScenario + +`TScenario` *extends* `Scenario` + +##### TArtifact + +`TArtifact` + +*** + ### DeepReadonly > **DeepReadonly**\<`T`\> = `T` *extends* (...`args`) => `unknown` ? `T` : `T` *extends* readonly infer TItem[] ? readonly [`DeepReadonly`](#deepreadonly)\<`TItem`\>[] : `T` *extends* `object` ? `{ readonly [TKey in keyof T]: DeepReadonly }` : `T` @@ -12620,6 +12743,41 @@ to the strategy contract (author-blind, conserved budget, one module out). *** +### PROMPT\_INSTRUCTION\_COMPONENT\_PREFIX + +> `const` **PROMPT\_INSTRUCTION\_COMPONENT\_PREFIX**: `"prompt.instruction:"` = `'prompt.instruction:'` + +Stable component-name prefix used for `profile.prompt.instructions`. + +*** + +### promptInstructionsProfileComponents + +> `const` **promptInstructionsProfileComponents**: [`ImproveProfileComponents`](#improveprofilecomponents) + +Canonical `ImproveProfileComponents` mapping for the ordered +`AgentProfile.prompt.instructions` list. + +Use it with `surface: 'agent-profile'` when an optimizer should rewrite the +exact instruction texts without being allowed to change their count, order, +labels, or any unrelated profile field: + +```ts +await improve(profile, { + surface: 'agent-profile', + profileComponents: promptInstructionsProfileComponents, + // method, scenarios, judge, executionRef, agent, ... +}) +``` + +Component names are zero-padded and stable. Runtime's existing component +materializer requires every candidate to preserve the exact key set and +verifies that `apply(read(profile))` reproduces the baseline profile. A +profile with no prompt instructions is refused rather than inventing a +sentinel instruction that could accidentally ship. + +*** + ### ROLLOUT\_POLICY\_EXTENSION > `const` **ROLLOUT\_POLICY\_EXTENSION**: `"structural-rollout"` = `'structural-rollout'` @@ -13954,6 +14112,36 @@ Build a complete method backed by Microsoft's official SkillOpt trainer. *** +### createProfileImprovementHarness() + +> **createProfileImprovementHarness**\<`TScenario`, `TArtifact`\>(`options`): [`ProfileImprovementHarness`](#profileimprovementharness)\<`TScenario`, `TArtifact`\> + +Bind one exact profile and executor into a repeatable self-improvement +harness. The returned `run` method remains generic over every existing +profile surface, optimization method, split, gate, and budget option. + +#### Type Parameters + +##### TScenario + +`TScenario` *extends* `Scenario` + +##### TArtifact + +`TArtifact` + +#### Parameters + +##### options + +[`CreateProfileImprovementHarnessOptions`](#createprofileimprovementharnessoptions)\<`TScenario`, `TArtifact`\> + +#### Returns + +[`ProfileImprovementHarness`](#profileimprovementharness)\<`TScenario`, `TArtifact`\> + +*** + ### rawTraceDistiller() > **rawTraceDistiller**\<`TScenario`, `TArtifact`\>(`options?`): (`input`) => `Promise`\ diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 629be290..e56c3a0b 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -15,7 +15,7 @@ Every subpath this package declares in `package.json` `exports`. Reach for these ### Root — task lifecycle, conversation, RSI verbs, observability -Import from `@tangle-network/agent-runtime` — 431 exports. +Import from `@tangle-network/agent-runtime` — 437 exports. | Symbol | Kind | Summary | |---|---|---| @@ -48,6 +48,7 @@ Import from `@tangle-network/agent-runtime` — 431 exports. | `createOpenInferenceFileExporter` | function | Create an exporter that APPENDS spans to a local OpenInference-JSONL file, one complete span per | | `createOtelExporter` | function | Create an OTEL exporter. Returns undefined when no endpoint is configured. | | `createProfileExecutionBackend` | function | Bind one exact profile and Runtime executor to the stable `AgentExecutionBackend` contract used | +| `createProfileImprovementHarness` | function | Bind one exact profile and executor into a repeatable self-improvement | | `createProtectedAgentCandidateModelPort` | function | Bind a protected model-grant service to the immutable candidate runtime. | | `createRuntimeEventCollector` | function | Build an in-memory collector that sanitizes and accumulates `AgentRuntimeEvent`s for inspection. | | `createRuntimeStreamEventCollector` | function | Streaming-event counterpart of `createRuntimeEventCollector`. Pass each | @@ -139,6 +140,8 @@ Import from `@tangle-network/agent-runtime` — 431 exports. | `FORWARD_HEADERS` | const | Standard names — lowercased so Headers maps interop on every runtime. | | `INTELLIGENCE_WIRE_VERSION` | const | Wire version the eval-runs ingest enforces (X-Tangle-Wire-Version + body). | | `optimizerMethod` | const | The shared method block every build/author prompt embeds. Domain framing | +| `PROMPT_INSTRUCTION_COMPONENT_PREFIX` | const | Stable component-name prefix used for `profile.prompt.instructions`. | +| `promptInstructionsProfileComponents` | const | Canonical `ImproveProfileComponents` mapping for the ordered | | `RESEARCH_SUPERVISOR_SYSTEM_PROMPT` | const | Standing prompt for a supervisor that grows a shared knowledge base through spawned researchers. | | `ROLLOUT_POLICY_EXTENSION` | const | The profile extensions namespace the policy persists under. | | `strategyAuthorMethod` | const | The senior authoring process for `authorStrategy` — the same method, shaped | @@ -213,6 +216,7 @@ Import from `@tangle-network/agent-runtime` — 431 exports. | `OfficialOptimizerContextOptions` | interface | Runtime context appended to an official optimizer's own configuration. | | `OpenAIChatTool` | interface | OpenAI Chat Completions tool descriptor. The shape mirrors the | | `PreparedAgentCandidateKnowledge` | interface | Exact file-backed knowledge admitted by the candidate bundle. | +| `ProfileImprovementHarness` | interface | A small, reusable front door over `improve(profile, options)`. | | `ProtectedAgentCandidateModelGrantContext` | interface | Values available only while one protected model grant is active. | | `ProviderModelAttemptEvidence` | interface | One provider/harness inference attempt. An empty observation list means the attempt started but | | `RouterEnv` | interface | Env keys the router base URL is resolved from. | @@ -275,7 +279,7 @@ Import from `@tangle-network/agent-runtime` — 431 exports. | `WorkerTraceUnavailableReason` | type | Why Runtime cannot provide structured tool-call evidence for one settled execution. | | `WorktreeCheckRunner` | type | The single shell-command-in-worktree runner seam (replaces the per-executor copies). | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidatePreparationEvidence`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateVerificationPorts`, `AgentCandidateWorkspaceArchiveLimits`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgenticGeneratorShotReceipt`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `AnalystRegistry`, `BackendCallPolicy`, `CanonicalCandidateDocument`, `CaptureAgentCandidateWorkspaceOptions`, `CapturedAgentCandidateWorkspace`, `ChatModelCandidate`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationJournalEntry`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `CreateAgentCandidateWorkspacePortOptions`, `CreateKnowledgeImprovementActivationExecutorOptions`, `CreateProtectedAgentCandidateModelPortOptions`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `DisposePreparedAgentCandidateOptions`, `Driver`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `ExactProcessCandidateExecutorOptions`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `HaltContext`, `HaltSignal`, `ImproveCodeBaseOptions`, `ImproveCodeResult`, `ImproveCustomCodeGeneratorOptions`, `ImprovementCodeCandidate`, `ImprovementProfileCandidate`, `ImproveMethodContext`, `ImproveMethodResult`, `ImproveRuntimeCodeGeneratorOptions`, `ImproveSkillsOptions`, `InMemoryAgentCandidateExecutionClaimStoreOptions`, `KnowledgeImprovementActivationExecutor`, `KnowledgeImprovementCandidatePair`, `KnowledgeImprovementExperimentBundles`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessDecision`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopResult`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `McpServeSpec`, `OfficialSensitiveCandidateInput`, `OtelAttribute`, `OtelExportConfig`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `RawTraceDistillerOptions`, `RecoverExpiredAgentCandidateOptions`, `ReflectiveGeneratorOptions`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolvedAgentCandidateContainer`, `ResolvedChatModel`, `RunAgentTaskOptions`, `RunAgentTaskStreamOptions`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeEventOtelOptions`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunCompleteInput`, `RuntimeRunCost`, `RuntimeRunHandle`, `RuntimeRunOptions`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSession`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeStreamEventSummary`, `RuntimeTelemetryOptions`, `SanitizedKnowledgeReadinessReport`, `SanitizedKnowledgeRequirement`, `ServerSentEventOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `VerifiedAgentCandidate`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentCandidateModelGrantActivateInput`, `AgentCandidateModelGrantReserveInput`, `AgentCandidateModelGrantSettleInput`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`, `AgenticGeneratorExecutorForWorktree`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ChatModelValidation`, `ControlDecision`, `ConversationStreamEvent`, `DeepReadonly`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `ImproveCandidateValidator`, `ImproveCodeOptions`, `ImprovementCandidate`, `ImprovementProfileCandidatePopulation`, `ImprovementProfilePopulationCandidate`, `ImprovementProfilePopulationLineage`, `ImproveMethodSource`, `ImproveOptimizationRunOptions`, `ImproveProfileSurface`, `ImproveResult`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeRunStatus`, `RuntimeStreamEvent`, `RuntimeStreamEventSink`, `SupervisedKnowledgeUpdater`, `TurnOrder`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentCandidateContainerPort`, `AgentCandidateExecutionAttemptRef`, `AgentCandidateExecutionPorts`, `AgentCandidateExecutorWorkspaceFile`, `AgentCandidateExecutorWorkspaceInput`, `AgentCandidateMemoryPort`, `AgentCandidateMemoryResetResult`, `AgentCandidateModelPort`, `AgentCandidatePreparationEvidence`, `AgentCandidateProtectedModelActivation`, `AgentCandidateProtectedModelReservation`, `AgentCandidateProtectedModelSettlement`, `AgentCandidateProtectedRunCapture`, `AgentCandidateVerificationPorts`, `AgentCandidateWorkspaceArchiveLimits`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgenticGeneratorShotReceipt`, `AgentKnowledgeProvider`, `AgentKnowledgeReadinessCheckOptions`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `AnalystRegistry`, `BackendCallPolicy`, `CanonicalCandidateDocument`, `CaptureAgentCandidateWorkspaceOptions`, `CapturedAgentCandidateWorkspace`, `ChatModelCandidate`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationJournalEntry`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `CreateAgentCandidateWorkspacePortOptions`, `CreateKnowledgeImprovementActivationExecutorOptions`, `CreateProfileImprovementHarnessOptions`, `CreateProtectedAgentCandidateModelPortOptions`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `DisposePreparedAgentCandidateOptions`, `Driver`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `ExactProcessCandidateExecutorOptions`, `ExecutePreparedAgentCandidateOptions`, `FileAgentCandidateExecutionClaimStoreOptions`, `HaltContext`, `HaltSignal`, `ImproveCodeBaseOptions`, `ImproveCodeResult`, `ImproveCustomCodeGeneratorOptions`, `ImprovementCodeCandidate`, `ImprovementProfileCandidate`, `ImproveMethodContext`, `ImproveMethodResult`, `ImproveRuntimeCodeGeneratorOptions`, `ImproveSkillsOptions`, `InMemoryAgentCandidateExecutionClaimStoreOptions`, `KnowledgeImprovementActivationExecutor`, `KnowledgeImprovementCandidatePair`, `KnowledgeImprovementExperimentBundles`, `KnowledgeImprovementJobMeasurement`, `KnowledgeImprovementJobResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessDecision`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopResult`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `McpServeSpec`, `OfficialSensitiveCandidateInput`, `OtelAttribute`, `OtelExportConfig`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `PrepareAgentCandidateExecutionOptions`, `PreparedAgentCandidateExecution`, `PreparedAgentCandidateInstruction`, `PreparedAgentCandidateLaunch`, `PreparedAgentCandidateTrace`, `RawTraceDistillerOptions`, `RecoverExpiredAgentCandidateOptions`, `ReflectiveGeneratorOptions`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolvedAgentCandidateContainer`, `ResolvedChatModel`, `RunAgentTaskOptions`, `RunAgentTaskStreamOptions`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunKnowledgeImprovementJobOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeEventOtelOptions`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunCompleteInput`, `RuntimeRunCost`, `RuntimeRunHandle`, `RuntimeRunOptions`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSession`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeStreamEventSummary`, `RuntimeTelemetryOptions`, `SanitizedKnowledgeReadinessReport`, `SanitizedKnowledgeRequirement`, `ServerSentEventOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `VerifiedAgentCandidate`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentCandidateModelGrantActivateInput`, `AgentCandidateModelGrantReserveInput`, `AgentCandidateModelGrantSettleInput`, `AgentCandidateOutputPurpose`, `AgentCandidateRetryRejection`, `AgentCandidateRunFinalization`, `AgenticGeneratorExecutorForWorktree`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ChatModelValidation`, `ControlDecision`, `ConversationStreamEvent`, `DeepReadonly`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `ImproveCandidateValidator`, `ImproveCodeOptions`, `ImprovementCandidate`, `ImprovementProfileCandidatePopulation`, `ImprovementProfilePopulationCandidate`, `ImprovementProfilePopulationLineage`, `ImproveMethodSource`, `ImproveOptimizationRunOptions`, `ImproveProfileSurface`, `ImproveResult`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `ProfileImprovementHarnessRunOptions`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeRunStatus`, `RuntimeStreamEvent`, `RuntimeStreamEventSink`, `SupervisedKnowledgeUpdater`, `TurnOrder`. ### Vertical agent — manifest + surface proposal source @@ -363,19 +367,23 @@ Import from `@tangle-network/agent-runtime/conversation` — 54 exports. ### Product chat turns — edge-safe streaming, persistence, and stable execution IDs -Import from `@tangle-network/agent-runtime/durable` — 8 exports. +Import from `@tangle-network/agent-runtime/durable` — 11 exports. | Symbol | Kind | Summary | |---|---|---| | `deriveExecutionId` | function | Derive a stable execution id from the run identity. | +| `discoverDurableSupervisionRun` | function | Discover the stable identities recorded by Runtime's durable supervision | | `handleChatTurn` | function | Run one chat turn. Returns immediately with a `ReadableStream` body; | | `ChatStreamEvent` | interface | The NDJSON line protocol every product chat client already speaks. | | `ChatTurnHooks` | interface | Product callbacks invoked while one chat turn runs. | | `ChatTurnIdentity` | interface | Identity of a chat turn. `tenantId` is the workspace id for workspace- | | `ChatTurnProducer` | interface | The live side of a turn returned by the product's `produce` hook. | | `ChatTurnResult` | interface | HTTP response values returned for one chat turn. | +| `DurableSupervisionDiscovery` | interface | Identities discoverable from one `supervise({ runDir })` directory without | | `RunChatTurnInput` | interface | Inputs for one streamed product chat turn. | +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `DurableCoordinationStreamIdentity`. + ### Bounded tool calls for browser and edge runtimes Import from `@tangle-network/agent-runtime/tool-loop` — 12 exports. diff --git a/src/durable/index.ts b/src/durable/index.ts index dd0b8081..b00f874c 100644 --- a/src/durable/index.ts +++ b/src/durable/index.ts @@ -10,6 +10,8 @@ * hook ordering. * - `deriveExecutionId`: convention helper for the stable id products * persist and pass as both execution and turn identity on dispatch. + * - `discoverDurableSupervisionRun`: inspect a durable supervision directory + * without already knowing the root/run identities written inside it. */ export type { @@ -22,3 +24,8 @@ export type { } from './chat-engine' export { handleChatTurn } from './chat-engine' export { deriveExecutionId } from './execution-handle' +export { + type DurableCoordinationStreamIdentity, + type DurableSupervisionDiscovery, + discoverDurableSupervisionRun, +} from './supervision-discovery' diff --git a/src/durable/supervision-discovery.test.ts b/src/durable/supervision-discovery.test.ts new file mode 100644 index 00000000..5682e367 --- /dev/null +++ b/src/durable/supervision-discovery.test.ts @@ -0,0 +1,169 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { discoverDurableSupervisionRun } from './supervision-discovery' + +const fixtureDirs: string[] = [] + +afterEach(() => { + for (const dir of fixtureDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }) + } +}) + +function fixtureDir(): string { + const dir = mkdtempSync(join(tmpdir(), 'durable-supervision-discovery-')) + fixtureDirs.push(dir) + return dir +} + +describe('discoverDurableSupervisionRun', () => { + it('discovers roots, run ids, owner ids, and legacy unscoped records', async () => { + const dir = fixtureDir() + writeFileSync( + join(dir, 'spawn-journal.jsonl'), + [ + JSON.stringify({ kind: 'begin', root: 'root-b', at: '2026-08-16T00:00:00Z' }), + JSON.stringify({ kind: 'event', root: 'root-b', event: { kind: 'fixture' } }), + JSON.stringify({ kind: 'begin', root: 'root-a', at: '2026-08-16T00:00:01Z' }), + '', + ].join('\n'), + ) + writeFileSync( + join(dir, 'coordination-log.jsonl'), + [ + JSON.stringify({ runId: 'run-b', ownerId: 'owner-2', event: { type: 'fixture' } }), + JSON.stringify({ runId: 'run-a', ownerId: 'owner-1', event: { type: 'fixture' } }), + JSON.stringify({ runId: 'run-b', ownerId: 'owner-1', event: { type: 'fixture' } }), + JSON.stringify({ runId: 'run-b', event: { type: 'legacy-fixture' } }), + '', + ].join('\n'), + ) + + await expect(discoverDurableSupervisionRun(dir)).resolves.toEqual({ + runDir: resolve(dir), + spawnJournalPath: join(resolve(dir), 'spawn-journal.jsonl'), + coordinationLogPath: join(resolve(dir), 'coordination-log.jsonl'), + roots: ['root-a', 'root-b'], + coordinationStreams: [ + { + runId: 'run-a', + ownerIds: ['owner-1'], + unscopedRecords: 0, + recordCount: 1, + }, + { + runId: 'run-b', + ownerIds: ['owner-1', 'owner-2'], + unscopedRecords: 1, + recordCount: 3, + }, + ], + }) + }) + + it('returns only top-level roots when spawned drivers own nested journal trees', async () => { + const dir = fixtureDir() + const nestedRoot = 'root-main/root-main:s0' + writeFileSync( + join(dir, 'spawn-journal.jsonl'), + [ + JSON.stringify({ kind: 'begin', root: 'root-main', at: '2026-08-16T00:00:00Z' }), + JSON.stringify({ + kind: 'event', + root: 'root-main', + event: { + kind: 'spawned', + id: 'root-main:s0', + label: 'nested driver', + runtime: 'driver', + ownedTreeRoot: nestedRoot, + }, + }), + JSON.stringify({ kind: 'begin', root: nestedRoot, at: '2026-08-16T00:00:01Z' }), + JSON.stringify({ + kind: 'event', + root: nestedRoot, + event: { kind: 'spawned', id: `${nestedRoot}:s0`, label: 'leaf', runtime: 'router' }, + }), + '', + ].join('\n'), + ) + + const result = await discoverDurableSupervisionRun(dir) + + expect(result.roots).toEqual(['root-main']) + }) + + it('returns an empty discovery for a valid directory with no durable files', async () => { + const dir = fixtureDir() + const result = await discoverDurableSupervisionRun(dir) + expect(result.roots).toEqual([]) + expect(result.coordinationStreams).toEqual([]) + expect(Object.isFrozen(result)).toBe(true) + }) + + it('ignores torn unacknowledged tails while retaining committed identities', async () => { + const dir = fixtureDir() + writeFileSync( + join(dir, 'spawn-journal.jsonl'), + `${JSON.stringify({ kind: 'begin', root: 'root-a' })}\n{"kind":"begin"`, + ) + writeFileSync( + join(dir, 'coordination-log.jsonl'), + `${JSON.stringify({ runId: 'run-a', ownerId: 'owner-a' })}\n{"runId":`, + ) + + await expect(discoverDurableSupervisionRun(dir)).resolves.toMatchObject({ + roots: ['root-a'], + coordinationStreams: [ + { runId: 'run-a', ownerIds: ['owner-a'], unscopedRecords: 0, recordCount: 1 }, + ], + }) + }) + + it('refuses an empty run directory identity', async () => { + await expect(discoverDurableSupervisionRun(' ')).rejects.toThrow(/non-empty string/) + }) + + it('refuses malformed committed spawn identities', async () => { + const missingRoot = fixtureDir() + writeFileSync( + join(missingRoot, 'spawn-journal.jsonl'), + `${JSON.stringify({ kind: 'begin', at: '2026-08-16T00:00:00Z' })}\n`, + ) + await expect(discoverDurableSupervisionRun(missingRoot)).rejects.toThrow(/root identity/) + + const malformedOwnedRoot = fixtureDir() + writeFileSync( + join(malformedOwnedRoot, 'spawn-journal.jsonl'), + [ + JSON.stringify({ kind: 'begin', root: 'root-a' }), + JSON.stringify({ + kind: 'event', + root: 'root-a', + event: { kind: 'spawned', id: 'root-a:s0', ownedTreeRoot: '' }, + }), + '', + ].join('\n'), + ) + await expect(discoverDurableSupervisionRun(malformedOwnedRoot)).rejects.toThrow(/ownedTreeRoot/) + }) + + it('refuses malformed committed coordination identities', async () => { + const missingRunId = fixtureDir() + writeFileSync( + join(missingRunId, 'coordination-log.jsonl'), + `${JSON.stringify({ ownerId: 'owner-a', event: { type: 'fixture' } })}\n`, + ) + await expect(discoverDurableSupervisionRun(missingRunId)).rejects.toThrow(/runId identity/) + + const malformedOwner = fixtureDir() + writeFileSync( + join(malformedOwner, 'coordination-log.jsonl'), + `${JSON.stringify({ runId: 'run-a', ownerId: '', event: { type: 'fixture' } })}\n`, + ) + await expect(discoverDurableSupervisionRun(malformedOwner)).rejects.toThrow(/ownerId/) + }) +}) diff --git a/src/durable/supervision-discovery.ts b/src/durable/supervision-discovery.ts new file mode 100644 index 00000000..28c35392 --- /dev/null +++ b/src/durable/supervision-discovery.ts @@ -0,0 +1,163 @@ +import { resolve } from 'node:path' +import type { NodeId } from '../runtime/supervise/types' +import { parseCommittedJsonLines } from './jsonl-file' + +export interface DurableCoordinationStreamIdentity { + readonly runId: string + /** Exact owner ids present in the side-log, sorted for deterministic display. */ + readonly ownerIds: readonly string[] + /** Records written before owner-scoped coordination identities were introduced. */ + readonly unscopedRecords: number + readonly recordCount: number +} + +/** + * Identities discoverable from one `supervise({ runDir })` directory without + * already knowing the root node or coordination run id stored inside it. + */ +export interface DurableSupervisionDiscovery { + readonly runDir: string + readonly spawnJournalPath: string + readonly coordinationLogPath: string + readonly roots: readonly NodeId[] + readonly coordinationStreams: readonly DurableCoordinationStreamIdentity[] +} + +type SpawnJournalIdentityRecord = { + readonly kind?: unknown + readonly root?: unknown + readonly event?: unknown +} + +type CoordinationIdentityRecord = { + readonly runId?: unknown + readonly ownerId?: unknown +} + +/** + * Discover the stable identities recorded by Runtime's durable supervision + * files. This is the developer-facing first step before calling + * `FileSpawnJournal.loadTree(root)`, `loadSpawnForest(journal, root)`, or + * `FileCoordinationLog.load(runId, ownerId)`. + * + * Missing files produce empty collections. A malformed committed JSONL record + * still fails loud through the same parser used by the runtime; a torn final + * append is ignored because it was never acknowledged as committed. + */ +export async function discoverDurableSupervisionRun( + runDir: string, +): Promise { + if (typeof runDir !== 'string' || runDir.trim().length === 0) { + throw new TypeError('discoverDurableSupervisionRun: runDir must be a non-empty string') + } + + const canonicalRunDir = resolve(runDir) + const spawnJournalPath = `${canonicalRunDir}/spawn-journal.jsonl` + const coordinationLogPath = `${canonicalRunDir}/coordination-log.jsonl` + const [spawnText, coordinationText] = await Promise.all([ + readOptionalText(spawnJournalPath), + readOptionalText(coordinationLogPath), + ]) + + const allRoots = new Set() + const nestedRoots = new Set() + if (spawnText !== undefined) { + for (const record of parseCommittedJsonLines( + spawnText, + spawnJournalPath, + )) { + if (record.kind === 'begin') { + if (typeof record.root !== 'string' || record.root.length === 0) { + throw new Error(`${spawnJournalPath}: begin record has no non-empty string root identity`) + } + allRoots.add(record.root as NodeId) + continue + } + if (record.kind !== 'event') continue + const event = record.event + if (!isRecord(event) || event.kind !== 'spawned') continue + if (!Object.hasOwn(event, 'ownedTreeRoot')) continue + if (typeof event.ownedTreeRoot !== 'string' || event.ownedTreeRoot.length === 0) { + throw new Error( + `${spawnJournalPath}: spawned event ownedTreeRoot must be a non-empty string when present`, + ) + } + nestedRoots.add(event.ownedTreeRoot as NodeId) + } + } + + const streams = new Map< + string, + { owners: Set; unscopedRecords: number; recordCount: number } + >() + if (coordinationText !== undefined) { + for (const record of parseCommittedJsonLines( + coordinationText, + coordinationLogPath, + )) { + if (typeof record.runId !== 'string' || record.runId.length === 0) { + throw new Error(`${coordinationLogPath}: record has no non-empty string runId identity`) + } + const stream = streams.get(record.runId) ?? { + owners: new Set(), + unscopedRecords: 0, + recordCount: 0, + } + stream.recordCount += 1 + if (record.ownerId === undefined) { + stream.unscopedRecords += 1 + } else if (typeof record.ownerId === 'string' && record.ownerId.length > 0) { + stream.owners.add(record.ownerId) + } else { + throw new Error(`${coordinationLogPath}: ownerId must be a non-empty string when present`) + } + streams.set(record.runId, stream) + } + } + + const coordinationStreams = [...streams.entries()] + .sort(([left], [right]) => compareText(left, right)) + .map(([runId, stream]) => + Object.freeze({ + runId, + ownerIds: Object.freeze([...stream.owners].sort(compareText)), + unscopedRecords: stream.unscopedRecords, + recordCount: stream.recordCount, + }), + ) + + return Object.freeze({ + runDir: canonicalRunDir, + spawnJournalPath, + coordinationLogPath, + roots: Object.freeze([...allRoots].filter((root) => !nestedRoots.has(root)).sort(compareText)), + coordinationStreams: Object.freeze(coordinationStreams), + }) +} + +async function readOptionalText(path: string): Promise { + const fs = await import('node:fs/promises') + try { + return await fs.readFile(path, 'utf8') + } catch (error) { + if (isNoEntError(error)) return undefined + throw error + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function compareText(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0 +} + +function isNoEntError(error: unknown): boolean { + return ( + typeof error === 'object' && + error !== null && + 'code' in error && + (error as { code: unknown }).code === 'ENOENT' + ) +} diff --git a/src/improvement/index.ts b/src/improvement/index.ts index 9c33115a..5a97b758 100644 --- a/src/improvement/index.ts +++ b/src/improvement/index.ts @@ -81,7 +81,17 @@ export { optimizerMethod, strategyAuthorMethod, } from './optimizer-prompt' +export { + type CreateProfileImprovementHarnessOptions, + createProfileImprovementHarness, + type ProfileImprovementHarness, + type ProfileImprovementHarnessRunOptions, +} from './profile-improvement-harness' export type { DeepReadonly, ReadonlyAgentProfile } from './profile-types' +export { + PROMPT_INSTRUCTION_COMPONENT_PREFIX, + promptInstructionsProfileComponents, +} from './prompt-instructions-profile-components' export { type RawTraceDistillerOptions, rawTraceDistiller, diff --git a/src/improvement/meta-harness.test.ts b/src/improvement/meta-harness.test.ts new file mode 100644 index 00000000..37264969 --- /dev/null +++ b/src/improvement/meta-harness.test.ts @@ -0,0 +1,278 @@ +import { + inMemoryCampaignStorage, + type OptimizationMethod, +} from '@tangle-network/agent-eval/campaign' +import type { + DispatchContext, + JudgeConfig, + MutableSurface, + Scenario, +} from '@tangle-network/agent-eval/contract' +import { type AgentProfile, canonicalCandidateDigest } from '@tangle-network/agent-interface' +import { describe, expect, it } from 'vitest' +import { ConfigError } from '../errors' +import { createProfileImprovementHarness } from './profile-improvement-harness' +import type { ReadonlyAgentProfile } from './profile-types' +import { + PROMPT_INSTRUCTION_COMPONENT_PREFIX, + promptInstructionsProfileComponents, +} from './prompt-instructions-profile-components' + +interface FixtureScenario extends Scenario { + kind: 'fixture' +} + +interface FixtureArtifact { + text: string +} + +const trainScenarios: FixtureScenario[] = [{ id: 'train', kind: 'fixture' }] +const selectionScenarios: FixtureScenario[] = [{ id: 'selection', kind: 'fixture' }] +const testScenarios: FixtureScenario[] = [ + { id: 'test-a', kind: 'fixture' }, + { id: 'test-b', kind: 'fixture' }, +] +let runSequence = 0 + +const improvementJudge: JudgeConfig = { + name: 'meta-harness-improvement', + dimensions: [{ key: 'quality', description: 'candidate contains the improvement marker' }], + score: ({ artifact }) => { + const quality = artifact.text.includes('improved') ? 1 : 0 + return { dimensions: { quality }, composite: quality, notes: '' } + }, +} + +const baselineProfile = (): AgentProfile => ({ + name: 'meta-harness-fixture', + prompt: { + systemPrompt: 'Keep this system prompt unchanged.', + instructions: ['Inspect the evidence.', 'State uncertainty explicitly.'], + }, + metadata: { owner: 'test' }, +}) + +async function paidProfile( + profile: ReadonlyAgentProfile, + _scenario: FixtureScenario, + context: DispatchContext, +): Promise { + const paid = await context.cost.runPaidCall({ + channel: 'agent', + actor: 'meta-harness-test-agent', + model: 'deterministic-meta-harness@2026-08-16', + maximumCharge: { externallyEnforcedMaximumUsd: 0.0001 }, + execute: async () => ({ text: profile.prompt?.systemPrompt ?? '' }), + receipt: () => ({ + model: 'deterministic-meta-harness@2026-08-16', + inputTokens: 1, + outputTokens: 1, + actualCostUsd: 0.0001, + }), + }) + if (!paid.succeeded) throw paid.error + return paid.value +} + +function fixedMethod( + winnerSurface: MutableSurface, +): OptimizationMethod { + return { + name: 'meta-harness-fixed-method', + async optimize() { + return { + winnerSurface, + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, + durationMs: 1, + } + }, + } +} + +function runOptions() { + runSequence += 1 + return { + method: fixedMethod('improved prompt'), + surface: 'prompt' as const, + trainScenarios, + selectionScenarios, + testScenarios, + judges: [improvementJudge], + runDir: `mem://meta-harness-${runSequence}`, + storage: inMemoryCampaignStorage(), + resamples: 40, + confidence: 0.95, + } +} + +describe('promptInstructionsProfileComponents', () => { + it('maps ordered instructions to stable labels and changes no unrelated profile field', () => { + const profile = baselineProfile() + const components = promptInstructionsProfileComponents.read(profile) + + expect(components).toEqual({ + [`${PROMPT_INSTRUCTION_COMPONENT_PREFIX}000000`]: 'Inspect the evidence.', + [`${PROMPT_INSTRUCTION_COMPONENT_PREFIX}000001`]: 'State uncertainty explicitly.', + }) + + const candidate = promptInstructionsProfileComponents.apply(profile, { + [`${PROMPT_INSTRUCTION_COMPONENT_PREFIX}000000`]: 'Inspect every cited artifact.', + [`${PROMPT_INSTRUCTION_COMPONENT_PREFIX}000001`]: 'Report calibrated uncertainty.', + }) + + expect(candidate.prompt).toEqual({ + systemPrompt: 'Keep this system prompt unchanged.', + instructions: ['Inspect every cited artifact.', 'Report calibrated uncertainty.'], + }) + expect(candidate.metadata).toEqual({ owner: 'test' }) + expect(profile.prompt?.instructions).toEqual([ + 'Inspect the evidence.', + 'State uncertainty explicitly.', + ]) + }) + + it('refuses missing instructions and component-key drift', () => { + expect(() => promptInstructionsProfileComponents.read({ name: 'empty-instructions' })).toThrow( + /must contain at least one instruction/, + ) + + expect(() => + promptInstructionsProfileComponents.apply(baselineProfile(), { + [`${PROMPT_INSTRUCTION_COMPONENT_PREFIX}000001`]: 'wrong first key', + }), + ).toThrow(/expected component/) + }) +}) + +describe('createProfileImprovementHarness', () => { + it('binds an immutable baseline and canonical identity once', () => { + const source = baselineProfile() + const executionRef = canonicalCandidateDigest({ fixture: 'bound-executor-v1' }) + const harness = createProfileImprovementHarness({ + profile: source, + executionRef, + agent: async () => 'unused', + }) + + source.prompt!.instructions![0] = 'mutated after construction' + + expect(harness.executionRef).toBe(executionRef) + expect(harness.profile.prompt?.instructions?.[0]).toBe('Inspect the evidence.') + expect(harness.profileDigest).toMatch(/^sha256:[0-9a-f]{64}$/) + expect(Object.isFrozen(harness)).toBe(true) + expect(Object.isFrozen(harness.profile)).toBe(true) + }) + + it('runs the exact bound profile, executor, and run-level validator', async () => { + const executionRef = canonicalCandidateDigest({ fixture: 'bound-executor-v2' }) + const rogueExecutionRef = canonicalCandidateDigest({ fixture: 'rogue-executor' }) + const observedProfiles: ReadonlyAgentProfile[] = [] + let boundAgentCalls = 0 + let rogueAgentCalls = 0 + let defaultValidatorCalls = 0 + let overrideValidatorCalls = 0 + const harness = createProfileImprovementHarness({ + profile: baselineProfile(), + executionRef, + agent: async (profile, scenario, context) => { + boundAgentCalls += 1 + observedProfiles.push(profile) + return paidProfile(profile, scenario, context) + }, + validateCandidate: () => { + defaultValidatorCalls += 1 + }, + }) + + const result = await harness.run({ + ...runOptions(), + executionRef: rogueExecutionRef, + agent: async () => { + rogueAgentCalls += 1 + return { text: 'rogue agent' } + }, + validateCandidate: () => { + overrideValidatorCalls += 1 + }, + } as never) + + expect(boundAgentCalls).toBeGreaterThan(0) + expect(rogueAgentCalls).toBe(0) + expect(defaultValidatorCalls).toBe(0) + expect(overrideValidatorCalls).toBeGreaterThan(0) + expect(result.lineage.executionRef).toBe(executionRef) + expect(result.lineage.baselineProfileDigest).toBe(harness.profileDigest) + expect(result.candidate.profile?.prompt?.systemPrompt).toBe('improved prompt') + expect( + observedProfiles.some( + (profile) => profile.prompt?.systemPrompt === 'Keep this system prompt unchanged.', + ), + ).toBe(true) + expect( + observedProfiles.some((profile) => profile.prompt?.systemPrompt === 'improved prompt'), + ).toBe(true) + }) + + it('uses the harness validator when a run does not override it', async () => { + let defaultValidatorCalls = 0 + const harness = createProfileImprovementHarness({ + profile: baselineProfile(), + executionRef: canonicalCandidateDigest({ fixture: 'bound-executor-v3' }), + agent: paidProfile, + validateCandidate: () => { + defaultValidatorCalls += 1 + }, + }) + + await harness.run(runOptions()) + + expect(defaultValidatorCalls).toBeGreaterThan(0) + }) + + it('fails before a run for invalid profile, execution identity, agent, or validator', () => { + expect(() => + createProfileImprovementHarness({ + profile: null as never, + executionRef: canonicalCandidateDigest({ fixture: 'valid' }), + agent: async () => 'unused', + }), + ).toThrow(ConfigError) + + expect(() => + createProfileImprovementHarness({ + profile: baselineProfile(), + executionRef: 'not-a-digest' as never, + agent: async () => 'unused', + }), + ).toThrow(ConfigError) + + expect(() => + createProfileImprovementHarness({ + profile: baselineProfile(), + executionRef: canonicalCandidateDigest({ fixture: 'valid-agent-check' }), + agent: null as never, + }), + ).toThrow(ConfigError) + + expect(() => + createProfileImprovementHarness({ + profile: baselineProfile(), + executionRef: canonicalCandidateDigest({ fixture: 'valid-validator-check' }), + agent: async () => 'unused', + validateCandidate: null as never, + }), + ).toThrow(ConfigError) + + const harness = createProfileImprovementHarness({ + profile: baselineProfile(), + executionRef: canonicalCandidateDigest({ fixture: 'valid-run-validator-check' }), + agent: async () => 'unused', + }) + expect(() => harness.run({ validateCandidate: null } as never)).toThrow(ConfigError) + }) +}) diff --git a/src/improvement/profile-improvement-harness.ts b/src/improvement/profile-improvement-harness.ts new file mode 100644 index 00000000..06dd8387 --- /dev/null +++ b/src/improvement/profile-improvement-harness.ts @@ -0,0 +1,117 @@ +import type { Scenario } from '@tangle-network/agent-eval/contract' +import { + type AgentProfile, + agentProfileSchema, + canonicalAgentProfileDigest, + type Sha256Digest, +} from '@tangle-network/agent-interface' +import { immutableCandidateValue } from '../candidate-execution/digest' +import { ConfigError } from '../errors' +import { improve } from './improve' +import type { + ImproveCandidateValidator, + ImproveMethodOptions, + ImproveMethodResult, + ImproveProfileAgent, +} from './improve-types' +import type { ReadonlyAgentProfile } from './profile-types' + +export interface CreateProfileImprovementHarnessOptions { + /** Exact baseline profile. It is parsed, detached, and frozen at construction. */ + profile: AgentProfile + /** + * Immutable identity of the bound executor, models, tools, component mapping, + * and every closure or external setting that can change measured behavior. + */ + executionRef: Sha256Digest + /** Execute one exact materialized profile on one scenario. */ + agent: ImproveProfileAgent + /** Optional validator shared by every run from this harness. */ + validateCandidate?: ImproveCandidateValidator +} + +export type ProfileImprovementHarnessRunOptions = Omit< + ImproveMethodOptions, + 'executionRef' | 'agent' | 'validateCandidate' +> & { + /** Override the harness-level validator for this run. */ + validateCandidate?: ImproveCandidateValidator +} + +/** + * A small, reusable front door over `improve(profile, options)`. + * + * The harness freezes the baseline and binds execution identity once, which + * removes the two easiest sources of accidental experiment drift when a + * developer runs several methods, surfaces, or held-out suites against the + * same agent. It does not replace or narrow `improve`; callers retain every + * method option and may still use the lower-level API directly. + */ +export interface ProfileImprovementHarness { + /** Detached immutable baseline actually used by every run. */ + readonly profile: ReadonlyAgentProfile + /** Canonical digest of the bound baseline profile. */ + readonly profileDigest: Sha256Digest + /** Exact execution identity bound at construction. */ + readonly executionRef: Sha256Digest + run( + options: ProfileImprovementHarnessRunOptions, + ): Promise +} + +/** + * Bind one exact profile and executor into a repeatable self-improvement + * harness. The returned `run` method remains generic over every existing + * profile surface, optimization method, split, gate, and budget option. + */ +export function createProfileImprovementHarness( + options: CreateProfileImprovementHarnessOptions, +): ProfileImprovementHarness { + const parsed = agentProfileSchema.safeParse(options.profile) + if (!parsed.success) { + throw new ConfigError( + `createProfileImprovementHarness: invalid AgentProfile: ${parsed.error.message}`, + ) + } + if (!/^sha256:[0-9a-f]{64}$/.test(options.executionRef)) { + throw new ConfigError( + 'createProfileImprovementHarness: executionRef must be a lowercase sha256 digest', + ) + } + if (typeof options.agent !== 'function') { + throw new ConfigError('createProfileImprovementHarness: agent must be a function') + } + if (options.validateCandidate !== undefined && typeof options.validateCandidate !== 'function') { + throw new ConfigError( + 'createProfileImprovementHarness: validateCandidate must be a function when present', + ) + } + + const profile = immutableCandidateValue(parsed.data) + const executionRef = options.executionRef + const agent = options.agent + const defaultValidator = options.validateCandidate + + return Object.freeze({ + profile, + profileDigest: canonicalAgentProfileDigest(profile), + executionRef, + run(runOptions: ProfileImprovementHarnessRunOptions) { + if ( + runOptions.validateCandidate !== undefined && + typeof runOptions.validateCandidate !== 'function' + ) { + throw new ConfigError( + 'ProfileImprovementHarness.run: validateCandidate must be a function when present', + ) + } + const validateCandidate = runOptions.validateCandidate ?? defaultValidator + return improve(profile, { + ...runOptions, + executionRef, + agent, + ...(validateCandidate === undefined ? {} : { validateCandidate }), + }) + }, + }) +} diff --git a/src/improvement/prompt-instructions-profile-components.ts b/src/improvement/prompt-instructions-profile-components.ts new file mode 100644 index 00000000..f6d77d40 --- /dev/null +++ b/src/improvement/prompt-instructions-profile-components.ts @@ -0,0 +1,85 @@ +import { agentProfileSchema } from '@tangle-network/agent-interface' +import { ConfigError } from '../errors' +import type { ImproveProfileComponents } from './improve-types' +import type { ReadonlyAgentProfile } from './profile-types' + +/** Stable component-name prefix used for `profile.prompt.instructions`. */ +export const PROMPT_INSTRUCTION_COMPONENT_PREFIX = 'prompt.instruction:' + +const COMPONENT_INDEX_WIDTH = 6 + +function componentName(index: number): string { + return `${PROMPT_INSTRUCTION_COMPONENT_PREFIX}${String(index).padStart(COMPONENT_INDEX_WIDTH, '0')}` +} + +function orderedInstructionValues(components: Readonly>): string[] { + const entries = Object.entries(components).sort(([left], [right]) => left.localeCompare(right)) + if (entries.length === 0) { + throw new ConfigError( + 'promptInstructionsProfileComponents: at least one prompt instruction is required', + ) + } + for (const [index, [name, value]] of entries.entries()) { + const expected = componentName(index) + if (name !== expected) { + throw new ConfigError( + `promptInstructionsProfileComponents: expected component ${JSON.stringify(expected)}, got ${JSON.stringify(name)}`, + ) + } + if (typeof value !== 'string') { + throw new ConfigError( + `promptInstructionsProfileComponents: component ${JSON.stringify(name)} must be a string`, + ) + } + } + return entries.map(([, value]) => value) +} + +/** + * Canonical `ImproveProfileComponents` mapping for the ordered + * `AgentProfile.prompt.instructions` list. + * + * Use it with `surface: 'agent-profile'` when an optimizer should rewrite the + * exact instruction texts without being allowed to change their count, order, + * labels, or any unrelated profile field: + * + * ```ts + * await improve(profile, { + * surface: 'agent-profile', + * profileComponents: promptInstructionsProfileComponents, + * // method, scenarios, judge, executionRef, agent, ... + * }) + * ``` + * + * Component names are zero-padded and stable. Runtime's existing component + * materializer requires every candidate to preserve the exact key set and + * verifies that `apply(read(profile))` reproduces the baseline profile. A + * profile with no prompt instructions is refused rather than inventing a + * sentinel instruction that could accidentally ship. + */ +export const promptInstructionsProfileComponents: ImproveProfileComponents = Object.freeze({ + read(profile: ReadonlyAgentProfile) { + const instructions = profile.prompt?.instructions ?? [] + if (instructions.length === 0) { + throw new ConfigError( + 'promptInstructionsProfileComponents: profile.prompt.instructions must contain at least one instruction', + ) + } + return Object.fromEntries( + instructions.map((instruction: string, index: number) => [componentName(index), instruction]), + ) + }, + apply( + profile: ReadonlyAgentProfile, + components: Readonly>, + ): ReadonlyAgentProfile { + const instructions = orderedInstructionValues(components) + return agentProfileSchema.parse({ + ...profile, + prompt: { + ...profile.prompt, + instructions, + }, + }) + }, +})