From a07c3cfdc34dc457aa6893d755aa6719d3e0304b Mon Sep 17 00:00:00 2001 From: drewstone Date: Thu, 20 Aug 2026 22:37:00 -0700 Subject: [PATCH 1/4] refactor(execution): remove Eval-owned paid model transports --- .claude/skills/agent-eval/SKILL.md | 2 +- CHANGELOG.md | 23 + README.md | 10 +- clients/python/README.md | 4 + clients/python/pyproject.toml | 2 +- clients/python/src/agent_eval_rpc/__init__.py | 2 +- clients/python/uv.lock | 2 +- docs/building-doctrine.md | 6 +- docs/campaign-proposers.md | 27 +- docs/multishot-golden-records.md | 8 +- docs/public-api.md | 112 ++-- docs/trace-analysis.md | 2 +- examples/README.md | 2 +- examples/_shared/extraction-task.ts | 23 +- examples/_shared/openai-compatible-owner.ts | 363 +++++++++++++ examples/_shared/optimizer-execution-owner.ts | 24 +- .../gsm8k/compare-optimization-methods.ts | 36 +- .../compare-optimization-methods/README.md | 2 +- .../compare-optimization-methods/index.ts | 16 +- examples/self-improve-optimizer/README.md | 2 +- examples/self-improve-optimizer/index.ts | 38 +- package.json | 2 +- scripts/record-multishot-golden.ts | 8 +- scripts/verify-package-exports.mjs | 7 +- src/analyst/adapters.test.ts | 115 ++-- src/analyst/adapters.ts | 6 +- src/analyst/benchmark-implementation.ts | 4 +- src/analyst/benchmark-public-model.ts | 5 + src/analyst/chat-client.ts | 103 +--- src/analyst/index.ts | 3 - src/campaign/index.ts | 4 - .../openai-compatible-execution-owner.test.ts | 247 --------- .../openai-compatible-execution-owner.ts | 116 ---- src/chat-json-call.ts | 87 +++ src/cli-config.test.ts | 37 +- src/cli-config.ts | 75 ++- src/cli.ts | 19 +- src/eval-campaign.test.ts | 28 +- src/eval-campaign.ts | 72 ++- src/index.ts | 19 +- src/integrity/preflight.test.ts | 195 +++---- src/integrity/preflight.ts | 67 +-- src/intent-match-judge.test.ts | 69 +-- src/intent-match-judge.ts | 59 +- src/llm-client.test.ts | 31 -- src/llm-client.ts | 202 +------ src/multishot/cost.ts | 30 ++ src/multishot/default-tools.ts | 22 +- src/multishot/golden/golden.test.ts | 15 - src/multishot/golden/harness.ts | 8 +- src/multishot/golden/matrix-scenarios.ts | 100 ++-- src/multishot/golden/scenarios.ts | 4 - src/multishot/index.ts | 11 +- src/multishot/judges.ts | 43 +- src/multishot/matrix.ts | 21 +- src/multishot/multishot.ts | 50 +- src/multishot/router.ts | 112 ---- src/multishot/types.ts | 24 +- src/reference-equivalence-judge.test.ts | 61 +-- src/semantic-concept-judge.test.ts | 79 +-- src/semantic-concept-judge.ts | 55 +- src/wire/handlers.ts | 73 +-- src/wire/rpc.ts | 9 +- src/wire/server.ts | 15 +- tests/consumer-contract.test.ts | 16 +- tests/eval-campaign.test.ts | 41 +- tests/llm-route-assertion.test.ts | 74 --- tests/multishot/cell-cost-accounting.test.ts | 99 ++-- tests/multishot/judges.test.ts | 153 ++---- tests/multishot/matrix-shot-seam.test.ts | 55 +- tests/multishot/matrix-transport.test.ts | 97 ++-- tests/multishot/multishot.test.ts | 507 ++++++------------ tests/multishot/shape-defaults.test.ts | 2 - tests/rl-rl-campaign.test.ts | 36 +- tests/wire/handlers.test.ts | 109 ++-- tests/wire/rpc.test.ts | 65 +-- 76 files changed, 1817 insertions(+), 2455 deletions(-) create mode 100644 examples/_shared/openai-compatible-owner.ts delete mode 100644 src/campaign/openai-compatible-execution-owner.test.ts delete mode 100644 src/campaign/openai-compatible-execution-owner.ts create mode 100644 src/chat-json-call.ts create mode 100644 src/multishot/cost.ts delete mode 100644 src/multishot/router.ts delete mode 100644 tests/llm-route-assertion.test.ts diff --git a/.claude/skills/agent-eval/SKILL.md b/.claude/skills/agent-eval/SKILL.md index c0ec1e2c..db66633e 100644 --- a/.claude/skills/agent-eval/SKILL.md +++ b/.claude/skills/agent-eval/SKILL.md @@ -57,7 +57,7 @@ Three public entry points improve or benchmark a surface. Route by intent. `selfImprove()` is the agent entry: it gives the method disjoint train and selection partitions, re-scores the selected surface on a held-out split, and returns a `gateDecision`. `compareOptimizationMethods()` is the measurement entry: it gives every method equal inputs and scores the selected surfaces on final cases no method received. Neither entry ever passes final comparison cases to a method. -When no package owns execution, build the owner with `createOpenAiCompatibleExecutionOwner` from `/campaign`. +The execution owner is always caller code: `profileOptimizerModelCall` from `@tangle-network/agent-runtime/kernel`, or your own `ExternalOptimizerModelCall` (copy `examples/_shared/openai-compatible-owner.ts`). The canonical doc is `docs/campaign-proposers.md`. Runnable paths: `examples/self-improve-optimizer/` (selfImprove + official GEPA) and `examples/compare-optimization-methods/` (method comparison). diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c398482..674b1ad2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,29 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- --- +## [0.160.0] — 2026-08-21 + +### Removed + +- Every Eval-owned paid model transport (#539). Agent Eval owns comparison, scoring, and durable evidence; it no longer executes a paid model, accepts a provider URL, or holds a credential. The caller supplies a `ChatClient`, and on Agent Runtime `profileChatClient` / `profileOptimizerModelCall` are that transport, so exact `AgentProfile` identity, retries, usage, cache accounting, and interruption safety stop being optional. + - `createChatClient` loses its `router`, `direct-provider`, and `cli-bridge` variants. `custom`, `sandbox-sdk`, and `mock` remain, and `ChatTransport` narrows to those three. + - The root barrel no longer exports `callLlm`, `callLlmJson`, `LlmClient`, `LlmClientOptions`, `assertLlmRoute`, `LlmRouteRequirements`, or `probeLlm`. `assertLlmRoute` and `probeLlm` are deleted outright: the caller holds the endpoint, so the caller owns both the route check and the reachability probe. The canonical contract stays public — `LlmCallRequest`, `LlmCallResult` (including `logprobs`, `toolCalls`, `servedModel`), `LlmMessage`, `LlmUsage`, `costReceiptFromLlm`, `costReceiptFromLlmError`, `maximumChargeForLlmRequest`, `isTransientLlmError`, `stripFencedJson`. + - `createOpenAiCompatibleExecutionOwner` is gone from `/campaign`. Agent Runtime already owns that role with `profileOptimizerModelCall`, which executes one exact `AgentProfile` and reports profile-digest evidence; two owners for one role was the defect. `examples/_shared/openai-compatible-owner.ts` is the caller-side reference implementation, and it is example code, not a published export. + - `multishot/router.ts` is deleted with `routerCompletion`, `requireRouterApiKey`, and `defaultRouterBaseUrl`. `runMultishot`, `runMultishotMatrix`, and `runJudge` now require a caller-supplied `MultishotTransport`; `JudgeConfig.transport` is required and `JUDGE_MODEL` is no longer read from the environment. `MultishotToolExecutor` receives `{ transport, signal }` instead of `{ apiKey, baseUrl, signal }`, and the optional `toolTransport` names the leg the built-in delegate tools run on. `estimateRouterCost` is now `estimateMultishotCost` in `multishot/cost.ts`. + - `preflightModels` and `assertModelsServed` take `request: ModelEndpointRequest` instead of `baseUrl` and `apiKey`. Agent Eval asks for a `list-models` or a `probe` check and reads the `Response`, so status, the provider's own `error.message`, `budgetExhausted`, and served-model substitution stay exactly as detectable as before. + - `runIntentMatchJudge`, `runSemanticConceptJudge`, `handleJudge`, `dispatchRpc`, and `createApp` take `chat: ChatClient` (plus optional `pricing`) instead of `llm: LlmClientOptions`. `/v1/judge` refuses with `llm_not_configured` (503) when no transport is configured, which replaces the old route assertion. + - `runEvalCampaign` takes `chatFactory: (wiring: CampaignChatWiring) => ChatClient` instead of `llmOpts`, and `CampaignRunContext.chat` replaces `ctx.llmOpts`. The campaign passes each run's `rawSink` and `runId` into the factory, so a transport that binds them still satisfies `assertRunCaptured`'s raw-coverage check. The campaign fingerprint now folds a caller-declared `executionRef` where it previously folded the base URL and provider it can no longer see. + +### Added + +- `paidJsonChat` collapses the four hand-rolled copies of "reserve the priced maximum, call the transport with a stable call id, settle the receipt, parse the JSON answer" that the two judges and the wire judge endpoint each carried. +- `LlmChargeBounds`: the narrow bound inputs `maximumChargeForLlmRequest` actually reads, so a caller can price a request without naming a transport options type. +- `examples/_shared/openai-compatible-owner.ts` exposes one OpenAI-compatible endpoint two ways — `openAiCompatibleChatClient` for judges and workers, `openAiCompatibleExecutionOwner` for the optimizer surface — as the reference for what caller-owned execution looks like. + +### Changed + +- The `agent-eval` binary is the one place in the package that reads a provider credential, and it is documented as such. `agent-eval serve` / `rpc` / `rpc-batch` build their own `ChatClient` from `AGENT_EVAL_LLM_*` (or the `OPENAI_*` / `TANGLE_*` equivalents) inside `src/cli-config.ts`. Both a base URL and a key are required; a half-configured server refuses instead of calling an unintended endpoint. + ## [0.159.1] — 2026-08-21 ### Added diff --git a/README.md b/README.md index 11b1071c..ca63af97 100644 --- a/README.md +++ b/README.md @@ -110,21 +110,25 @@ Every row is a function you call. Each links to a runnable example. ## Configure Model Calls Benchmarks, user drivers, executors, built-in judges, completion checkers, and judge adapters all take the same `ChatClient`. +You own model execution: Agent Eval issues no provider request and never receives a provider credential. ```ts import { createChatClient } from '@tangle-network/agent-eval' const chat = createChatClient({ - transport: 'router', - apiKey: process.env.TANGLE_API_KEY!, + transport: 'custom', defaultModel: 'openai/gpt-4.1', maximumAttempts: 3, + chat: async (request, opts) => myProviderClient(request, opts), }) ``` -Use `direct-provider` for an OpenAI-compatible endpoint, `cli-bridge` for a local subscription, `sandbox-sdk` for Sandbox, or `custom` to adapt another SDK. +On Agent Runtime, `profileChatClient({ profile, executor, context })` from `@tangle-network/agent-runtime/kernel` is that transport: every call runs one exact `AgentProfile` and reports its measured usage, retries, and served model identity. +Use `sandbox-sdk` for Sandbox and `mock` in tests. A custom adapter must return a `ChatResponse` and declare `maximumAttempts` before a capped cost account can dispatch it. +`ChatResponse` carries the whole execution record across that boundary: the served model id, measured input/output/reasoning/cached tokens, billed USD or an explicit unknown, the finish reason, and the per-token log probabilities the expectation judge scores on. + The official GEPA and SkillOpt optimizers run through a Python bridge. Install commands, version pins, and the reason for each pin: [GEPA](./docs/campaign-proposers.md#install-official-gepa), diff --git a/clients/python/README.md b/clients/python/README.md index 23a89b43..c1eaae62 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -25,6 +25,10 @@ export AGENT_EVAL_LLM_MODEL=gpt-4.1-mini `OPENAI_BASE_URL`, `OPENAI_API_KEY`, and `OPENAI_MODEL` are also accepted. The endpoint receives the content, rubric, and context passed to `client.judge()`. +The `agent-eval` binary is the only part of the package that reads a provider credential. +It is a server process, so it configures its own endpoint the way every server does; the TypeScript library holds no key and executes no paid model. +Without both a base URL and a key, `judge()` fails with `llm_not_configured` instead of calling an unintended endpoint. + ## Judge Content ```python diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 9cad5339..db8c2552 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-eval-rpc" -version = "0.159.1" +version = "0.160.0" description = "Python RPC client, official optimizer bridge, and DSPy metric adapter for @tangle-network/agent-eval." readme = "README.md" requires-python = ">=3.10" diff --git a/clients/python/src/agent_eval_rpc/__init__.py b/clients/python/src/agent_eval_rpc/__init__.py index 5b9cdada..cede44cb 100644 --- a/clients/python/src/agent_eval_rpc/__init__.py +++ b/clients/python/src/agent_eval_rpc/__init__.py @@ -53,7 +53,7 @@ try: __version__ = version("agent-eval-rpc") except PackageNotFoundError: - __version__ = "0.159.1" + __version__ = "0.160.0" __all__ = [ "Client", diff --git a/clients/python/uv.lock b/clients/python/uv.lock index d6db0b72..044e94f1 100644 --- a/clients/python/uv.lock +++ b/clients/python/uv.lock @@ -34,7 +34,7 @@ conflicts = [[ [[package]] name = "agent-eval-rpc" -version = "0.159.1" +version = "0.160.0" source = { editable = "." } dependencies = [ { name = "filelock" }, diff --git a/docs/building-doctrine.md b/docs/building-doctrine.md index 86dc5b6c..a550d093 100644 --- a/docs/building-doctrine.md +++ b/docs/building-doctrine.md @@ -6,7 +6,7 @@ How every fleet agent that consumes `agent-eval` is built. Each rule is mechanic Every hard-coded model id or endpoint default is verifiable against the live router. Membership in `{baseUrl}/models` is the free check; an optional 1-token probe per model confirms the router will actually serve it. A default the router cannot serve is a config bug caught before the run, not a runtime surprise that silently degrades into a stub. Backend ids are namespaced by binding: cli-bridge ids (`claude-code/*`, `kimi-code/*`, `opencode/*`) never appear as defaults in code reachable from production: bridge use is an explicit env opt-in, never an implicit fallback. -Enforced by: `preflightModels` (membership + optional probe) and `assertModelsServed` (gate that names every unreachable id with status + detail). +Enforced by: `preflightModels` (membership + optional probe, over a caller-owned `request` function) and `assertModelsServed` (gate that names every unreachable id with status + detail). ## 1a. Reachable is not the same as identified @@ -20,14 +20,14 @@ Snapshot resolution (`gpt-4o-mini` → `gpt-4o-mini-2024-07-18`) pins a floating Substitution (`gpt-4.1-mini` → `gemini-2.5-flash-lite`) mislabels every number the call produces. A response that echoes no model id at all is unproven, which fails closed rather than defaulting to agreement. -Enforced by: `assertServedModel` / `assertServedModels` per call, `assertCrossFamilyServed` for panel diversity computed over the ids that answered, `LlmClientOptions.assertServedModel` to enforce it at the transport, and `assertModelsServed` (probe mode), which now fails a substituted id exactly as it fails a dead one. +Enforced by: `assertServedModel` / `assertServedModels` per call, `assertCrossFamilyServed` for panel diversity computed over the ids that answered, `ChatResponse.servedModel` carrying the identity the caller's transport observed, and `assertModelsServed` (probe mode), which now fails a substituted id exactly as it fails a dead one. `assertCrossFamily` reads requested ids and therefore proves configuration only — reach for the served-side check wherever the diversity claim is load-bearing. ## 2. Probe the platform before peeling client layers When a request fails, one direct call against the live endpoint bisects platform-versus-client before any code-level debugging begins. A 401 from the router on a `model_not_found` is the platform telling you the default is dead; a connection refused is the platform being unreachable. Establish which side is at fault with a probe first, then debug only the side that is actually broken. -Enforced by: `preflightModels({ probe: true })`: the probe is the platform-side bisection, carrying the router's own `error.message` back to the caller. +Enforced by: `preflightModels({ probe: true, request })`: the probe is the platform-side bisection, carrying the endpoint's own `error.message` back to the caller. ## 3. Agent-produced findings are hypotheses diff --git a/docs/campaign-proposers.md b/docs/campaign-proposers.md index d812b106..da4902d8 100644 --- a/docs/campaign-proposers.md +++ b/docs/campaign-proposers.md @@ -313,21 +313,30 @@ With `optimizer`, every recipe stage must use the standard `gepa` engine or a me Agent Eval receives no provider key, enforces the declared request and token budget, and records the execution owner's exact usage and opaque finite JSON evidence. `maxProposerCostUsd` also limits each individual GEPA engine stage. -When no execution package owns the call, build the callback with `createOpenAiCompatibleExecutionOwner` from `/campaign`: +`optimizer.call` is always caller code. +Agent Eval owns no model transport and never receives a provider credential. -```ts -import { createOpenAiCompatibleExecutionOwner } from '@tangle-network/agent-eval/campaign' +On agent-runtime, use `profileOptimizerModelCall`, which executes one exact `AgentProfile` and reports profile-digest evidence: -const call = createOpenAiCompatibleExecutionOwner({ - baseUrl: 'https://api.openai.com/v1', - apiKey: process.env.LLM_API_KEY!, - model: 'gpt-4.1-mini', +```ts +import { profileOptimizerModelCall } from '@tangle-network/agent-runtime/kernel' + +const call = profileOptimizerModelCall({ + profile: optimizerProfile, + context: 'prompt optimizer', + executor: { + backend: 'router', + routerBaseUrl: process.env.LLM_BASE_URL!, + routerKey: process.env.LLM_API_KEY!, + }, pricing: { inputUsdPerMillion: 0.4, outputUsdPerMillion: 1.6 }, }) ``` -It executes each admitted request against any OpenAI-compatible `/chat/completions` endpoint and returns the typed outcome with a JSON-clean receipt. -The credential stays inside the owner closure; the proxy still enforces every budget and identity check. +Without agent-runtime, implement `ExternalOptimizerModelCall` over the OpenAI-compatible client you already have. +`examples/_shared/openai-compatible-owner.ts` is a complete minimal implementation to copy. +The callback resolves with one success or failure result and never rejects, because a rejection loses the execution record and fails the optimizer attempt. +The credential stays in your process; the proxy still enforces every budget and identity check. ### Metered agent CLI engines diff --git a/docs/multishot-golden-records.md b/docs/multishot-golden-records.md index adba50a1..556609d4 100644 --- a/docs/multishot-golden-records.md +++ b/docs/multishot-golden-records.md @@ -45,13 +45,13 @@ It refuses an `only` id the catalog does not hold, so a stale id after a rename An external conversation engine plugs into the matrix through one seam: `RunMultishotMatrixOptions.runShot`. The matrix runs that engine for every cell and reads its result through `MultishotCellOutput`, so the same golden checks grade any engine that implements the seam. -The matrix pair is `assertMultishotMatrixGoldenScenario` / `checkMultishotMatrixGoldenScenario`; both take a `runDir` the engine may write into, and both install a deterministic judge wire on `globalThis.fetch` for the duration of the run. -That wire is process-wide, so run matrix checks serially within one process and keep other fetch traffic out of it. -Both rules are enforced, not just documented: a second concurrent install throws, and the wire fails loud on any request it does not recognise rather than answering it. +The matrix pair is `assertMultishotMatrixGoldenScenario` / `checkMultishotMatrixGoldenScenario`; both take a `runDir` the engine may write into. +Each judge carries a scripted `MultishotTransport`, exactly like the agent and driver legs, so a matrix check owns no process-wide resource and two checks may run at once. +The judge transport fails loud on a system prompt it does not recognise rather than answering it. ## Determinism rules -Every scenario is a closed system: scripted transports, scripted tool executors, a fixed persona and profile, fixed token budgets. +Every scenario is a closed system: scripted transports on every leg including the judges, scripted tool executors, a fixed persona and profile, fixed token budgets. No network, no random number, and no clock in a COMPARED field — the fixture envelope carries a `recordedAt` stamp as provenance, and nothing compares it. Matrix cells run one at a time, so the request ledger is a property of the conversation engine rather than of how two engines interleave their microtasks. diff --git a/docs/public-api.md b/docs/public-api.md index ac1f86ec..d6295e68 100644 --- a/docs/public-api.md +++ b/docs/public-api.md @@ -7,10 +7,10 @@ Generated by `pnpm api:census` on demand — this is a dated reading, not a gate | measure | count | | --- | --- | | export subpaths | 27 | -| published value exports (subpath x symbol) | 1360 | -| distinct symbols | 1191 | -| production | 910 | -| planned | 212 | +| published value exports (subpath x symbol) | 1352 | +| distinct symbols | 1183 | +| production | 903 | +| planned | 211 | | none | 238 | Type-only exports are not listed: a type binds no runtime surface, and removing one cannot break a caller at run time. @@ -77,7 +77,7 @@ A `none` row is a deletion candidate, not a deletion order. A symbol stays when ### `.` -323 value exports — 272 production, 32 planned, 19 none. +318 value exports — 267 production, 32 planned, 19 none. | symbol | consumer | evidence | | --- | --- | --- | @@ -103,7 +103,6 @@ A `none` row is a deletion candidate, not a deletion order. A symbol stays when | `assertCapabilityHeadroom` | production | blueprint-agent:scripts/experiments/lib/validity-gates.ts | | `assertCrossFamily` | production | agent-builder:frontier/judges/artifact-head-to-head.ts | | `assertCrossFamilyServed` | planned | doc: docs/building-doctrine.md | -| `assertLlmRoute` | production | creative-agent:eval/leaderboard.ts | | `assertNoHiddenLeak` | planned | doc: docs/design/statistics-decisions.md | | `assertProductBenchmarkRun` | production | creative-agent:eval/research-package.ts | | `assertRealBackend` | production | blueprint-agent:scripts/experiments/lib/canonical/assert-real-backend.ts | @@ -131,15 +130,12 @@ A `none` row is a deletion candidate, not a deletion order. A symbol stays when | `buildTrajectory` | production | agent-runtime:src/runtime/supervise/trajectory-recorder.ts | | `calibrateJudge` | production | creative-agent:eval/calibrate-judges.ts | | `calibrateJudgeContinuous` | planned | consumer tests: tax-agent:tests/eval/lib/judge-sentinel.ts | -| `callLlm` | production | agent-dev-container:products/intelligence/api/src/lib/optimization-engine.ts | -| `callLlmJson` | production | agent-builder:scripts/regenerate.ts | -| `canonicalize` | production | agent-dev-container:products/intelligence/api/src/lib/ingest-optimization-mapping.ts | | `canonicalJson` | production | agent-knowledge:src/benchmarks/memory-recovery.ts | | `capabilityHeadroom` | production | blueprint-agent:scripts/experiments/lib/validity-gates.ts | | `captureFetchToRawSink` | production | creative-agent:eval/canonical-runner.ts | | `certificationEvidenceDigest` | production | this package: src/completion-verifier.ts:38 | | `checkCanaries` | production | blueprint-agent:scripts/experiments/lib/contamination-preflight.ts | -| `checkServedModel` | production | this package: src/integrity/preflight.ts:29 | +| `checkServedModel` | production | this package: src/integrity/preflight.ts:33 | | `checkTraceContracts` | production | creative-agent:eval/lib/trace-contracts.ts | | `clamp01` | production | agent-dev-container:products/intelligence/api/src/lib/consultant/executor.ts | | `classifyFailure` | production | phony:products/builder/api/src/eval/autoresearch/trace-analyst.ts | @@ -197,7 +193,7 @@ A `none` row is a deletion candidate, not a deletion order. A symbol stays when | `domainEvidencePattern` | production | blueprint-agent:scripts/experiments/lib/analyze-vb-run/session-loader.ts | | `dominates` | production | blueprint-agent:scripts/experiments/lib/competition-analytics.ts | | `ensembleJudge` | production | blueprint-agent:scripts/experiments/lib/qa-grader.ts | -| `eProcess` | production | this package: src/campaign/gates/sequential.ts:36 | +| `eProcess` | production | this package: src/campaign/gates/sequential.ts:35 | | `EquivalenceProtocolError` | planned | doc: docs/verification-strategies.md | | `equivalenceVerdict` | planned | doc: docs/verdicts.md | | `ERROR_COUNT_PATTERNS` | production | blueprint-agent:scripts/experiments/lib/error-count-extractor.ts | @@ -266,13 +262,13 @@ A `none` row is a deletion candidate, not a deletion order. A symbol stays when | `knowledgeReadinessTracePayload` | production | agent-dev-container:products/intelligence/api/src/routes/project-engines.ts | | `leaderboard` | production | blueprint-agent:scripts/experiments/lib/cross-harness-leaderboard.ts | | `LlmCallError` | production | agent-dev-container:products/intelligence/api/src/lib/router-model-owner.ts | -| `LlmClient` | production | agent-builder:scripts/harness-eval-run.ts | | `llmJudge` | production | agent-runtime:examples/agentic-data-creation/offline-fixtures.ts | | `LlmResponseError` | production | agent-dev-container:products/intelligence/api/src/lib/optimization-engine.ts | | `loadScorecard` | production | ai-trading-blueprint:evals/src/trading/scorecard-integration.ts | | `localCommandRunner` | production | blueprint-agent:scripts/experiments/lib/command-runner.ts | | `makeFinding` | production | agent-dev-container:products/intelligence/api/src/lib/consultant/candidates.ts | | `makeProposalFinding` | production | agent-runtime:bench/src/swe-arena/outer-loop.mts | +| `manifestContentDigest` | production | this package: src/campaign/gates/sequential.ts:34 | | `MANN_WHITNEY_EXACT_MAX_STATES` | none | — | | `MANN_WHITNEY_EXACT_MAX_WORK` | none | — | | `mannWhitneyU` | production | blueprint-agent:scripts/experiments/lib/competition-analytics.ts | @@ -320,7 +316,6 @@ A `none` row is a deletion candidate, not a deletion order. A symbol stays when | `passAtK` | planned | doc: docs/design/statistics-decisions.md | | `pearsonR` | production | this package: src/builder-eval/correlation.ts:18 | | `preflightModels` | production | loops:src/preflight.ts | -| `probeLlm` | production | blueprint-agent:scripts/experiments/lib/llm-router.ts | | `productBenchmarkRepoIdentity` | production | creative-agent:eval/research-package.ts | | `ProductClient` | production | creative-agent:eval/e2e/creative-product-harness.ts | | `profile` | production | agent-app:src/profile/index.ts | @@ -367,7 +362,7 @@ A `none` row is a deletion candidate, not a deletion order. A symbol stays when | `selfImprove` | production | agent-app:src/eval-campaign/index.ts | | `SEMANTIC_CONCEPT_JUDGE_VERSION` | production | blueprint-agent:scripts/experiments/lib/semantic-audit.ts | | `ServedCrossFamilyError` | none | only this package's tests: src/integrity/served-model.test.ts:2 | -| `servedModelAcceptable` | production | this package: src/integrity/preflight.ts:29 | +| `servedModelAcceptable` | production | this package: src/integrity/preflight.ts:33 | | `spearmanR` | production | blueprint-agent:scripts/experiments/lib/benchmark-validity-report.mjs | | `stripFencedJson` | production | blueprint-agent:scripts/experiments/compare-analysis-reports.ts | | `subjectiveEval` | production | creative-agent:eval/control/creative-onboarding.ts | @@ -392,7 +387,7 @@ A `none` row is a deletion candidate, not a deletion order. A symbol stays when | `verbosityBias` | production | blueprint-agent:scripts/experiments/calibration/judge-agreement.ts | | `VERIFICATION_STRATEGIES` | planned | example: examples/verify-without-an-answer-key/index.ts:13 | | `VERIFICATION_STRATEGY_SOURCES` | none | only this package's tests: src/verification-strategy.test.ts:11 | -| `verifyAgentProfileCell` | production | this package: src/eval-campaign.ts:41 | +| `verifyAgentProfileCell` | production | this package: src/eval-campaign.ts:39 | | `verifyCompletion` | production | agent-app:src/eval/index.ts | | `viteDeployRunner` | production | blueprint-agent:scripts/experiments/lib/deploy-runner.ts | | `weightedComposite` | production | agent-app:src/eval/index.ts | @@ -589,7 +584,7 @@ A `none` row is a deletion candidate, not a deletion order. A symbol stays when ### `./campaign` -118 value exports — 75 production, 16 planned, 27 none. +117 value exports — 75 production, 15 planned, 27 none. | symbol | consumer | evidence | | --- | --- | --- | @@ -620,9 +615,8 @@ A `none` row is a deletion candidate, not a deletion order. A symbol stays when | `componentSurfaceIdentityMaterial` | none | — | | `composeGate` | production | discovery-lab:tools/confirmation-gate.mjs | | `costFromLedgerSummary` | production | supervisor-lab:bench/drain/seat.ts | -| `createOpenAiCompatibleExecutionOwner` | planned | example: examples/_shared/optimizer-execution-owner.ts:1 | | `createProfileMatrixPlan` | production | discovery-lab:tools/run-profile-confirmation.mjs | -| `createReferenceEquivalenceJudge` | none | only this package's tests: src/reference-equivalence-judge.test.ts:8 | +| `createReferenceEquivalenceJudge` | none | only this package's tests: src/reference-equivalence-judge.test.ts:13 | | `createRunCostLedger` | production | agent-dev-container:products/intelligence/api/src/lib/eval-engine.ts | | `createSearchHistoryReceipt` | production | this package: src/campaign/search-ledger-recording.ts:21 | | `crowdedFrontierParent` | planned | doc: docs/campaign-proposers.md | @@ -725,7 +719,7 @@ A `none` row is a deletion candidate, not a deletion order. A symbol stays when | `compareOptimizationMethods` | production | agent-app:src/eval-campaign/index.ts | | `composeGate` | production | discovery-lab:tools/confirmation-gate.mjs | | `createChatClient` | production | agent-dev-container:products/intelligence/api/src/lib/intent-audit-analyst.ts | -| `createReferenceEquivalenceJudge` | none | only this package's tests: src/reference-equivalence-judge.test.ts:8 | +| `createReferenceEquivalenceJudge` | none | only this package's tests: src/reference-equivalence-judge.test.ts:13 | | `defaultProductionGate` | production | agent-app:src/eval-campaign/index.ts | | `defineAgentEval` | planned | example: examples/evaluate-a-change/index.ts:10 | | `diffGenerations` | none | only this package's tests: tests/contract-diff.test.ts:2 | @@ -759,14 +753,14 @@ A `none` row is a deletion candidate, not a deletion order. A symbol stays when | `parseCodeAgentJsonl` | planned | named in discovery-lab (bind not in the import graph) | | `parseCodeAgentJsonlFile` | none | only this package's tests: tests/contract-code-agent-intake.test.ts:6 | | `partitionRunsByAuthoringModel` | none | only this package's tests: tests/intake-agent-trace.test.ts:3 | -| `REFERENCE_EQUIVALENCE_INPUT_LIMITS` | none | only this package's tests: src/reference-equivalence-judge.test.ts:8 | -| `REFERENCE_EQUIVALENCE_JUDGE_VERSION` | none | only this package's tests: src/reference-equivalence-judge.test.ts:8 | +| `REFERENCE_EQUIVALENCE_INPUT_LIMITS` | none | only this package's tests: src/reference-equivalence-judge.test.ts:13 | +| `REFERENCE_EQUIVALENCE_JUDGE_VERSION` | none | only this package's tests: src/reference-equivalence-judge.test.ts:13 | | `runAgentProfileImprovementExperiment` | production | agent-runtime:src/intelligence/authored-profile-improvement.ts | | `runCampaign` | production | agent-app:src/eval-campaign/index.ts | | `runCandidateExperiment` | production | agent-runtime:src/intelligence/improvement-cycle.ts | | `runEval` | production | ai-trading-blueprint:evals/src/sim/multishot-user-sim.ts | | `runImprovementLoop` | production | blueprint-agent:scripts/experiments/vb-improve-trajectory.ts | -| `runReferenceEquivalenceJudge` | none | only this package's tests: src/reference-equivalence-judge.test.ts:8 | +| `runReferenceEquivalenceJudge` | none | only this package's tests: src/reference-equivalence-judge.test.ts:13 | | `sealAgentProfileImprovementExperiment` | production | agent-runtime:src/intelligence/authored-profile-improvement.ts | | `sealAgentProfileImprovementSuite` | production | agent-runtime:src/intelligence/profile-improvement-experiment.ts | | `sealAgentProfileImprovementTask` | production | agent-runtime:scripts/fixtures/packed-cohort-consumer.ts | @@ -798,32 +792,31 @@ A `none` row is a deletion candidate, not a deletion order. A symbol stays when | `BOOTSTRAP_GATE_MIN_N` | production | discovery-lab:tools/confirmation-activation.mjs | | `buildEvidenceVector` | planned | doc: docs/experiment.md | | `buildFunnel` | planned | doc: docs/experiment.md | -| `canonicalize` | production | agent-dev-container:products/intelligence/api/src/lib/ingest-optimization-mapping.ts | | `classifyReissue` | none | only this package's tests: tests/experiment/preregistration-acceptance.test.ts:11 | | `clusteredPower` | planned | doc: docs/charter.md | | `comparePairedArms` | production | agent-knowledge:src/memory/experiment/learning-metrics.ts | | `composeFunnels` | planned | doc: docs/experiment.md | -| `computeEstimand` | production | this package: src/experiment/define.ts:20 | -| `computeInterval` | production | this package: src/experiment/define.ts:20 | +| `computeEstimand` | production | this package: src/experiment/define.ts:21 | +| `computeInterval` | production | this package: src/experiment/define.ts:21 | | `createEvidenceReceipt` | planned | consumer tests: agent-runtime:tests/integration/runtime-eval-pursuit-evidence.test.ts | | `defineExperiment` | planned | doc: docs/charter.md | | `DesignRefusalError` | none | only this package's tests: tests/experiment/power.test.ts:10 | -| `eProcess` | production | this package: src/campaign/gates/sequential.ts:36 | +| `eProcess` | production | this package: src/campaign/gates/sequential.ts:35 | | `evaluateCondition` | planned | named in agent-dev-container (bind not in the import graph) | -| `evaluateHaltRule` | production | this package: src/experiment/define.ts:20 | -| `evaluateHypothesis` | none | only this package's tests: tests/tier2.test.ts:7 | -| `evaluateIdentityGate` | production | this package: src/experiment/define.ts:20 | -| `evaluateOracleDeterminismGate` | production | this package: src/experiment/define.ts:20 | -| `evaluatePopulationReproducibilityGate` | production | this package: src/experiment/define.ts:20 | -| `evaluatePowerFloorGate` | production | this package: src/experiment/define.ts:20 | +| `evaluateHaltRule` | production | this package: src/experiment/define.ts:21 | +| `evaluateHypothesis` | none | only this package's tests: tests/tier2.test.ts:8 | +| `evaluateIdentityGate` | production | this package: src/experiment/define.ts:21 | +| `evaluateOracleDeterminismGate` | production | this package: src/experiment/define.ts:21 | +| `evaluatePopulationReproducibilityGate` | production | this package: src/experiment/define.ts:21 | +| `evaluatePowerFloorGate` | production | this package: src/experiment/define.ts:21 | | `evaluatePredicate` | production | this package: src/experiment/funnel.ts:17 | -| `evaluateProvenanceGate` | production | this package: src/experiment/define.ts:20 | +| `evaluateProvenanceGate` | production | this package: src/experiment/define.ts:21 | | `EVIDENCE_AUTHORITY_KINDS` | none | — | | `EVIDENCE_RECEIPT_VERSION` | none | — | | `EVIDENCE_STATES` | none | only this package's tests: src/experiment/evidence-record.test.ts:4 | | `evidenceRegistryRecordSchema` | none | — | -| `executeAdmissionRule` | production | this package: src/experiment/define.ts:57 | -| `executeDecisionRule` | production | this package: src/experiment/define.ts:20 | +| `executeAdmissionRule` | production | this package: src/experiment/define.ts:58 | +| `executeDecisionRule` | production | this package: src/experiment/define.ts:21 | | `ExperimentTracker` | production | phony:products/builder/api/src/eval/evolution.ts | | `fileExperimentStore` | production | phony:products/builder/api/src/eval/evolution.ts | | `FunnelIntegrityError` | none | only this package's tests: tests/experiment/funnel.test.ts:7 | @@ -833,7 +826,8 @@ A `none` row is a deletion candidate, not a deletion order. A symbol stays when | `INDEPENDENT_EVIDENCE_AUTHORITY_KINDS` | none | — | | `inMemoryExperimentStore` | none | only this package's tests: src/experiment-tracker.test.ts:2 | | `isIndependentEvidence` | planned | consumer tests: agent-runtime:tests/integration/runtime-eval-pursuit-evidence.test.ts | -| `MatchedBudgetError` | none | only this package's tests: tests/experiment/budget-and-seal.test.ts:7 | +| `manifestContentDigest` | production | this package: src/campaign/gates/sequential.ts:34 | +| `MatchedBudgetError` | none | only this package's tests: tests/experiment/budget-and-seal.test.ts:8 | | `mcnemar` | production | agent-runtime:bench/src/swe-arena/analyze.ts | | `mcnemarPower` | production | supervisor-lab:bench/deepswe/headroom.ts | | `mcnemarRequiredN` | production | blueprint-agent:scripts/experiments/analyze-convergence.ts | @@ -852,16 +846,16 @@ A `none` row is a deletion candidate, not a deletion order. A symbol stays when | `paretoSignificanceGate` | production | agent-app:src/eval-campaign/index.ts | | `parseEvidenceRegistryRecord` | none | only this package's tests: src/experiment/evidence-record.test.ts:4 | | `powerPreflight` | production | discovery-lab:tools/run-sequential-profile-improvement.mjs | -| `projectNLadderBudget` | production | this package: src/experiment/define.ts:20 | +| `projectNLadderBudget` | production | this package: src/experiment/define.ts:21 | | `readField` | planned | named in agent-dev-container (bind not in the import graph) | | `renderEvidenceIndex` | none | only this package's tests: src/experiment/evidence-record.test.ts:4 | | `renderFunnelTable` | planned | example: examples/sealed-experiment/index.ts:12 | | `requiredPairedSampleSize` | production | discovery-lab:tools/design-gate.mjs | | `requiredSampleSize` | planned | doc: docs/design/statistics-decisions.md | -| `runSelectionRule` | production | this package: src/experiment/define.ts:20 | -| `runUniformPassBudget` | production | this package: src/experiment/define.ts:20 | +| `runSelectionRule` | production | this package: src/experiment/define.ts:21 | +| `runUniformPassBudget` | production | this package: src/experiment/define.ts:21 | | `sealExperiment` | planned | example: examples/sealed-experiment/index.ts:12 | -| `SealIntegrityError` | none | only this package's tests: tests/experiment/budget-and-seal.test.ts:7 | +| `SealIntegrityError` | none | only this package's tests: tests/experiment/budget-and-seal.test.ts:8 | | `sequentialCrossingHorizon` | none | only this package's tests: tests/sequential.test.ts:2 | | `sequentialDecide` | planned | doc: docs/experiment.md | | `sequentialPairedGate` | planned | consumer tests: legal-agent:tests/eval/self-improve.ts | @@ -869,7 +863,7 @@ A `none` row is a deletion candidate, not a deletion order. A symbol stays when | `validateEvidenceRegistry` | none | only this package's tests: src/experiment/evidence-record.test.ts:4 | | `verifyEvidenceReceipt` | planned | consumer tests: agent-runtime:tests/integration/runtime-eval-pursuit-evidence.test.ts | | `verifyManifest` | production | phony:products/builder/api/src/eval/champion-sign.ts | -| `verifyMatchedBudgets` | production | this package: src/experiment/define.ts:56 | +| `verifyMatchedBudgets` | production | this package: src/experiment/define.ts:57 | | `verifySealedExperiment` | planned | example: examples/sealed-experiment/index.ts:12 | | `wilson` | production | agent-dev-container:products/intelligence/api/src/lib/project-analysis-coverage-recommendations.ts | @@ -914,15 +908,16 @@ A `none` row is a deletion candidate, not a deletion order. A symbol stays when ### `./ledger-core` -15 value exports — 14 production, 0 planned, 1 none. +16 value exports — 15 production, 0 planned, 1 none. | symbol | consumer | evidence | | --- | --- | --- | | `appendLedgerLine` | production | this package: src/campaign/search-ledger-file.ts:3 | | `AtomicFileLockError` | production | this package: src/ledger-core/journal-file.ts:16 | -| `canonicalString` | production | this package: src/analyst/benchmark-response-cache.ts:6 | +| `canonicalString` | production | this package: src/agent-profile.ts:5 | | `FileLedgerJournal` | production | this package: src/campaign/search-ledger.ts:22 | | `hashCanonical` | production | discovery-lab:tools/provenance.mjs | +| `jsonDocument` | production | this package: src/analyst/benchmark-command-artifact.ts:2 | | `LEDGER_HASH_PATTERN` | production | this package: src/ledger-core/trusted-head.ts:55 | | `LedgerCanonicalizationError` | none | only this package's tests: src/ledger-core/canonical.test.ts:5 | | `probeAtomicFileLock` | production | this package: src/campaign/single-run-lock.ts:17 | @@ -966,7 +961,7 @@ A `none` row is a deletion candidate, not a deletion order. A symbol stays when ### `./multishot` -21 value exports — 13 production, 4 planned, 4 none. +18 value exports — 10 production, 4 planned, 4 none. | symbol | consumer | evidence | | --- | --- | --- | @@ -977,17 +972,14 @@ A `none` row is a deletion candidate, not a deletion order. A symbol stays when | `defaultDelegationTools` | production | gtm-agent:eval/lib/multishot-graph.ts | | `defaultMultishotDriverSystemPrompt` | none | only this package's tests: tests/multishot/shape-defaults.test.ts:10 | | `defaultMultishotOpener` | none | only this package's tests: tests/multishot/shape-defaults.test.ts:10 | -| `defaultRouterBaseUrl` | production | gtm-agent:eval/lib/multishot-graph.ts | | `defaultShapeFromProfile` | production | gtm-agent:eval/lib/multishot-graph.ts | -| `estimateRouterCost` | production | gtm-agent:eval/lib/multishot-graph.ts | +| `estimateMultishotCost` | production | this package: src/multishot/default-tools.ts:8 | | `MultishotDriverEmptyError` | production | gtm-agent:eval/lib/multishot-graph.ts | | `MultishotFatalToolError` | production | gtm-agent:eval/lib/multishot-graph.ts | | `MultishotShotResultError` | none | only this package's tests: tests/multishot/matrix-shot-seam.test.ts:18 | | `renderDimensions` | planned | consumer tests: tax-agent:tests/eval/multishot.ts | | `renderJsonFooter` | production | gtm-agent:eval/scoring/multishot-judges.ts | | `renderPersonaFacts` | none | only this package's tests: tests/multishot/shape-defaults.test.ts:10 | -| `requireRouterApiKey` | production | gtm-agent:eval/lib/multishot-graph.ts | -| `routerCompletion` | production | gtm-agent:eval/lib/multishot-graph.ts | | `runJudge` | production | gtm-agent:eval/monitoring/live-soak.ts | | `runMultishot` | production | agent-runtime:examples/p1-parity/arms.ts | | `runMultishotMatrix` | production | gtm-agent:eval/lib/multishot-matrix-graph.ts | @@ -1044,15 +1036,15 @@ A `none` row is a deletion candidate, not a deletion order. A symbol stays when | `AGENT_PROFILE_KINDS` | production | blueprint-agent:scripts/experiments/lib/agent-profile-cell.ts | | `agentProfileCellHashMaterial` | planned | consumer tests: blueprint-agent:scripts/experiments/lib/__tests__/agent-profile-cell.test.ts | | `agentProfileCellKey` | planned | consumer tests: blueprint-agent:scripts/experiments/lib/__tests__/agent-profile-cell.test.ts | -| `AgentProfileCellValidationError` | none | only this package's tests: tests/agent-profile-cell.test.ts:2 | -| `assertRunAgentProfileCell` | none | only this package's tests: tests/agent-profile-cell.test.ts:2 | -| `buildAgentInterfaceProfileCell` | none | only this package's tests: tests/agent-profile-cell.test.ts:2 | +| `AgentProfileCellValidationError` | none | only this package's tests: tests/agent-profile-cell.test.ts:3 | +| `assertRunAgentProfileCell` | none | only this package's tests: tests/agent-profile-cell.test.ts:3 | +| `buildAgentInterfaceProfileCell` | none | only this package's tests: tests/agent-profile-cell.test.ts:3 | | `buildAgentProfileCell` | production | ai-trading-blueprint:evals/src/trading/agent-profile-cell.ts | | `groupRunsByAgentProfileCell` | planned | consumer tests: legal-agent:tests/eval/matrix-multi.ts | -| `requireAgentProfileCell` | none | only this package's tests: tests/agent-profile-cell.test.ts:2 | +| `requireAgentProfileCell` | none | only this package's tests: tests/agent-profile-cell.test.ts:3 | | `toAgentProfileJson` | production | blueprint-agent:scripts/experiments/lib/agent-profile-cell.ts | | `validateAgentProfileCell` | production | blueprint-agent:apps/web/src/lib/.server/services/leaderboards/submission-profile-config.ts | -| `verifyAgentProfileCell` | production | this package: src/eval-campaign.ts:41 | +| `verifyAgentProfileCell` | production | this package: src/eval-campaign.ts:39 | ### `./reporting` @@ -1071,7 +1063,7 @@ A `none` row is a deletion candidate, not a deletion order. A symbol stays when | `pairedEvalueSequence` | production | creative-agent:src/lib/experiments/ab-design.ts | | `paretoChart` | production | workcomp-agent:eval/benchmark/select.ts | | `RESEARCH_REPORT_HARD_PAIR_FLOOR` | planned | doc: docs/research-report-methodology.md | -| `researchReport` | production | this package: src/eval-campaign.ts:60 | +| `researchReport` | production | this package: src/eval-campaign.ts:58 | | `rubricPredictiveValidity` | production | physim:apps/server/src/scripts/agent-eval/validate-rubrics.ts | | `summaryTable` | production | blueprint-agent:scripts/experiments/vb-runrecord-analysis.ts | | `wilcoxonSignedRank` | production | agent-builder:src/lib/.server/eval/loops/differential-eval.ts | @@ -1438,7 +1430,7 @@ A `none` row is a deletion candidate, not a deletion order. A symbol stays when | `createOtelTracingStore` | none | only this package's tests: tests/trace-contracts.test.ts:4 | | `DEFAULT_REDACTION_RULES` | production | agent-dev-container:products/intelligence/api/src/lib/redact.ts | | `DEFAULT_TRACE_ANALYST_BUDGETS` | production | agent-builder:src/lib/.server/eval/stores/d1-trace-analysis-store-adapter.ts | -| `defaultProviderRedactor` | production | this package: src/llm-client.ts:36 | +| `defaultProviderRedactor` | production | this package: src/llm-client.ts:34 | | `defaultTraceInsightPanel` | none | only this package's tests: src/trace-analyst/insights.test.ts:3 | | `describeTraceInsightScope` | production | blueprint-agent:scripts/experiments/lib/analyze-vb-run/report-html.ts | | `domainEvidencePattern` | production | blueprint-agent:scripts/experiments/lib/analyze-vb-run/session-loader.ts | @@ -1492,7 +1484,7 @@ A `none` row is a deletion candidate, not a deletion order. A symbol stays when | `OUTPUT_VALUE` | production | agent-runtime:src/runtime/supervise-surface.ts | | `planTraceInsightQuestions` | none | only this package's tests: src/trace-analyst/insights.test.ts:3 | | `projectOtlpFlatLine` | production | this package: src/trace-analyst/otlp-to-run-records.ts:63 | -| `providerFromBaseUrl` | production | this package: src/llm-client.ts:36 | +| `providerFromBaseUrl` | production | this package: src/llm-client.ts:34 | | `REDACTION_VERSION` | production | agent-dev-container:products/intelligence/api/src/lib/redact.ts | | `redactString` | production | agent-dev-container:products/intelligence/api/src/lib/redact.ts | | `redactValue` | production | traces:src/index.ts | @@ -1599,17 +1591,17 @@ A `none` row is a deletion candidate, not a deletion order. A symbol stays when | `ErrorResponseSchema` | production | this package: src/wire/openapi.ts:15 | | `FeedbackIngestResponseSchema` | production | this package: src/wire/openapi.ts:15 | | `FeedbackTrajectorySchema` | production | insurance-agent:src/routes/api.feedback.ts | -| `getBuiltinRubric` | production | this package: src/wire/handlers.ts:29 | +| `getBuiltinRubric` | production | this package: src/wire/handlers.ts:21 | | `handleFeedbackIngest` | production | this package: src/wire/server.ts:20 | | `handleJudge` | production | this package: src/wire/rpc.ts:15 | | `handleListRubrics` | production | this package: src/wire/rpc.ts:15 | | `handleTracesIngest` | production | this package: src/wire/server.ts:20 | | `handleVersion` | production | this package: src/cli.ts:19 | -| `hashRubric` | production | this package: src/wire/handlers.ts:30 | +| `hashRubric` | production | this package: src/wire/handlers.ts:22 | | `HealthResponseSchema` | production | this package: src/wire/openapi.ts:15 | | `JudgeRequestSchema` | production | this package: src/wire/openapi.ts:15 | | `JudgeResultSchema` | production | this package: src/wire/openapi.ts:15 | -| `listBuiltinRubrics` | production | this package: src/wire/handlers.ts:29 | +| `listBuiltinRubrics` | production | this package: src/wire/handlers.ts:21 | | `ListRubricsResponseSchema` | production | this package: src/wire/openapi.ts:15 | | `RubricDimensionSchema` | none | only this package's tests: tests/wire/schemas.test.ts:11 | | `RubricSchema` | none | only this package's tests: tests/wire/schemas.test.ts:11 | @@ -1621,5 +1613,5 @@ A `none` row is a deletion candidate, not a deletion order. A symbol stays when | `TracesIngestRequestSchema` | production | this package: src/wire/openapi.ts:15 | | `TracesIngestResponseSchema` | production | this package: src/wire/openapi.ts:15 | | `VersionResponseSchema` | production | this package: src/wire/openapi.ts:15 | -| `WIRE_VERSION` | production | this package: src/wire/handlers.ts:30 | +| `WIRE_VERSION` | production | this package: src/wire/handlers.ts:22 | | `WireError` | production | this package: src/wire/rpc.ts:15 | diff --git a/docs/trace-analysis.md b/docs/trace-analysis.md index d41a2fde..4a11468b 100644 --- a/docs/trace-analysis.md +++ b/docs/trace-analysis.md @@ -18,7 +18,7 @@ DSPy runs the research loop and a sandboxed Python interpreter. Agent Eval owns trace access, cancellation, cost accounting, and output validation. The calling application owns model execution, credentials, retries, and provider policy. -`callLlmJson()` and `createPublicBenchmarkDirectRunner()` are direct-call baselines. +`createPublicBenchmarkDirectRunner()` is the direct-call baseline; it runs through the loopback model proxy against a caller-owned execution owner, so agent-eval issues no provider request. They are not trace analysts. ## Install diff --git a/examples/README.md b/examples/README.md index 2d2f7dad..234c1162 100644 --- a/examples/README.md +++ b/examples/README.md @@ -48,7 +48,7 @@ pnpm tsx examples/compare-optimization-methods/index.ts Any OpenAI-compatible endpoint works; point `LLM_BASE_URL` at it and set `LLM_MODEL` to a model it serves. -The optimizer's reflection calls run through a default execution owner built from `LLM_BASE_URL` and `LLM_API_KEY` (`createOpenAiCompatibleExecutionOwner` from `/campaign`). +The optimizer's reflection calls run through a caller-owned execution owner. These examples supply their own, `_shared/openai-compatible-owner.ts`, built from `LLM_BASE_URL` and `LLM_API_KEY`. Set `OPTIMIZER_EXECUTION_OWNER_MODULE` to route them through your own execution package instead. Replace the example rates with the exact rates for your endpoint. Use `OPTIMIZERS=skillopt` for SkillOpt, or `OPTIMIZERS=gepa,skillopt` for a shared comparison. diff --git a/examples/_shared/extraction-task.ts b/examples/_shared/extraction-task.ts index 4d9dace5..b8431441 100644 --- a/examples/_shared/extraction-task.ts +++ b/examples/_shared/extraction-task.ts @@ -5,14 +5,13 @@ */ import { createHash } from 'node:crypto' +import type { ChatClient } from '../../src/analyst/chat-client' import type { DispatchContext, JudgeConfig, JudgeScore, Scenario } from '../../src/campaign' import type { CustomTokenPricing } from '../../src/cost-ledger' import { - callLlm, costReceiptFromLlm, costReceiptFromLlmError, type LlmCallRequest, - type LlmClientOptions, maximumChargeForLlmRequest, } from '../../src/llm-client' import type { RunRecord } from '../../src/run-record' @@ -216,7 +215,8 @@ export function parseJsonLoose(raw: string): Record | null { } export interface ExtractionWorkerOptions { - llm: LlmClientOptions + /** Caller-owned transport. Agent Eval executes no paid model. */ + chat: ChatClient model: string /** Per-call RunRecord sink used by assertRealBackend at the end. */ records: RunRecord[] @@ -234,10 +234,6 @@ export interface ExtractionWorkerOptions { * The returned function can be passed to runImprovementLoop or * compareOptimizationMethods. */ export function makeExtractionWorker(opts: ExtractionWorkerOptions) { - const llm = { - ...opts.llm, - ...(opts.customTokenPricing ? { customTokenPricing: opts.customTokenPricing } : {}), - } const timeoutMs = opts.timeoutMs ?? 30_000 const experimentId = opts.experimentId ?? 'extraction-task' return async function dispatchWithSurface( @@ -259,10 +255,15 @@ export function makeExtractionWorker(opts: ExtractionWorkerOptions) { const paid = await ctx.cost.runPaidCall({ actor: 'worker', model: opts.model, - maximumCharge: maximumChargeForLlmRequest(request, llm), - execute: (signal, callId) => callLlm(request, { ...llm, signal, idempotencyKey: callId }), - receipt: costReceiptFromLlm, - receiptFromError: costReceiptFromLlmError, + maximumCharge: maximumChargeForLlmRequest(request, { + ...(opts.chat.maximumAttempts === undefined + ? {} + : { maximumAttempts: opts.chat.maximumAttempts }), + ...(opts.customTokenPricing ? { customTokenPricing: opts.customTokenPricing } : {}), + }), + execute: (signal, callId) => opts.chat.chat(request, { signal, idempotencyKey: callId }), + receipt: (result) => costReceiptFromLlm(result, opts.customTokenPricing), + receiptFromError: (error) => costReceiptFromLlmError(error, opts.customTokenPricing), }) if (!paid.succeeded) throw paid.error const res = paid.value diff --git a/examples/_shared/openai-compatible-owner.ts b/examples/_shared/openai-compatible-owner.ts new file mode 100644 index 00000000..1ae96391 --- /dev/null +++ b/examples/_shared/openai-compatible-owner.ts @@ -0,0 +1,363 @@ +/** + * Caller-owned execution for the metered optimizer-model path. + * + * Agent Eval never executes a paid model. Its loopback proxy owns admission, + * budgets, identity checks, response bounds, and cost-ledger recording, then + * hands the exact admitted request to the package that owns execution. A + * product built on agent-runtime supplies `profileOptimizerModelCall`, which + * runs one exact `AgentProfile` and reports profile-digest evidence. + * + * This file is the minimal transport for a caller who has only an + * OpenAI-compatible `/chat/completions` endpoint. It is example code on + * purpose: the credential lives with the caller, not inside the package. + * Copy it into your own project and replace the transport with whatever + * client you already use. It exposes the same endpoint two ways — as the + * `ChatClient` every Agent Eval judge and worker takes, and as the + * `ExternalOptimizerModelCall` the optimizer surface takes. + */ + +import type { ChatClient, ChatRequest, ChatResponse } from '../../src/analyst/chat-client' +import type { + ExternalOptimizerChatRequest, + ExternalOptimizerModelCall, + ExternalOptimizerModelCallResult, +} from '../../src/campaign' +import type { CostReceiptInput, CustomTokenPricing } from '../../src/cost-ledger' +import { costForTokenPricing } from '../../src/cost-ledger' + +export interface OpenAiCompatibleOwnerOptions { + /** OpenAI-compatible base URL, ending at the `/v1` prefix. Always explicit. */ + baseUrl: string + /** Bearer credential. It stays in this process and never reaches the optimizer child. */ + apiKey: string + /** Model id recorded on a failure receipt when no provider response exists. */ + model: string + /** Exact endpoint rates, used when the provider omits a billed amount. */ + pricing?: CustomTokenPricing + /** Per-request deadline in milliseconds. Default 300,000. */ + timeoutMs?: number + /** Total provider attempts per admitted call. Default 2. */ + maximumAttempts?: number + /** Transport override for offline tests. */ + fetch?: typeof fetch +} + +interface WireUsage { + prompt_tokens?: unknown + completion_tokens?: unknown + total_tokens?: unknown + prompt_tokens_details?: { cached_tokens?: unknown } + completion_tokens_details?: { reasoning_tokens?: unknown } +} + +interface WireResponse { + model?: unknown + usage?: WireUsage + _response_cost?: unknown + cost_usd?: unknown + choices?: Array<{ + message?: { content?: string | null; tool_calls?: unknown } + finish_reason?: string | null + }> +} + +class HttpStatusError extends Error { + constructor( + readonly status: number, + message: string, + ) { + super(message) + this.name = 'HttpStatusError' + } +} + +/** Retry a rate limit, a server fault, and a bare network failure. Any other + * answer is the provider's real answer and must not be paid for twice. */ +function isRetryable(error: unknown): boolean { + if (error instanceof HttpStatusError) return error.status === 429 || error.status >= 500 + return error instanceof Error && error.name !== 'AbortError' +} + +/** + * The same endpoint as the `ChatClient` every judge, worker, and driver takes. + * + * `maximumAttempts` is declared so a capped cost account can price the worst + * case before dispatching, which is what Agent Eval requires of an opaque + * transport. + */ +export function openAiCompatibleChatClient( + options: OpenAiCompatibleOwnerOptions & { defaultModel?: string }, +): ChatClient { + const post = endpoint(options) + const defaultModel = options.defaultModel ?? options.model + return { + transport: 'custom', + defaultModel, + maximumAttempts: options.maximumAttempts ?? 2, + chat: async (request, callOpts) => + post( + { ...request, model: request.model ?? defaultModel }, + callOpts?.signal, + callOpts?.idempotencyKey, + ), + } +} + +/** + * Build the `ExternalOptimizerModelCall` Agent Eval's optimizer surface takes. + * + * The callback resolves with one success or failure result and never rejects: + * a rejection loses the execution record, which fails the optimizer attempt. + */ +export function openAiCompatibleExecutionOwner( + options: OpenAiCompatibleOwnerOptions, +): ExternalOptimizerModelCall { + const post = endpoint(options) + return async ({ callId, request, signal }): Promise => { + try { + const response = await post(request, signal, callId) + return { + succeeded: true, + response, + receipt: receiptFor(response, options.pricing), + execution: { + owner: 'openai-compatible', + endpoint: options.baseUrl, + model: response.model, + callId, + durationMs: response.durationMs, + }, + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + return { + succeeded: false, + error: message, + // No provider response arrived, so usage and cost stay UNKNOWN. A + // guessed zero would read downstream as a free call. + receipt: { + model: options.model, + inputTokens: 0, + outputTokens: 0, + usageUnknown: true, + costUnknown: true, + }, + execution: { + owner: 'openai-compatible', + endpoint: options.baseUrl, + model: options.model, + callId, + failed: true, + error: message, + }, + } + } + } +} + +/** + * One admitted request against the endpoint, retried only on a rate limit, a + * server fault, or a bare network failure. A caller cancel is final: retrying a + * cancelled intent spends money the caller already refused. + */ +function endpoint( + options: OpenAiCompatibleOwnerOptions, +): ( + request: ChatRequest | ExternalOptimizerChatRequest, + signal: AbortSignal | undefined, + idempotencyKey: string | undefined, +) => Promise { + for (const field of ['baseUrl', 'apiKey', 'model'] as const) { + const value = options[field] + if (typeof value !== 'string' || !value.trim() || value.trim() !== value) { + throw new Error( + `openAiCompatibleExecutionOwner: ${field} must be a trimmed, non-empty string`, + ) + } + } + const timeoutMs = options.timeoutMs ?? 300_000 + const maximumAttempts = options.maximumAttempts ?? 2 + const transport = options.fetch ?? fetch + const url = `${options.baseUrl.replace(/\/+$/, '')}/chat/completions` + + return async (request, signal, idempotencyKey) => { + const startedAt = Date.now() + let lastError: unknown = new Error('no attempt was made') + for (let attempt = 0; attempt < maximumAttempts; attempt++) { + const timeout = new AbortController() + const timer = setTimeout(() => timeout.abort(), timeoutMs) + try { + const res = await transport(url, { + method: 'POST', + headers: { + authorization: `Bearer ${options.apiKey}`, + 'content-type': 'application/json', + // Stable per-call id, reused across attempts so the provider can + // deduplicate a redriven paid call. + ...(idempotencyKey ? { 'idempotency-key': idempotencyKey } : {}), + }, + body: JSON.stringify(wireBody(request)), + signal: signal ? AbortSignal.any([timeout.signal, signal]) : timeout.signal, + }) + if (!res.ok) { + const text = await res.text().catch(() => '') + throw new HttpStatusError(res.status, `${res.status} ${text.slice(0, 400)}`) + } + return canonicalResponse( + (await res.json()) as WireResponse, + request.model ?? options.model, + Date.now() - startedAt, + options.pricing, + ) + } catch (error) { + lastError = error + if (signal?.aborted) break + if (attempt + 1 >= maximumAttempts || !isRetryable(error)) break + } finally { + clearTimeout(timer) + } + } + throw lastError instanceof Error ? lastError : new Error(String(lastError)) + } +} + +function wireBody(request: ChatRequest | ExternalOptimizerChatRequest): Record { + const body: Record = { + ...(request.model === undefined ? {} : { model: request.model }), + messages: request.messages.map((message) => + message.role === 'tool' + ? { role: 'tool', tool_call_id: message.toolCallId, content: message.content } + : message.toolCalls === undefined + ? { role: message.role, content: message.content } + : { + role: message.role, + content: message.content, + tool_calls: message.toolCalls.map((call) => ({ + id: call.id, + type: 'function', + function: { name: call.name, arguments: call.argumentsJson }, + })), + }, + ), + temperature: request.temperature ?? 0, + } + if (request.maxTokens != null) body.max_tokens = request.maxTokens + if (request.tools !== undefined) body.tools = request.tools + if (request.toolChoice !== undefined) body.tool_choice = request.toolChoice + if (request.thinking !== undefined) body.thinking = { type: request.thinking } + if (request.jsonSchema) { + body.response_format = { + type: 'json_schema', + json_schema: { + name: request.jsonSchema.name, + schema: request.jsonSchema.schema, + strict: true, + }, + } + } else if (request.jsonMode) { + body.response_format = { type: 'json_object' } + } + return body +} + +function canonicalResponse( + body: WireResponse, + requestedModel: string, + durationMs: number, + pricing: CustomTokenPricing | undefined, +): ChatResponse { + const choice = body.choices?.[0] + const content = choice?.message?.content ?? '' + const promptTokens = tokenCount(body.usage?.prompt_tokens) + const completionTokens = tokenCount(body.usage?.completion_tokens) + const totalTokens = tokenCount(body.usage?.total_tokens) + const cachedPromptTokens = tokenCount(body.usage?.prompt_tokens_details?.cached_tokens) + const reasoningTokens = tokenCount(body.usage?.completion_tokens_details?.reasoning_tokens) + const captured = promptTokens !== undefined && completionTokens !== undefined + const billedCostUsd = finiteCost(body._response_cost ?? body.cost_usd) + // The echoed id, kept apart from the attribution id: a provider that omits + // it reads as unproven, never as "the model I asked for". + const servedModel = typeof body.model === 'string' && body.model.trim() !== '' ? body.model : null + const estimated = + billedCostUsd === undefined && captured && pricing + ? costForTokenPricing(pricing, { + inputTokens: (promptTokens ?? 0) - (cachedPromptTokens ?? 0), + ...(cachedPromptTokens ? { cachedTokens: cachedPromptTokens } : {}), + outputTokens: completionTokens ?? 0, + }) + : undefined + return { + content, + ...(parseToolCalls(choice?.message?.tool_calls) ?? {}), + // 'tool_calls' is the OpenAI wire echo for a tool-calling stop; the + // canonical contract names the same stop cause 'tool_use'. + finishReason: + choice?.finish_reason === 'tool_calls' ? 'tool_use' : (choice?.finish_reason ?? null), + contentEmpty: content.trim().length === 0, + usage: { + promptTokens: promptTokens ?? 0, + completionTokens: completionTokens ?? 0, + totalTokens: totalTokens ?? (promptTokens ?? 0) + (completionTokens ?? 0), + captured, + reasoningTokens, + cachedPromptTokens, + }, + costUsd: billedCostUsd ?? estimated ?? null, + model: servedModel ?? requestedModel, + servedModel, + durationMs, + raw: body as unknown as Record, + } +} + +function receiptFor( + response: ChatResponse, + pricing: CustomTokenPricing | undefined, +): CostReceiptInput { + const cachedTokens = response.usage.cachedPromptTokens ?? 0 + const raw = response.raw as WireResponse + const billedCostUsd = finiteCost(raw._response_cost ?? raw.cost_usd) + return { + model: response.model, + inputTokens: Math.max(0, response.usage.promptTokens - cachedTokens), + outputTokens: response.usage.completionTokens, + ...(response.usage.reasoningTokens === undefined + ? {} + : { reasoningTokens: response.usage.reasoningTokens }), + ...(cachedTokens > 0 ? { cachedTokens } : {}), + ...(billedCostUsd !== undefined + ? { actualCostUsd: billedCostUsd } + : pricing && response.usage.captured !== false + ? { customTokenPricing: pricing } + : response.costUsd === null + ? { costUnknown: true } + : { estimatedCostUsd: response.costUsd }), + usageUnknown: response.usage.captured === false, + } +} + +function parseToolCalls(value: unknown): { toolCalls: ChatResponse['toolCalls'] } | undefined { + if (!Array.isArray(value) || value.length === 0) return undefined + return { + toolCalls: value.map((entry, index) => { + const record = entry as { id?: unknown; function?: { name?: unknown; arguments?: unknown } } + const fn = record?.function + if ( + typeof record?.id !== 'string' || + typeof fn?.name !== 'string' || + typeof fn?.arguments !== 'string' + ) { + throw new Error(`tool_calls[${index}] is not a function call with string arguments`) + } + return { id: record.id, name: fn.name, argumentsJson: fn.arguments } + }), + } +} + +function tokenCount(value: unknown): number | undefined { + return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : undefined +} + +function finiteCost(value: unknown): number | undefined { + return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : undefined +} diff --git a/examples/_shared/optimizer-execution-owner.ts b/examples/_shared/optimizer-execution-owner.ts index 020f8cd2..d08c8c64 100644 --- a/examples/_shared/optimizer-execution-owner.ts +++ b/examples/_shared/optimizer-execution-owner.ts @@ -1,9 +1,7 @@ -import { - createOpenAiCompatibleExecutionOwner, - type ExternalOptimizerModelCall, -} from '../../src/campaign' +import type { ExternalOptimizerModelCall } from '../../src/campaign' import type { CustomTokenPricing } from '../../src/cost-ledger' import { optionalNonNegativeNumberEnv } from './env' +import { openAiCompatibleExecutionOwner } from './openai-compatible-owner' export interface OptimizerExecutionOwner { /** Stable public identity for the exact execution configuration. */ @@ -21,12 +19,14 @@ interface OptimizerExecutionOwnerModule { /** * Resolve the execution owner for optimizer-model calls. * - * `OPTIMIZER_EXECUTION_OWNER_MODULE` selects a caller-owned execution package; - * Discovery supplies a module backed by Runtime and an exact AgentProfile. - * When it is unset, the owner is the package's OpenAI-compatible transport, - * built from the `LLM_BASE_URL` and `LLM_API_KEY` the examples already use. - * Optional `PRICE_IN_PER_M` and `PRICE_OUT_PER_M` supply cost estimates when - * the endpoint omits billed cost. + * Agent Eval owns no model transport, so the owner is always caller code. + * `OPTIMIZER_EXECUTION_OWNER_MODULE` selects an execution package — Discovery + * supplies a module backed by Runtime and an exact AgentProfile, and + * `profileOptimizerModelCall` from `@tangle-network/agent-runtime/kernel` is + * the production path. When it is unset, these examples fall back to their own + * minimal owner in `openai-compatible-owner.ts`, built from the `LLM_BASE_URL` + * and `LLM_API_KEY` they already use. Optional `PRICE_IN_PER_M` and + * `PRICE_OUT_PER_M` supply cost estimates when the endpoint omits billed cost. */ export async function loadOptimizerExecutionOwner(model: string): Promise { const moduleSpecifier = process.env.OPTIMIZER_EXECUTION_OWNER_MODULE?.trim() @@ -60,7 +60,7 @@ function defaultExecutionOwner(model: string): OptimizerExecutionOwner { ] if (missing.length > 0) { throw new Error( - `The default optimizer execution owner requires: ${missing.join(', ')}. ` + + `The example optimizer execution owner requires: ${missing.join(', ')}. ` + 'Set them, or set OPTIMIZER_EXECUTION_OWNER_MODULE to a module exporting createOptimizerExecutionOwner(model).', ) } @@ -79,7 +79,7 @@ function defaultExecutionOwner(model: string): OptimizerExecutionOwner { } return { callRef: `openai-compatible:${baseUrl}:${model}`, - call: createOpenAiCompatibleExecutionOwner({ + call: openAiCompatibleExecutionOwner({ baseUrl, apiKey, model, diff --git a/examples/benchmarks/gsm8k/compare-optimization-methods.ts b/examples/benchmarks/gsm8k/compare-optimization-methods.ts index f41ef8dd..b521ef07 100644 --- a/examples/benchmarks/gsm8k/compare-optimization-methods.ts +++ b/examples/benchmarks/gsm8k/compare-optimization-methods.ts @@ -60,11 +60,9 @@ import { summarizeBackendIntegrity, } from '../../../src/integrity/backend-integrity' import { - callLlm, costReceiptFromLlm, costReceiptFromLlmError, type LlmCallRequest, - type LlmClientOptions, maximumChargeForLlmRequest, } from '../../../src/llm-client' import type { RunRecord } from '../../../src/run-record' @@ -75,6 +73,7 @@ import { } from '../../_shared/env' import { GEPA_REFLECTION_ENGINE_CONFIG } from '../../_shared/gepa-reflection' import { assertMatchedMethodLimits } from '../../_shared/matched-method-limits' +import { openAiCompatibleChatClient } from '../../_shared/openai-compatible-owner' import { loadOptimizerExecutionOwner } from '../../_shared/optimizer-execution-owner' import { optimizerModelBudgetFromEnv } from '../../_shared/optimizer-model-budget' import { missingGsm8kEnv } from './env-validation' @@ -94,7 +93,13 @@ const BASE_URL = ( process.env.TANGLE_ROUTER_URL || 'https://router.tangle.tools/v1' ).trim() +if (!API_KEY) { + throw new Error( + 'LLM_API_KEY (or TANGLE_API_KEY) is required: this example owns its own transport', + ) +} const MODEL = process.env.LLM_MODEL || 'deepseek-v4-pro' +const WORKER_MAXIMUM_ATTEMPTS = 2 const PRICE_IN_PER_M = optionalNonNegativeNumberEnv('PRICE_IN_PER_M') const PRICE_CACHED_IN_PER_M = optionalNonNegativeNumberEnv('PRICE_CACHED_IN_PER_M') const PRICE_CACHE_WRITE_IN_PER_M = optionalNonNegativeNumberEnv('PRICE_CACHE_WRITE_IN_PER_M') @@ -182,13 +187,15 @@ interface Artifact { text: string } -const llm: LlmClientOptions = { - apiKey: API_KEY, +// The worker transport is caller code: Agent Eval holds no provider key. +const chat = openAiCompatibleChatClient({ baseUrl: BASE_URL, - maximumAttempts: 2, - defaultTimeoutMs: CALL_TIMEOUT_MS, - ...(CUSTOM_TOKEN_PRICING ? { customTokenPricing: CUSTOM_TOKEN_PRICING } : {}), -} + apiKey: API_KEY, + model: MODEL, + maximumAttempts: WORKER_MAXIMUM_ATTEMPTS, + timeoutMs: CALL_TIMEOUT_MS, + ...(CUSTOM_TOKEN_PRICING ? { pricing: CUSTOM_TOKEN_PRICING } : {}), +}) const records: RunRecord[] = [] @@ -211,10 +218,13 @@ function makeWorker() { const paid = await ctx.cost.runPaidCall({ actor: 'worker', model: MODEL, - maximumCharge: maximumChargeForLlmRequest(request, llm), - execute: (signal, callId) => callLlm(request, { ...llm, signal, idempotencyKey: callId }), - receipt: costReceiptFromLlm, - receiptFromError: costReceiptFromLlmError, + maximumCharge: maximumChargeForLlmRequest(request, { + maximumAttempts: WORKER_MAXIMUM_ATTEMPTS, + ...(CUSTOM_TOKEN_PRICING ? { customTokenPricing: CUSTOM_TOKEN_PRICING } : {}), + }), + execute: (signal, callId) => chat.chat(request, { signal, idempotencyKey: callId }), + receipt: (result) => costReceiptFromLlm(result, CUSTOM_TOKEN_PRICING), + receiptFromError: (error) => costReceiptFromLlmError(error, CUSTOM_TOKEN_PRICING), }) if (!paid.succeeded) throw paid.error const res = paid.value @@ -477,7 +487,7 @@ async function main() { worker: { requestTimeoutMs: CALL_TIMEOUT_MS, maxOutputTokens: WORKER_MAX_TOKENS, - maximumAttempts: llm.maximumAttempts, + maximumAttempts: WORKER_MAXIMUM_ATTEMPTS, temperature: 0, customTokenPricing: CUSTOM_TOKEN_PRICING ?? null, }, diff --git a/examples/compare-optimization-methods/README.md b/examples/compare-optimization-methods/README.md index f0b49f94..1345a714 100644 --- a/examples/compare-optimization-methods/README.md +++ b/examples/compare-optimization-methods/README.md @@ -91,7 +91,7 @@ OPTIMIZERS=skillopt pnpm tsx examples/compare-optimization-methods/index.ts Set `SKILLOPT_PRICE_IN_PER_M` and `SKILLOPT_PRICE_OUT_PER_M` to the current exact rates for your endpoint before running SkillOpt. The example passes SkillOpt's `openai_compatible` traffic through Agent Eval's local proxy and then through the execution owner. -By default that owner is `createOpenAiCompatibleExecutionOwner` from `/campaign`, built from `LLM_BASE_URL` and `LLM_API_KEY`. +By default that owner is this repository's example owner, `examples/_shared/openai-compatible-owner.ts`, built from `LLM_BASE_URL` and `LLM_API_KEY`. Agent Eval owns no model transport; on agent-runtime the production owner is `profileOptimizerModelCall`. An `OPTIMIZER_EXECUTION_OWNER_MODULE` override must export `createOptimizerExecutionOwner(model)` and return `{ call, callRef }`. Discovery uses this module boundary to execute the model through Runtime with one exact AgentProfile. Set `SKILLOPT_MODEL` to use a different optimizer model. diff --git a/examples/compare-optimization-methods/index.ts b/examples/compare-optimization-methods/index.ts index 091e4fc3..c62b4438 100644 --- a/examples/compare-optimization-methods/index.ts +++ b/examples/compare-optimization-methods/index.ts @@ -25,7 +25,6 @@ import { skillOptOptimizationMethod, } from '../../src/campaign' import { assertRealBackend, summarizeBackendIntegrity } from '../../src/integrity/backend-integrity' -import type { LlmClientOptions } from '../../src/llm-client' import type { RunRecord } from '../../src/run-record' import { optionalNonNegativeNumberEnv, positiveIntegerEnv, positiveNumberEnv } from '../_shared/env' import { @@ -40,6 +39,7 @@ import { } from '../_shared/extraction-task' import { GEPA_REFLECTION_ENGINE_CONFIG } from '../_shared/gepa-reflection' import { assertMatchedMethodLimits } from '../_shared/matched-method-limits' +import { openAiCompatibleChatClient } from '../_shared/openai-compatible-owner' import { loadOptimizerExecutionOwner, type OptimizerExecutionOwner, @@ -214,17 +214,19 @@ const skillOptModelBudget = selectedNames.includes('skillopt') ? optimizerModelBudgetFromEnv('SKILLOPT', MAX_OPTIMIZER_MODEL_COST_USD, customTokenPricing) : undefined -const llm: LlmClientOptions = { - apiKey: API_KEY, +// The worker transport is caller code: Agent Eval holds no provider key. +const chat = openAiCompatibleChatClient({ baseUrl: BASE_URL, + apiKey: API_KEY, + model: MODEL, maximumAttempts: 2, - defaultTimeoutMs: CALL_TIMEOUT_MS, - ...(customTokenPricing ? { customTokenPricing } : {}), -} + timeoutMs: CALL_TIMEOUT_MS, + ...(customTokenPricing ? { pricing: customTokenPricing } : {}), +}) const records: RunRecord[] = [] const worker = makeExtractionWorker({ - llm, + chat, model: MODEL, records, ...(customTokenPricing ? { customTokenPricing } : {}), diff --git a/examples/self-improve-optimizer/README.md b/examples/self-improve-optimizer/README.md index 2be86a5b..30e87198 100644 --- a/examples/self-improve-optimizer/README.md +++ b/examples/self-improve-optimizer/README.md @@ -50,7 +50,7 @@ The script validates every required variable before it makes a paid call. ## Why It Is Built This Way - The ten cases live inline in `index.ts`; `selfImprove()` derives every partition from that one list, so GEPA can never see the held-out cases. -- The default execution owner (`createOpenAiCompatibleExecutionOwner`) supplies the metered model call GEPA reflection runs through; the provider key never reaches the Python child, and each reflection call is metered against the declared budget. +- The example execution owner (`_shared/openai-compatible-owner.ts`) supplies the metered model call GEPA reflection runs through; the provider key never reaches Agent Eval or the Python child, and each reflection call is metered against the declared budget. - The judge is deterministic field matching, so a score change traces to prompt content, not judge noise. - `budget.generations` stays unset because the external method owns its rounds. - `assertRealBackend` fails the run when any cell lacks a real backend receipt. diff --git a/examples/self-improve-optimizer/index.ts b/examples/self-improve-optimizer/index.ts index baa76537..49bad34f 100644 --- a/examples/self-improve-optimizer/index.ts +++ b/examples/self-improve-optimizer/index.ts @@ -16,15 +16,13 @@ // IN-REPO: relative imports so the example typechecks against the workspace. // COPY-PASTE INTO YOUR OWN PROJECT: change these to // import { selfImprove } from '@tangle-network/agent-eval/contract' -// import { -// createOpenAiCompatibleExecutionOwner, -// gepaOptimizationMethod, -// } from '@tangle-network/agent-eval/campaign' -// The public subpaths expose these names with the same shapes. -import { createOpenAiCompatibleExecutionOwner, gepaOptimizationMethod } from '../../src/campaign' +// import { gepaOptimizationMethod } from '@tangle-network/agent-eval/campaign' +// The public subpaths expose these names with the same shapes. The execution +// owner is yours: copy `_shared/openai-compatible-owner.ts`, or use +// `profileOptimizerModelCall` from `@tangle-network/agent-runtime/kernel`. +import { gepaOptimizationMethod } from '../../src/campaign' import { selfImprove } from '../../src/contract' import { assertRealBackend, summarizeBackendIntegrity } from '../../src/integrity/backend-integrity' -import type { LlmClientOptions } from '../../src/llm-client' import type { RunRecord } from '../../src/run-record' import { positiveIntegerEnv, positiveNumberEnv } from '../_shared/env' import { @@ -34,6 +32,10 @@ import { makeExtractionWorker, } from '../_shared/extraction-task' import { GEPA_REFLECTION_ENGINE_CONFIG } from '../_shared/gepa-reflection' +import { + openAiCompatibleChatClient, + openAiCompatibleExecutionOwner, +} from '../_shared/openai-compatible-owner' import { optimizerModelBudgetFromEnv } from '../_shared/optimizer-model-budget' // ── Environment, validated before any paid call ───────────────────────── @@ -137,14 +139,16 @@ const BASELINE_SURFACE = 'Extract the transaction info from the message as JSON. // ── Agent, judge, and the GEPA method ──────────────────────────────────── const records: RunRecord[] = [] -const llm: LlmClientOptions = { - apiKey: API_KEY, +// The worker transport is caller code: Agent Eval holds no provider key. +const chat = openAiCompatibleChatClient({ baseUrl: BASE_URL, + apiKey: API_KEY, + model: MODEL, maximumAttempts: 2, - defaultTimeoutMs: CALL_TIMEOUT_MS, -} + timeoutMs: CALL_TIMEOUT_MS, +}) const worker = makeExtractionWorker({ - llm, + chat, model: MODEL, records, timeoutMs: CALL_TIMEOUT_MS, @@ -152,11 +156,11 @@ const worker = makeExtractionWorker({ experimentId: 'self-improve-optimizer', }) -// The default execution owner wraps one OpenAI-compatible endpoint as the -// metered model call every official optimizer requires. Agent Eval's loopback -// proxy meters each reflection call against `budget`, and the provider key -// never reaches the Python child. -const optimizerCall = createOpenAiCompatibleExecutionOwner({ +// The execution owner is caller code: it wraps one OpenAI-compatible endpoint +// as the metered model call every official optimizer requires. Agent Eval's +// loopback proxy meters each reflection call against `budget`, and the +// provider key never reaches Agent Eval or the Python child. +const optimizerCall = openAiCompatibleExecutionOwner({ baseUrl: BASE_URL, apiKey: API_KEY, model: GEPA_MODEL, diff --git a/package.json b/package.json index 5b26225e..1462c6a6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-eval", - "version": "0.159.1", + "version": "0.160.0", "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.", "homepage": "https://github.com/tangle-network/agent-eval#readme", "repository": { diff --git a/scripts/record-multishot-golden.ts b/scripts/record-multishot-golden.ts index b905f897..0907ef32 100644 --- a/scripts/record-multishot-golden.ts +++ b/scripts/record-multishot-golden.ts @@ -154,13 +154,7 @@ async function captureMatrixScenario( const runDir = mkdtempSync(join(tmpdir(), 'multishot-golden-')) try { const runCase = scenario.build(runDir) - const restore = runCase.installJudgeWire() - let matrix: Awaited> - try { - matrix = await engine(runCase.options) - } finally { - restore() - } + const matrix = await engine(runCase.options) return { id: scenario.id, description: scenario.description, diff --git a/scripts/verify-package-exports.mjs b/scripts/verify-package-exports.mjs index 49d8f025..b9dbeada 100644 --- a/scripts/verify-package-exports.mjs +++ b/scripts/verify-package-exports.mjs @@ -170,7 +170,6 @@ try { type CostLedgerHandle as RootCostLedgerHandle, type ExactRegistryRunOpts as RootExactRegistryRunOpts, type LlmJudgeOptions as RootLlmJudgeOptions, - type LlmClientOptions, type Run, type RunRecord, type RunTokenUsage, @@ -303,8 +302,10 @@ try { // @ts-expect-error provider SDK types are not part of the public API type RemovedProviderSdk = import('@tangle-network/agent-eval')[${JSON.stringify(removedSdkType)}] const removedProviderSdk: RemovedProviderSdk = {} - // @ts-expect-error LlmClientOptions uses total maximumAttempts - type RemovedLlmMaxRetries = LlmClientOptions['maxRetries'] + // @ts-expect-error the credential-bearing transport left the public API (#539) + type RemovedLlmClientOptions = import('@tangle-network/agent-eval').LlmClientOptions + // @ts-expect-error agent-eval executes no paid model: callLlm left the public API (#539) + type RemovedCallLlm = typeof import('@tangle-network/agent-eval').callLlm // @ts-expect-error CostLedgerEntry was removed from the current-only API type RemovedCostLedgerEntry = import('@tangle-network/agent-eval').CostLedgerEntry // @ts-expect-error fixed-prompt judge factories were removed diff --git a/src/analyst/adapters.test.ts b/src/analyst/adapters.test.ts index 09d15261..b19f53b0 100644 --- a/src/analyst/adapters.test.ts +++ b/src/analyst/adapters.test.ts @@ -1,38 +1,17 @@ import { describe, expect, it, vi } from 'vitest' import { createSemanticConceptJudgeAdapter } from './adapters' +import { type ChatResponse, createChatClient } from './chat-client' import { AnalystRegistry } from './registry' describe('createSemanticConceptJudgeAdapter', () => { it('records one provider receipt instead of copying one cost onto every finding', async () => { - const fetchImpl = (async () => - new Response( - JSON.stringify({ - model: 'gpt-4o', - choices: [ - { - message: { - content: JSON.stringify({ - summary: 'all three concepts are absent', - concepts: ['one', 'two', 'three'].map((concept) => ({ - concept, - present: false, - score: 0, - evidence: `${concept} is absent from src/App.tsx`, - severity: 'major', - })), - }), - }, - }, - ], - usage: { prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 }, - _response_cost: 0.25, - }), - { status: 200, headers: { 'content-type': 'application/json' } }, - )) as unknown as typeof globalThis.fetch const registry = new AnalystRegistry() registry.register( createSemanticConceptJudgeAdapter({ - options: { model: 'gpt-4o', llm: { fetch: fetchImpl } }, + options: { + model: 'gpt-4o', + chat: callerTransport(async () => judgeResponse(['one', 'two', 'three'], 0.25)), + }, }), ) @@ -66,8 +45,8 @@ describe('createSemanticConceptJudgeAdapter', () => { const started = new Promise((resolve) => { markStarted = resolve }) - let finishProvider!: (response: Response) => void - const provider = new Promise((resolve) => { + let finishProvider!: (response: ChatResponse) => void + const provider = new Promise((resolve) => { finishProvider = resolve }) const registry = new AnalystRegistry() @@ -77,14 +56,10 @@ describe('createSemanticConceptJudgeAdapter', () => { options: { model: 'gpt-4o', maxTokens: 64, - llm: { - baseUrl: 'https://provider.invalid/v1', - maximumAttempts: 1, - fetch: async () => { - markStarted() - return provider - }, - }, + chat: callerTransport(async () => { + markStarted() + return provider + }), }, }), ) @@ -104,7 +79,7 @@ describe('createSemanticConceptJudgeAdapter', () => { await Promise.resolve() expect(completed).toBe(false) - finishProvider(providerResponse(0.25)) + finishProvider(judgeResponse(['one'], 0.25)) const result = await run expect(result.per_analyst[0]?.usage).toEqual({ @@ -129,14 +104,10 @@ describe('createSemanticConceptJudgeAdapter', () => { options: { model: 'gpt-4o', maxTokens: 64, - llm: { - baseUrl: 'https://provider.invalid/v1', - maximumAttempts: 1, - fetch: async () => { - markStarted() - return new Promise(() => {}) - }, - }, + chat: callerTransport(async () => { + markStarted() + return new Promise(() => {}) + }), }, }), ) @@ -162,6 +133,16 @@ describe('createSemanticConceptJudgeAdapter', () => { }) }) +/** Caller-owned transport: agent-eval issues no provider request itself. */ +function callerTransport(chat: () => Promise) { + return createChatClient({ + transport: 'custom', + defaultModel: 'gpt-4o', + maximumAttempts: 1, + chat, + }) +} + function semanticInput() { return { custom: { @@ -174,31 +155,23 @@ function semanticInput() { } } -function providerResponse(costUsd: number): Response { - return new Response( - JSON.stringify({ - model: 'gpt-4o', - choices: [ - { - message: { - content: JSON.stringify({ - summary: 'the requested concept is absent', - concepts: [ - { - concept: 'one', - present: false, - score: 0, - evidence: 'one is absent from src/App.tsx', - severity: 'major', - }, - ], - }), - }, - }, - ], - usage: { prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 }, - _response_cost: costUsd, +function judgeResponse(concepts: string[], costUsd: number): ChatResponse { + return { + content: JSON.stringify({ + summary: 'none of the requested concepts are implemented in src/App.tsx', + concepts: concepts.map((concept) => ({ + concept, + present: false, + score: 0, + evidence: `${concept} is absent from src/App.tsx`, + severity: 'major', + })), }), - { status: 200, headers: { 'content-type': 'application/json' } }, - ) + usage: { promptTokens: 100, completionTokens: 50, totalTokens: 150, captured: true }, + costUsd, + model: 'gpt-4o', + servedModel: 'gpt-4o', + durationMs: 1, + raw: { _response_cost: costUsd }, + } } diff --git a/src/analyst/adapters.ts b/src/analyst/adapters.ts index 5eb160fd..a0c249d0 100644 --- a/src/analyst/adapters.ts +++ b/src/analyst/adapters.ts @@ -278,13 +278,13 @@ export interface SemanticConceptJudgeAdapterOpts { id?: string area?: string /** Registry context owns cancellation and the per-analyst cost ledger. */ - options?: Omit + options: Omit /** Maximum post-cancellation wait for a provider receipt. Default 5 seconds. */ settlementTimeoutMs?: number } export function createSemanticConceptJudgeAdapter( - opts: SemanticConceptJudgeAdapterOpts = {}, + opts: SemanticConceptJudgeAdapterOpts, ): Analyst { const id = opts.id ?? 'semantic-concept-judge' const area = opts.area ?? 'concept-coverage' @@ -296,7 +296,7 @@ export function createSemanticConceptJudgeAdapter( inputKind: 'custom', cost: { kind: 'llm', - models: opts.options?.model ? [opts.options.model] : undefined, + models: opts.options.model ? [opts.options.model] : undefined, settlement_timeout_ms: settlementTimeoutMs, }, version: `${SEMANTIC_CONCEPT_JUDGE_VERSION}-adapter-${ADAPTER_REV}`, diff --git a/src/analyst/benchmark-implementation.ts b/src/analyst/benchmark-implementation.ts index dd039d87..db620474 100644 --- a/src/analyst/benchmark-implementation.ts +++ b/src/analyst/benchmark-implementation.ts @@ -10,7 +10,7 @@ export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([ ]) export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = - '642c8f09df3572c66e88420fa6a7cddc77e1271834ab579d185dd623b115b2bc' + '92d16f9b7f2f4d1b69855f4550a08dfd9873081c781d65b5de354a45852ff709' /** The published benchmark evidence was produced at this package version, by * the retired one-shot direct runner, before trace analysts moved to the @@ -137,7 +137,7 @@ export const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([ ]) export const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = - '5c4bdaeb3fbfb4760e9299bcc309c262134fecdee2f2f8653f0e99e2656c093e' + 'd04c601555d7de691495312c69a7b8584467ae03d841f91e80a1530705ccf69d' export function analystBenchmarkImplementationDigest() { return ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 diff --git a/src/analyst/benchmark-public-model.ts b/src/analyst/benchmark-public-model.ts index 6b96a433..5bd023da 100644 --- a/src/analyst/benchmark-public-model.ts +++ b/src/analyst/benchmark-public-model.ts @@ -376,6 +376,11 @@ export function runChunkedAnalystDefinition( callId: providerCallId, ...(context.signal ? { signal: context.signal } : {}), }) + // The only endpoint this client ever targets is the loopback + // model proxy started above: `modelProxy.baseUrl` is + // `http://127.0.0.1:/v1` with an ephemeral token, and the + // caller-owned execution owner behind it makes the paid call. + // agent-eval issues no provider request here. const llmOptions: LlmClientOptions = { baseUrl: modelProxy.baseUrl, apiKey: modelProxy.apiKey, diff --git a/src/analyst/chat-client.ts b/src/analyst/chat-client.ts index 5c252e89..a3d132b4 100644 --- a/src/analyst/chat-client.ts +++ b/src/analyst/chat-client.ts @@ -1,17 +1,13 @@ /** * Provider-neutral chat contract for every model call made by agent-eval. * - * Callers choose the transport at the package boundary with `createChatClient`. - * Evaluation code receives canonical requests and results without importing a - * provider SDK. + * The caller owns model execution. agent-eval issues no provider request and + * holds no provider credential: `createChatClient` binds a transport the + * caller supplies, and evaluation code receives canonical requests and results + * without importing a provider SDK. */ -import { - type LlmCallRequest, - type LlmCallResult, - LlmClient, - type LlmClientOptions, -} from '../llm-client' +import type { LlmCallRequest, LlmCallResult } from '../llm-client' /** * Unified chat interface using the package's canonical LLM request and result. @@ -29,10 +25,7 @@ export interface ChatClient { } export type ChatTransport = - | 'router' // router.tangle.tools — production paid models | 'sandbox-sdk' // box.streamPrompt() — chat completion via sandbox SDK - | 'cli-bridge' // local cli-bridge for dev / local-only runs - | 'direct-provider' // direct OpenAI / Anthropic / etc. — bypass router | 'custom' // caller-adapted SDK or transport | 'mock' // test-time injection @@ -56,13 +49,7 @@ export interface ChatCallOpts { // ── Factory ───────────────────────────────────────────────────────── -export type CreateChatClientOpts = - | RouterTransportOpts - | CliBridgeTransportOpts - | DirectProviderTransportOpts - | SandboxSdkTransportOpts - | CustomTransportOpts - | MockTransportOpts +export type CreateChatClientOpts = SandboxSdkTransportOpts | CustomTransportOpts | MockTransportOpts interface BaseTransportOpts { defaultModel?: string @@ -70,24 +57,6 @@ interface BaseTransportOpts { maximumAttempts?: number } -export interface RouterTransportOpts extends BaseTransportOpts { - transport: 'router' - baseUrl?: string - apiKey: string -} - -export interface CliBridgeTransportOpts extends BaseTransportOpts { - transport: 'cli-bridge' - baseUrl?: string - bearer?: string -} - -export interface DirectProviderTransportOpts extends BaseTransportOpts { - transport: 'direct-provider' - baseUrl: string - apiKey: string -} - /** * Sandbox-SDK transport. The caller supplies a canonical chat function for an * already-configured Sandbox handle, so agent-eval does not import the SDK. @@ -118,36 +87,6 @@ export interface MockTransportOpts extends BaseTransportOpts { */ export function createChatClient(opts: CreateChatClientOpts): ChatClient { switch (opts.transport) { - case 'router': - return wrapLlmClient( - opts.transport, - opts.defaultModel, - new LlmClient({ - baseUrl: opts.baseUrl ?? 'https://router.tangle.tools/v1', - apiKey: opts.apiKey, - maximumAttempts: opts.maximumAttempts, - } as LlmClientOptions), - ) - case 'cli-bridge': - return wrapLlmClient( - opts.transport, - opts.defaultModel, - new LlmClient({ - baseUrl: opts.baseUrl ?? 'http://127.0.0.1:3344/v1', - apiKey: opts.bearer ?? '', - maximumAttempts: opts.maximumAttempts, - } as LlmClientOptions), - ) - case 'direct-provider': - return wrapLlmClient( - opts.transport, - opts.defaultModel, - new LlmClient({ - baseUrl: opts.baseUrl, - apiKey: opts.apiKey, - maximumAttempts: opts.maximumAttempts, - } as LlmClientOptions), - ) case 'sandbox-sdk': return { transport: 'sandbox-sdk', @@ -172,36 +111,6 @@ export function createChatClient(opts: CreateChatClientOpts): ChatClient { } } -function wrapLlmClient( - transport: ChatTransport, - defaultModel: string | undefined, - inner: LlmClient, -): ChatClient { - return { - transport, - defaultModel, - maximumAttempts: inner.maximumAttempts, - chat: (req, callOpts) => { - const resolved = resolveModel(req, defaultModel) - const request: LlmCallRequest = { - model: resolved.model!, - messages: req.messages, - jsonMode: req.jsonMode, - jsonSchema: req.jsonSchema, - logprobs: req.logprobs, - temperature: req.temperature, - maxTokens: req.maxTokens, - thinking: req.thinking, - timeoutMs: req.timeoutMs, - } - return inner.call(request, { - signal: callOpts?.signal, - idempotencyKey: callOpts?.idempotencyKey, - }) - }, - } -} - function resolveModel(req: ChatRequest, defaultModel: string | undefined): ChatRequest { if (req.model) return req if (!defaultModel) { diff --git a/src/analyst/index.ts b/src/analyst/index.ts index 8f85d26b..645450e9 100644 --- a/src/analyst/index.ts +++ b/src/analyst/index.ts @@ -211,12 +211,9 @@ export type { ChatRequest, ChatResponse, ChatTransport, - CliBridgeTransportOpts, CreateChatClientOpts, CustomTransportOpts, - DirectProviderTransportOpts, MockTransportOpts, - RouterTransportOpts, SandboxSdkTransportOpts, } from './chat-client' export { createChatClient } from './chat-client' diff --git a/src/campaign/index.ts b/src/campaign/index.ts index ffa1dd88..2cf21411 100644 --- a/src/campaign/index.ts +++ b/src/campaign/index.ts @@ -219,10 +219,6 @@ export { LabeledScenarioStoreError, } from './labeled-store/fs-adapter' export { neutralizeText } from './neutralize' -export { - createOpenAiCompatibleExecutionOwner, - type OpenAiCompatibleExecutionOwnerOptions, -} from './openai-compatible-execution-owner' export type { OpenAICompatibleOptimizerModel, OptimizerModelBudget, diff --git a/src/campaign/openai-compatible-execution-owner.test.ts b/src/campaign/openai-compatible-execution-owner.test.ts deleted file mode 100644 index e2a96cca..00000000 --- a/src/campaign/openai-compatible-execution-owner.test.ts +++ /dev/null @@ -1,247 +0,0 @@ -import { describe, expect, it } from 'vitest' -import type { CostReceiptInput } from '../cost-ledger' -import { assertJsonValue, type ExternalOptimizerChatRequest } from './external-optimizer-contracts' -import { createOpenAiCompatibleExecutionOwner } from './openai-compatible-execution-owner' - -const REQUEST = Object.freeze({ - model: 'router/optimizer-model', - messages: Object.freeze([Object.freeze({ role: 'user' as const, content: 'reflect' })]), - maxTokens: 64, -}) as unknown as ExternalOptimizerChatRequest - -function okBody(): object { - return { - model: 'router/optimizer-model', - choices: [ - { message: { role: 'assistant', content: 'improved prompt' }, finish_reason: 'stop' }, - ], - usage: { prompt_tokens: 120, completion_tokens: 40, total_tokens: 160 }, - } -} - -function assertNoUndefinedValues(receipt: CostReceiptInput): void { - expect(() => assertJsonValue(receipt, 'receipt')).not.toThrow() - expect(Object.values(receipt)).not.toContain(undefined) -} - -describe('createOpenAiCompatibleExecutionOwner', () => { - it('executes the exact admitted request and returns a JSON-clean receipt', async () => { - const seen: Array<{ url: string; init: RequestInit }> = [] - const call = createOpenAiCompatibleExecutionOwner({ - baseUrl: 'https://endpoint.test/v1', - apiKey: 'secret-key', - model: 'router/optimizer-model', - fetch: (async (url: string, init: RequestInit) => { - seen.push({ url, init }) - return new Response(JSON.stringify(okBody()), { - status: 200, - headers: { 'content-type': 'application/json' }, - }) - }) as unknown as typeof fetch, - }) - - const result = await call({ - callId: 'call-1', - request: REQUEST, - signal: new AbortController().signal, - }) - - expect(result.succeeded).toBe(true) - if (!result.succeeded) throw new Error(result.error) - expect(result.response.content).toBe('improved prompt') - expect(result.receipt).toMatchObject({ - model: 'router/optimizer-model', - inputTokens: 120, - outputTokens: 40, - usageUnknown: false, - }) - assertNoUndefinedValues(result.receipt) - expect(() => assertJsonValue(result.execution, 'execution')).not.toThrow() - expect(result.execution).toMatchObject({ - owner: 'openai-compatible', - endpoint: 'https://endpoint.test/v1', - callId: 'call-1', - }) - - expect(seen).toHaveLength(1) - expect(seen[0]!.url).toBe('https://endpoint.test/v1/chat/completions') - const headers = seen[0]!.init.headers as Record - expect(headers.Authorization).toBe('Bearer secret-key') - expect(headers['Idempotency-Key']).toBe('call-1') - const body = JSON.parse(String(seen[0]!.init.body)) as Record - expect(body.model).toBe('router/optimizer-model') - expect(body.messages).toEqual([{ role: 'user', content: 'reflect' }]) - }) - - it('passes canonical tools to the wire and returns canonical tool calls', async () => { - const seen: Array<{ init: RequestInit }> = [] - const call = createOpenAiCompatibleExecutionOwner({ - baseUrl: 'https://endpoint.test/v1', - apiKey: 'secret-key', - model: 'router/optimizer-model', - fetch: (async (_url: string, init: RequestInit) => { - seen.push({ init }) - return new Response( - JSON.stringify({ - model: 'router/optimizer-model', - choices: [ - { - message: { - role: 'assistant', - content: null, - tool_calls: [ - { - id: 'call_1', - type: 'function', - function: { name: 'Bash', arguments: '{"command":"ls"}' }, - }, - ], - }, - finish_reason: 'tool_calls', - }, - ], - usage: { prompt_tokens: 120, completion_tokens: 40, total_tokens: 160 }, - }), - { status: 200, headers: { 'content-type': 'application/json' } }, - ) - }) as unknown as typeof fetch, - }) - - const tools = [ - { - type: 'function' as const, - function: { - name: 'Bash', - description: 'Run a command', - parameters: { type: 'object', properties: { command: { type: 'string' } } }, - }, - }, - ] - const request = Object.freeze({ - ...REQUEST, - tools: Object.freeze(tools), - toolChoice: 'auto', - }) as unknown as ExternalOptimizerChatRequest - - const result = await call({ - callId: 'call-tools', - request, - signal: new AbortController().signal, - }) - - expect(result.succeeded).toBe(true) - if (!result.succeeded) throw new Error(result.error) - expect(result.response.content).toBe('') - expect(result.response.toolCalls).toEqual([ - { id: 'call_1', name: 'Bash', argumentsJson: '{"command":"ls"}' }, - ]) - expect(result.response.finishReason).toBe('tool_use') - assertNoUndefinedValues(result.receipt) - - const body = JSON.parse(String(seen[0]!.init.body)) as Record - expect(body.tools).toEqual(tools) - expect(body.tool_choice).toBe('auto') - }) - - it('treats a tool-free answer to a tool-carrying request as a valid answer', async () => { - const call = createOpenAiCompatibleExecutionOwner({ - baseUrl: 'https://endpoint.test/v1', - apiKey: 'secret-key', - model: 'router/optimizer-model', - fetch: (async () => - new Response(JSON.stringify(okBody()), { - status: 200, - headers: { 'content-type': 'application/json' }, - })) as unknown as typeof fetch, - }) - - const request = Object.freeze({ - ...REQUEST, - tools: Object.freeze([ - { - type: 'function' as const, - function: { name: 'Bash', parameters: { type: 'object' } }, - }, - ]), - }) as unknown as ExternalOptimizerChatRequest - - const result = await call({ - callId: 'call-no-tools-used', - request, - signal: new AbortController().signal, - }) - - expect(result.succeeded).toBe(true) - if (!result.succeeded) throw new Error(result.error) - expect(result.response.content).toBe('improved prompt') - expect(result.response.toolCalls).toBeUndefined() - expect(result.response.finishReason).toBe('stop') - }) - - it('estimates cost from the configured pricing when the provider omits billed cost', async () => { - const pricing = { inputUsdPerMillion: 1, outputUsdPerMillion: 2 } - const call = createOpenAiCompatibleExecutionOwner({ - baseUrl: 'https://endpoint.test/v1', - apiKey: 'secret-key', - model: 'router/optimizer-model', - pricing, - fetch: (async () => - new Response(JSON.stringify(okBody()), { - status: 200, - headers: { 'content-type': 'application/json' }, - })) as unknown as typeof fetch, - }) - - const result = await call({ - callId: 'call-2', - request: REQUEST, - signal: new AbortController().signal, - }) - - expect(result.succeeded).toBe(true) - if (!result.succeeded) throw new Error(result.error) - expect(result.receipt.customTokenPricing).toEqual(pricing) - expect(result.receipt.actualCostUsd).toBeUndefined() - assertNoUndefinedValues(result.receipt) - }) - - it('returns a typed failure with an unknown-usage receipt on a provider error', async () => { - const call = createOpenAiCompatibleExecutionOwner({ - baseUrl: 'https://endpoint.test/v1', - apiKey: 'secret-key', - model: 'router/optimizer-model', - fetch: (async () => - new Response('{"error":"invalid api key"}', { status: 401 })) as unknown as typeof fetch, - }) - - const result = await call({ - callId: 'call-3', - request: REQUEST, - signal: new AbortController().signal, - }) - - expect(result.succeeded).toBe(false) - if (result.succeeded) throw new Error('expected a typed failure') - expect(result.error).toContain('401') - expect(result.receipt).toEqual({ - model: 'router/optimizer-model', - inputTokens: 0, - outputTokens: 0, - costUnknown: true, - usageUnknown: true, - }) - assertNoUndefinedValues(result.receipt) - expect(result.execution).toMatchObject({ failed: true, callId: 'call-3' }) - expect(() => assertJsonValue(result.execution, 'execution')).not.toThrow() - }) - - it('rejects a blank credential before any call executes', () => { - expect(() => - createOpenAiCompatibleExecutionOwner({ - baseUrl: 'https://endpoint.test/v1', - apiKey: ' ', - model: 'router/optimizer-model', - }), - ).toThrow(/apiKey/) - }) -}) diff --git a/src/campaign/openai-compatible-execution-owner.ts b/src/campaign/openai-compatible-execution-owner.ts deleted file mode 100644 index 94bce84f..00000000 --- a/src/campaign/openai-compatible-execution-owner.ts +++ /dev/null @@ -1,116 +0,0 @@ -import type { CustomTokenPricing } from '../cost-ledger' -import { - callLlm, - costReceiptFromLlm, - costReceiptFromLlmError, - type LlmCallRequest, -} from '../llm-client' -import type { - ExternalOptimizerModelCall, - ExternalOptimizerModelCallResult, -} from './external-optimizer-contracts' - -export interface OpenAiCompatibleExecutionOwnerOptions { - /** OpenAI-compatible base URL, ending at the `/v1` prefix. No default: the paid endpoint is always explicit. */ - baseUrl: string - /** Bearer credential sent as `Authorization`. It stays inside this owner and never reaches the optimizer process. */ - apiKey: string - /** Model id recorded on a failure receipt when no provider response exists. */ - model: string - /** Exact endpoint rates, used to estimate cost when the provider omits billed cost. */ - pricing?: CustomTokenPricing - /** Per-request deadline in milliseconds. Default: the transport default (300,000 ms). */ - timeoutMs?: number - /** Total provider attempts per admitted call. Default: 2. */ - maximumAttempts?: number - /** Fetch implementation override for tests. */ - fetch?: typeof fetch -} - -/** - * Execution owner for the metered optimizer-model path, backed by any - * OpenAI-compatible `/chat/completions` endpoint. - * - * The loopback proxy owns admission, budgets, identity checks, and - * cost-ledger recording. This owner only executes the exact admitted - * request and resolves with a typed outcome: a canonical response plus a - * JSON-clean receipt on success, or a public error plus an honest receipt - * on failure. It never rejects, because a rejection loses the execution - * record and fails the optimizer attempt. - * - * Canonical `tools`/`toolChoice` pass through to the wire as - * `tools`/`tool_choice`; response `tool_calls` come back as canonical - * `toolCalls`. A response that carries none when tools were sent is a valid - * model answer, not an error. - */ -export function createOpenAiCompatibleExecutionOwner( - options: OpenAiCompatibleExecutionOwnerOptions, -): ExternalOptimizerModelCall { - for (const field of ['baseUrl', 'apiKey', 'model'] as const) { - const value = options[field] - if (typeof value !== 'string' || !value.trim() || value.trim() !== value) { - throw new Error( - `createOpenAiCompatibleExecutionOwner: ${field} must be a trimmed, non-empty string`, - ) - } - } - const { baseUrl, apiKey, model, pricing, timeoutMs, maximumAttempts, fetch: fetchImpl } = options - return async ({ callId, request, signal }): Promise => { - // The proxy freezes the canonical request; the transport needs a mutable copy. - const transportRequest = structuredClone(request) as unknown as LlmCallRequest - try { - const wireResponse = await callLlm(transportRequest, { - baseUrl, - apiKey, - signal, - idempotencyKey: callId, - maximumAttempts: maximumAttempts ?? 2, - ...(timeoutMs === undefined ? {} : { defaultTimeoutMs: timeoutMs }), - ...(pricing ? { customTokenPricing: pricing } : {}), - ...(fetchImpl ? { fetch: fetchImpl } : {}), - }) - // 'tool_calls' is the OpenAI wire echo for a tool-calling stop; the - // canonical contract names the same stop cause 'tool_use'. - const response = - wireResponse.finishReason === 'tool_calls' - ? { ...wireResponse, finishReason: 'tool_use' } - : wireResponse - return { - succeeded: true, - response, - receipt: costReceiptFromLlm(response, pricing), - execution: { - owner: 'openai-compatible', - endpoint: baseUrl, - model: response.model, - callId, - durationMs: response.durationMs, - }, - } - } catch (error) { - const cause = error instanceof Error ? error : new Error(String(error)) - // A structured-response failure keeps its completed provider receipt. - // A transport fault carries none, so usage and cost stay unknown - // rather than becoming a guessed zero. - const receipt = costReceiptFromLlmError(cause, pricing) ?? { - model, - inputTokens: 0, - outputTokens: 0, - costUnknown: true, - usageUnknown: true, - } - return { - succeeded: false, - error: cause.message, - receipt, - execution: { - owner: 'openai-compatible', - endpoint: baseUrl, - model, - callId, - failed: true, - }, - } - } - } -} diff --git a/src/chat-json-call.ts b/src/chat-json-call.ts new file mode 100644 index 00000000..274f06fd --- /dev/null +++ b/src/chat-json-call.ts @@ -0,0 +1,87 @@ +/** + * One paid JSON model call through a caller-owned `ChatClient`, metered by the + * cost ledger. + * + * agent-eval executes no paid model: the transport is supplied by the caller + * and the credential never enters this package. What stays here is the + * accounting around the call — the priced maximum reserved before it runs, the + * stable call id forwarded as the provider idempotency key, the receipt + * settled from the response, and an honest unknown-usage receipt when the + * transport failed. + * + * The judges and the wire judge endpoint all make the same call in the same + * order; this is the one copy of that sequence. + */ + +import type { ChatClient, ChatResponse } from './analyst/chat-client' +import type { CostChannel, CostLedgerHandle, CostReceipt, CustomTokenPricing } from './cost-ledger' +import { + costReceiptFromLlm, + costReceiptFromLlmError, + extractJsonPayload, + type LlmCallRequest, + maximumChargeForLlmRequest, +} from './llm-client' + +export interface PaidJsonChatInput { + /** Caller-owned transport. One `chat()` call. */ + chat: ChatClient + /** The exact canonical request, including its JSON-mode or schema fields. */ + request: LlmCallRequest + ledger: CostLedgerHandle + channel: CostChannel + phase: string + actor: string + tags?: Record + signal?: AbortSignal + /** Endpoint rates used when the transport reports no billed amount. */ + pricing?: CustomTokenPricing +} + +export type PaidJsonChatResult = + | { succeeded: true; value: T; response: ChatResponse; receipt: CostReceipt } + | { succeeded: false; error: Error; receipt?: CostReceipt } + +/** Parse a JSON answer out of a model response. The transport may fence it. */ +export function parseJsonAnswer(response: ChatResponse, actor: string): T { + try { + return JSON.parse(extractJsonPayload(response.content)) as T + } catch (error) { + throw new Error( + `${actor}: model answer was not JSON — ${error instanceof Error ? error.message : String(error)}`, + ) + } +} + +export async function paidJsonChat(input: PaidJsonChatInput): Promise> { + const paid = await input.ledger.runPaidCall({ + channel: input.channel, + phase: input.phase, + actor: input.actor, + model: input.request.model, + ...(input.tags && Object.keys(input.tags).length > 0 ? { tags: input.tags } : {}), + maximumCharge: maximumChargeForLlmRequest(input.request, { + ...(input.chat.maximumAttempts === undefined + ? {} + : { maximumAttempts: input.chat.maximumAttempts }), + ...(input.pricing ? { customTokenPricing: input.pricing } : {}), + }), + ...(input.signal ? { signal: input.signal } : {}), + execute: (signal, callId) => input.chat.chat(input.request, { signal, idempotencyKey: callId }), + receipt: (response) => costReceiptFromLlm(response, input.pricing), + receiptFromError: (error) => costReceiptFromLlmError(error, input.pricing), + }) + if (!paid.succeeded) { + return { + succeeded: false, + error: paid.error, + ...(paid.receipt ? { receipt: paid.receipt } : {}), + } + } + return { + succeeded: true, + value: parseJsonAnswer(paid.value, input.actor), + response: paid.value, + receipt: paid.receipt, + } +} diff --git a/src/cli-config.test.ts b/src/cli-config.test.ts index a4ef0202..603788fb 100644 --- a/src/cli-config.test.ts +++ b/src/cli-config.test.ts @@ -1,22 +1,23 @@ import { describe, expect, it } from 'vitest' -import { resolveCliLlmConfig } from './cli-config' +import { resolveCliLlmConfig, resolveCliProviderRoute } from './cli-config' -describe('resolveCliLlmConfig', () => { - it('maps standard OpenAI variables to an explicit provider config', () => { +describe('resolveCliProviderRoute', () => { + it('maps standard OpenAI variables to an explicit provider route', () => { expect( - resolveCliLlmConfig({ + resolveCliProviderRoute({ OPENAI_API_KEY: ' openai-key ', OPENAI_MODEL: 'gpt-test', }), ).toEqual({ - client: { apiKey: 'openai-key', baseUrl: 'https://api.openai.com/v1' }, + apiKey: 'openai-key', + baseUrl: 'https://api.openai.com/v1', model: 'gpt-test', }) }) it('prefers agent-eval variables over provider-specific fallbacks', () => { expect( - resolveCliLlmConfig({ + resolveCliProviderRoute({ AGENT_EVAL_LLM_API_KEY: 'explicit-key', AGENT_EVAL_LLM_BASE_URL: 'https://provider.example/v1', AGENT_EVAL_LLM_MODEL: 'explicit-model', @@ -24,12 +25,32 @@ describe('resolveCliLlmConfig', () => { OPENAI_MODEL: 'fallback-model', }), ).toEqual({ - client: { apiKey: 'explicit-key', baseUrl: 'https://provider.example/v1' }, + apiKey: 'explicit-key', + baseUrl: 'https://provider.example/v1', model: 'explicit-model', }) }) - it('returns no provider config when no supported variables are set', () => { + it('refuses a half-configured route rather than calling an unintended endpoint', () => { + expect( + resolveCliProviderRoute({ AGENT_EVAL_LLM_BASE_URL: 'https://provider.example/v1' }), + ).toBeUndefined() + expect(resolveCliProviderRoute({})).toBeUndefined() + }) +}) + +describe('resolveCliLlmConfig', () => { + it('binds the resolved route into the transport the wire handlers take', () => { + const config = resolveCliLlmConfig({ + AGENT_EVAL_LLM_API_KEY: 'explicit-key', + AGENT_EVAL_LLM_BASE_URL: 'https://provider.example/v1', + AGENT_EVAL_LLM_MODEL: 'explicit-model', + }) + expect(config.model).toBe('explicit-model') + expect(config.chat?.defaultModel).toBe('explicit-model') + }) + + it('returns no transport when no supported variables are set', () => { expect(resolveCliLlmConfig({})).toEqual({}) }) }) diff --git a/src/cli-config.ts b/src/cli-config.ts index 250eb8f2..c2482405 100644 --- a/src/cli-config.ts +++ b/src/cli-config.ts @@ -1,11 +1,40 @@ -import type { LlmClientOptions } from './llm-client' +/** + * Provider configuration for the `agent-eval` binary. + * + * This is the ONE place in the package that turns an environment credential + * into a model transport, and it exists only inside the binary. The `agent-eval` + * server is a deployed process whose caller is a JSON-RPC or HTTP client in + * another language, so it cannot be handed a `ChatClient`; it reads its own + * credential the way every server does. The library never does: a TypeScript + * consumer binds its own transport and agent-eval holds no provider key. + */ + +import type { ChatClient } from './analyst/chat-client' +import { LlmClient, type LlmClientOptions } from './llm-client' export interface CliLlmConfig { - client?: LlmClientOptions + /** Judge transport for the wire handlers. Absent when no provider is configured. */ + chat?: ChatClient model?: string } -export function resolveCliLlmConfig(env: NodeJS.ProcessEnv = process.env): CliLlmConfig { +/** The endpoint the binary resolved from its environment. */ +export interface CliProviderRoute { + baseUrl: string + apiKey: string + model?: string +} + +/** + * Environment precedence for the binary's provider route, with no client built. + * + * Both halves are required: a base URL without a key, or a key without an + * endpoint, is a half-configured server. `/v1/judge` then refuses with + * `llm_not_configured` instead of calling an unintended endpoint. + */ +export function resolveCliProviderRoute( + env: NodeJS.ProcessEnv = process.env, +): CliProviderRoute | undefined { const explicitBaseUrl = nonEmpty(env.AGENT_EVAL_LLM_BASE_URL) const explicitApiKey = nonEmpty(env.AGENT_EVAL_LLM_API_KEY) const openAiApiKey = nonEmpty(env.OPENAI_API_KEY) @@ -20,11 +49,41 @@ export function resolveCliLlmConfig(env: NodeJS.ProcessEnv = process.env): CliLl const model = nonEmpty(env.AGENT_EVAL_LLM_MODEL) ?? nonEmpty(env.OPENAI_MODEL) ?? nonEmpty(env.TANGLE_MODEL) - const client = - baseUrl || apiKey - ? { ...(baseUrl ? { baseUrl } : {}), ...(apiKey ? { apiKey } : {}) } - : undefined - return { ...(client ? { client } : {}), ...(model ? { model } : {}) } + if (!baseUrl || !apiKey) return undefined + return { baseUrl, apiKey, ...(model ? { model } : {}) } +} + +export function resolveCliLlmConfig(env: NodeJS.ProcessEnv = process.env): CliLlmConfig { + const route = resolveCliProviderRoute(env) + if (!route) return {} + return { + chat: cliChatClient({ baseUrl: route.baseUrl, apiKey: route.apiKey }, route.model), + ...(route.model ? { model: route.model } : {}), + } +} + +function cliChatClient(options: LlmClientOptions, defaultModel: string | undefined): ChatClient { + const client = new LlmClient(options) + return { + transport: 'custom', + ...(defaultModel ? { defaultModel } : {}), + ...(client.maximumAttempts === undefined ? {} : { maximumAttempts: client.maximumAttempts }), + chat: (req, callOpts) => { + const model = req.model ?? defaultModel + if (!model) { + throw new Error( + 'agent-eval: no model on the request and no AGENT_EVAL_LLM_MODEL configured', + ) + } + return client.call( + { ...req, model }, + { + ...(callOpts?.signal ? { signal: callOpts.signal } : {}), + ...(callOpts?.idempotencyKey ? { idempotencyKey: callOpts.idempotencyKey } : {}), + }, + ) + }, + } } function nonEmpty(value: string | undefined): string | undefined { diff --git a/src/cli.ts b/src/cli.ts index 28d578bc..b964a0aa 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -84,7 +84,9 @@ Commands: Judge provider: Set AGENT_EVAL_LLM_BASE_URL, AGENT_EVAL_LLM_API_KEY, and AGENT_EVAL_LLM_MODEL. - OPENAI_* and TANGLE_* equivalents are also accepted. + OPENAI_* and TANGLE_* equivalents are also accepted. This binary is the only + place that reads a provider credential; the library never does. Without both + a base URL and a key, /v1/judge refuses with llm_not_configured. Without arguments, prints this help.` @@ -108,9 +110,8 @@ async function main(): Promise { const { server } = await startServerAsync({ port, host, - llm: llm.client, - judgeModel: llm.model, - llmRouteRequirements: { requireExplicitBaseUrl: true }, + ...(llm.chat ? { chat: llm.chat } : {}), + ...(llm.model ? { judgeModel: llm.model } : {}), }) // Keep process alive on SIGINT/SIGTERM const shutdown = (sig: string) => { @@ -130,18 +131,16 @@ async function main(): Promise { const [method] = positional const llm = resolveCliLlmConfig() return await runRpcOnce(method, { - llm: llm.client, - judgeModel: llm.model, - llmRouteRequirements: { requireExplicitBaseUrl: true }, + ...(llm.chat ? { chat: llm.chat } : {}), + ...(llm.model ? { judgeModel: llm.model } : {}), }) } case 'rpc-batch': { const [method] = positional const llm = resolveCliLlmConfig() return await runRpcBatch(method, { - llm: llm.client, - judgeModel: llm.model, - llmRouteRequirements: { requireExplicitBaseUrl: true }, + ...(llm.chat ? { chat: llm.chat } : {}), + ...(llm.model ? { judgeModel: llm.model } : {}), }) } case 'openapi': { diff --git a/src/eval-campaign.test.ts b/src/eval-campaign.test.ts index 3206b76a..fd9cb7a4 100644 --- a/src/eval-campaign.test.ts +++ b/src/eval-campaign.test.ts @@ -1,18 +1,21 @@ import { describe, expect, it } from 'vitest' +import { createChatClient } from './analyst/chat-client' import type { CampaignRunContext, CampaignRunOutcome, EvalCampaignOptions } from './eval-campaign' import { finalizeAbort, runEvalCampaign } from './eval-campaign' -import { assertLlmRoute, type LlmClientOptions } from './llm-client' import { TraceEmitter } from './trace/emitter' import { NoopRawProviderSink } from './trace/raw-provider-sink' import { InMemoryTraceStore } from './trace/store' -// A minimally-valid LLM config. routeRequirements is set to `{}` in every -// campaign below so assertLlmRoute does not gate on baseUrl/auth. -const LLM_OPTS: LlmClientOptions = { - baseUrl: 'https://api.example.test/v1', - apiKey: 'test-key', - provider: 'test', -} +/** The caller owns execution; no test here needs a real model answer. */ +const chatFactory = () => + createChatClient({ + transport: 'custom', + defaultModel: 'm@1', + maximumAttempts: 1, + chat: async () => { + throw new Error('no campaign test in this file calls the model') + }, + }) const TOKENS = { input: 1, output: 1 } @@ -45,8 +48,7 @@ function baseOpts( scenarios: [{ scenarioId: 's0' }], seeds: [0], commitSha: 'sha', - llmOpts: LLM_OPTS, - routeRequirements: {}, + chatFactory, storeFactory: () => new InMemoryTraceStore(), rawSinkFactory: () => new NoopRawProviderSink(), integrity: { @@ -234,9 +236,3 @@ describe('runEvalCampaign — genuine-error orphan handling', () => { expect(result.runs[0]!.outcome.holdoutScore).toBe(1) }) }) - -describe('preflight sanity', () => { - it('assertLlmRoute is exercised by the campaign (smoke)', () => { - expect(() => assertLlmRoute(LLM_OPTS, {})).not.toThrow() - }) -}) diff --git a/src/eval-campaign.ts b/src/eval-campaign.ts index b1324db5..e5738fdc 100644 --- a/src/eval-campaign.ts +++ b/src/eval-campaign.ts @@ -12,12 +12,10 @@ * `EvalCampaign` is the structural fix — consumers don't wire the * integrity surface themselves; the campaign owns it. Specifically: * - * - calls `assertLlmRoute` once at preflight before any work runs * - constructs a per-run `TraceStore` and `RawProviderSink` via factories - * - constructs the `TraceEmitter` with `onRunComplete: [analyst hook]` - * - hands the runner an `LlmClientOptions` pre-wired with the sink and - * trace context — the runner can't accidentally call an LLM without - * capturing the raw HTTP envelope + * - builds the run's `ChatClient` through `chatFactory`, handing it the + * run's raw sink and trace context — a transport built any other way has + * no raw HTTP envelope, and `assertRunCaptured` says so * - calls `assertRunCaptured` after every `endRun` and routes failures * through a configurable policy (`throw` / `mark_failed` / `log`) * - assembles per-run `RunRecord`s and runs `researchReport` at the end @@ -35,7 +33,7 @@ * - Distributed/cluster execution (concurrency is local async) * - Adaptive sampling / sequential interim looks * - Resume from partial state across crashes - * - LLM-call retry beyond what `LlmClient` already does + * - LLM-call retry beyond what the caller's transport already does */ import { @@ -44,7 +42,7 @@ import { buildAgentProfileCell, verifyAgentProfileCell, } from './agent-profile-cell' -import { assertLlmRoute, type LlmClientOptions, type LlmRouteRequirements } from './llm-client' +import type { ChatClient } from './analyst/chat-client' import { hashJson } from './pre-registration' import type { JudgeScoresRecord, @@ -103,11 +101,21 @@ export interface CampaignRunContext { store: TraceStore rawSink: RawProviderSink /** - * Pre-wired LLM client options — `rawSink` and `traceContext` are populated - * so any `callLlm(req, ctx.llmOpts)` automatically captures raw HTTP. The - * runner can spread additional fields if needed. + * The run's model transport, built by `chatFactory` with this run's + * `rawSink` and `runId` already bound. */ - llmOpts: LlmClientOptions + chat: ChatClient +} + +/** What the campaign binds into the run's transport. */ +export interface CampaignChatWiring { + /** + * Raw provider sink for this run. Bind it into the transport: the campaign's + * integrity check requires every LLM span to carry a matching raw request + * event, so a transport built without it fails `assertRunCaptured`. + */ + rawSink: RawProviderSink + runId: string } interface CampaignRunOutcomeFields { @@ -166,17 +174,20 @@ export interface EvalCampaignOptions { /** Git SHA the campaign is run against. Mandatory; `RunRecord` rejects unset. */ commitSha: string /** - * LLM client config. Augmented per-run with `rawSink` and `traceContext` - * before being passed to the runner. The campaign asserts this config - * matches `routeRequirements` once at preflight. + * Build the model transport for one run. agent-eval executes no paid model: + * the caller owns the transport and the credential never enters this + * package. The campaign calls this once per run and passes the run's raw + * provider sink and `runId`, so a transport that binds them captures the + * raw HTTP envelope `assertRunCaptured` checks for. */ - llmOpts: LlmClientOptions + chatFactory: (wiring: CampaignChatWiring) => ChatClient /** - * Default `{ requireExplicitBaseUrl: true, requireAuth: true }` — fail - * loud if the campaign would silently fall back to the public router or - * run unauthenticated. Override with an empty object to disable. + * Caller-declared identity of the execution route, folded into the campaign + * fingerprint so two campaigns run against different endpoints do not share + * one identity. agent-eval no longer knows the endpoint; the owner of + * execution names it. */ - routeRequirements?: LlmRouteRequirements + executionRef?: string /** * Per-run TraceStore factory. Common shape: a fresh store per run keyed * on `runId`. Implementations that share a store across the campaign @@ -273,7 +284,7 @@ export interface FailedRun { export interface EvalCampaignResult { campaignId: string - /** SHA-256 over canonicalised `(variantIds, scenarioIds, seeds, comparator, splitTag, baseUrl, provider, preregistrationHash)`. */ + /** SHA-256 over canonicalised `(variantIds, scenarioIds, seeds, comparator, splitTag, executionRef, preregistrationHash)`. */ campaignFingerprint: string preregistrationHash: string | null /** Successful runs only. Failed runs land in `failedRuns`. */ @@ -295,17 +306,10 @@ const DEFAULT_INTEGRITY: RunIntegrityExpectations = { requireOutcome: true, } -const DEFAULT_ROUTE: LlmRouteRequirements = { - requireExplicitBaseUrl: true, - requireAuth: true, -} - export async function runEvalCampaign( opts: EvalCampaignOptions, ): Promise { // ── Preflight ────────────────────────────────────────────────────── - assertLlmRoute(opts.llmOpts, opts.routeRequirements ?? DEFAULT_ROUTE) - if (opts.variants.length === 0) { throw new Error('runEvalCampaign: variants must be non-empty.') } @@ -341,8 +345,7 @@ export async function runEvalCampaign( const integrity = { ...DEFAULT_INTEGRITY, ...(opts.integrity ?? {}) } const onIntegrityFailure: CampaignIntegrityPolicy = opts.onIntegrityFailure ?? 'mark_failed' const now = opts.now ?? (() => Date.now()) - const baseUrl = (opts.llmOpts.baseUrl ?? '').replace(/\/+$/, '') - const provider = opts.llmOpts.provider ?? null + const executionRef = opts.executionRef ?? null const preregistrationHash = opts.preregistrationHash ?? null const rawSinkFactory = opts.rawSinkFactory ?? defaultRawSinkFactory(opts.workDir) @@ -355,8 +358,7 @@ export async function runEvalCampaign( seeds: [...seeds].sort((a, b) => a - b), splitTag, comparator: opts.report?.comparator ?? null, - baseUrl, - provider, + executionRef, preregistrationHash, }) @@ -447,11 +449,7 @@ export async function runEvalCampaign( // finalize it instead of orphaning it. Removed in the finally below. openRuns.set(runId, emitter) - const llmOpts: LlmClientOptions = { - ...opts.llmOpts, - rawSink, - traceContext: { runId }, - } + const chat = opts.chatFactory({ rawSink, runId }) const ctx: CampaignRunContext = { runId, @@ -465,7 +463,7 @@ export async function runEvalCampaign( emitter, store, rawSink, - llmOpts, + chat, } try { diff --git a/src/index.ts b/src/index.ts index 53861570..08dabd6a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -907,7 +907,10 @@ export type { VerdictCacheStore } from './verdict-cache' export { canonicalJson, contentHash, fileVerdictCache } from './verdict-cache' // ── utilities ───────────────────────────────────────────────────────── -// Provider-neutral LLM clients and shared error types. +// Provider-neutral model contracts and shared error types. The transport that +// executes a paid model is NOT part of this surface: a consumer binds its own +// through `createChatClient({ transport: 'custom' })`, or uses +// `profileChatClient` from `@tangle-network/agent-runtime/kernel`. export type { ChatCallOpts, @@ -923,28 +926,28 @@ export { AgentEvalError, ConfigError, JudgeError, NotFoundError, ValidationError export type { RunRecordBackend } from './eval-trace-store' export { jsonlRunRecordBackend } from './eval-trace-store' export { assignFeedbackSplit } from './feedback-trajectory' +export type { ModelEndpointCheck, ModelEndpointRequest } from './integrity/preflight' export { preflightModels } from './integrity/preflight' export type { KnowledgeBundle } from './knowledge/types' export type { LlmCallMetadata, LlmCallRequest, LlmCallResult, - LlmClientOptions, + LlmChargeBounds, LlmMessage, - LlmRouteRequirements, + LlmTokenLogprob, + LlmToolCall, + LlmToolChoice, + LlmToolDefinition, + LlmUsage, } from './llm-client' export { - assertLlmRoute, - callLlm, - callLlmJson, costReceiptFromLlm, costReceiptFromLlmError, isTransientLlmError, LlmCallError, - LlmClient, LlmResponseError, maximumChargeForLlmRequest, - probeLlm, stripFencedJson, } from './llm-client' export type { ModelSeats } from './model-seats' diff --git a/src/integrity/preflight.test.ts b/src/integrity/preflight.test.ts index f6af7160..d9ccdeee 100644 --- a/src/integrity/preflight.test.ts +++ b/src/integrity/preflight.test.ts @@ -1,10 +1,12 @@ import { describe, expect, it } from 'vitest' -import { assertModelsServed, ModelsUnreachableError, preflightModels } from './preflight' +import { + assertModelsServed, + type ModelEndpointRequest, + ModelsUnreachableError, + preflightModels, +} from './preflight' import { PROBE_MAX_TOKENS } from './served-model' -const BASE = 'https://router.tangle.tools/v1' -const KEY = 'test-key' - function listResponse(ids: string[]): Response { return new Response(JSON.stringify({ data: ids.map((id) => ({ id })) }), { status: 200, @@ -13,38 +15,32 @@ function listResponse(ids: string[]): Response { } /** - * Build a fetch fake whose chat-completions responses are keyed by model id. - * A 200 with no explicit body echoes the requested model, matching what an + * Build an endpoint fake whose probe responses are keyed by model id. A 200 + * with no explicit body echoes the requested model, matching what an * OpenAI-compatible provider sends; pass a body with a different `model` to * simulate a gateway substituting one. */ -function makeFetch( +function makeRequest( listedIds: string[], probeByModel: Record = {}, -): typeof fetch { - return (async (input: RequestInfo | URL, init?: RequestInit) => { - const url = String(input) - if (url.endsWith('/models')) return listResponse(listedIds) - if (url.endsWith('/chat/completions')) { - const model = JSON.parse(String(init?.body)).model as string - const spec = probeByModel[model] ?? { status: 200 } - const body = spec.body === undefined && spec.status === 200 ? { model } : (spec.body ?? {}) - return new Response(JSON.stringify(body), { - status: spec.status, - headers: { 'content-type': 'application/json' }, - }) - } - throw new Error(`unexpected url ${url}`) - }) as typeof fetch +): ModelEndpointRequest { + return async (check) => { + if (check.kind === 'list-models') return listResponse(listedIds) + const spec = probeByModel[check.model] ?? { status: 200 } + const body = + spec.body === undefined && spec.status === 200 ? { model: check.model } : (spec.body ?? {}) + return new Response(JSON.stringify(body), { + status: spec.status, + headers: { 'content-type': 'application/json' }, + }) + } } describe('preflightModels — membership only', () => { it('marks listed vs unlisted models, served null when not probed', async () => { const out = await preflightModels({ - baseUrl: BASE, - apiKey: KEY, models: ['claude-sonnet-4-6', 'opencode/zai-coding-plan/glm-5.1'], - fetchImpl: makeFetch(['claude-sonnet-4-6', 'deepseek-v4-pro']), + request: makeRequest(['claude-sonnet-4-6', 'deepseek-v4-pro']), }) expect(out.succeeded).toBe(true) expect(out.error).toBeNull() @@ -69,26 +65,14 @@ describe('preflightModels — membership only', () => { }, ]) }) - - it('tolerates a trailing slash on baseUrl', async () => { - const out = await preflightModels({ - baseUrl: `${BASE}/`, - apiKey: KEY, - models: ['claude-haiku-4-5'], - fetchImpl: makeFetch(['claude-haiku-4-5']), - }) - expect(out.value?.[0]?.listed).toBe(true) - }) }) describe('preflightModels — probe', () => { it('served true on 200', async () => { const out = await preflightModels({ - baseUrl: BASE, - apiKey: KEY, models: ['claude-sonnet-4-6'], probe: true, - fetchImpl: makeFetch(['claude-sonnet-4-6'], { 'claude-sonnet-4-6': { status: 200 } }), + request: makeRequest(['claude-sonnet-4-6'], { 'claude-sonnet-4-6': { status: 200 } }), }) expect(out.value).toEqual([ { @@ -112,11 +96,9 @@ describe('preflightModels — probe', () => { it('served false on 401 and captures the body error.message as detail', async () => { const out = await preflightModels({ - baseUrl: BASE, - apiKey: KEY, models: ['opencode/zai-coding-plan/glm-5.1'], probe: true, - fetchImpl: makeFetch([], { + request: makeRequest([], { 'opencode/zai-coding-plan/glm-5.1': { status: 401, body: { @@ -141,11 +123,9 @@ describe('preflightModels — probe', () => { it('served false on 503 with no usable body message', async () => { const out = await preflightModels({ - baseUrl: BASE, - apiKey: KEY, models: ['deepseek-v4-pro'], probe: true, - fetchImpl: makeFetch(['deepseek-v4-pro'], { 'deepseek-v4-pro': { status: 503, body: {} } }), + request: makeRequest(['deepseek-v4-pro'], { 'deepseek-v4-pro': { status: 503, body: {} } }), }) expect(out.value).toEqual([ { @@ -162,11 +142,9 @@ describe('preflightModels — probe', () => { it('reads error.message nested under error', async () => { const out = await preflightModels({ - baseUrl: BASE, - apiKey: KEY, models: ['gpt-4.1-mini'], probe: true, - fetchImpl: makeFetch(['gpt-4.1-mini'], { + request: makeRequest(['gpt-4.1-mini'], { 'gpt-4.1-mini': { status: 429, body: { error: { message: 'rate limited' } } }, }), }) @@ -176,14 +154,12 @@ describe('preflightModels — probe', () => { describe('preflightModels — network failure', () => { it('GET failure returns a typed outcome, never throws', async () => { - const fetchImpl = (async () => { + const request: ModelEndpointRequest = async () => { throw new Error('ECONNREFUSED') - }) as typeof fetch + } const out = await preflightModels({ - baseUrl: BASE, - apiKey: KEY, models: ['claude-sonnet-4-6'], - fetchImpl, + request, }) expect(out.succeeded).toBe(false) expect(out.value).toBeNull() @@ -191,29 +167,24 @@ describe('preflightModels — network failure', () => { }) it('non-2xx /models returns a typed outcome with the status', async () => { - const fetchImpl = (async () => new Response('forbidden', { status: 403 })) as typeof fetch + const request: ModelEndpointRequest = async () => new Response('forbidden', { status: 403 }) const out = await preflightModels({ - baseUrl: BASE, - apiKey: KEY, models: ['claude-sonnet-4-6'], - fetchImpl, + request, }) expect(out.succeeded).toBe(false) expect(out.error).toContain('403') }) it('probe POST failure returns a typed outcome', async () => { - const fetchImpl = (async (input: RequestInfo | URL) => { - const url = String(input) - if (url.endsWith('/models')) return listResponse(['claude-sonnet-4-6']) + const request: ModelEndpointRequest = async (check) => { + if (check.kind === 'list-models') return listResponse(['claude-sonnet-4-6']) throw new Error('socket hang up') - }) as typeof fetch + } const out = await preflightModels({ - baseUrl: BASE, - apiKey: KEY, models: ['claude-sonnet-4-6'], probe: true, - fetchImpl, + request, }) expect(out.succeeded).toBe(false) expect(out.error).toContain('socket hang up') @@ -224,7 +195,7 @@ describe('assertModelsServed', () => { it('passes silently when every model is served', async () => { const models = ['claude-sonnet-4-6', 'deepseek-v4-pro', 'gpt-4.1-mini'] await expect( - assertModelsServed({ baseUrl: BASE, apiKey: KEY, models, fetchImpl: makeFetch(models) }), + assertModelsServed({ models, request: makeRequest(models) }), ).resolves.toHaveLength(3) }) @@ -238,11 +209,9 @@ describe('assertModelsServed', () => { let thrown: unknown try { await assertModelsServed({ - baseUrl: BASE, - apiKey: KEY, models, probe: true, - fetchImpl: makeFetch(['claude-sonnet-4-6', 'claude-code/dead-c'], { + request: makeRequest(['claude-sonnet-4-6', 'claude-code/dead-c'], { 'claude-sonnet-4-6': { status: 200 }, 'opencode/dead-a': { status: 401, @@ -276,11 +245,9 @@ describe('assertModelsServed', () => { it('a listed-but-probe-failed model is dead (no partial silent pass)', async () => { await expect( assertModelsServed({ - baseUrl: BASE, - apiKey: KEY, models: ['deepseek-v4-pro'], probe: true, - fetchImpl: makeFetch(['deepseek-v4-pro'], { 'deepseek-v4-pro': { status: 503, body: {} } }), + request: makeRequest(['deepseek-v4-pro'], { 'deepseek-v4-pro': { status: 503, body: {} } }), }), ).rejects.toThrow(ModelsUnreachableError) }) @@ -292,11 +259,9 @@ describe('assertModelsServed', () => { let thrown: unknown try { await assertModelsServed({ - baseUrl: BASE, - apiKey: KEY, models: ['gpt-4.1-mini'], probe: true, - fetchImpl: makeFetch(['gpt-4.1-mini'], { + request: makeRequest(['gpt-4.1-mini'], { 'gpt-4.1-mini': { status: 200, body: { model: 'gemini-2.5-flash-lite' } }, }), }) @@ -316,11 +281,9 @@ describe('assertModelsServed', () => { it('fails a 200 that echoes no model id — reachable is not identified', async () => { await expect( assertModelsServed({ - baseUrl: BASE, - apiKey: KEY, models: ['deepseek-v4-pro'], probe: true, - fetchImpl: makeFetch(['deepseek-v4-pro'], { + request: makeRequest(['deepseek-v4-pro'], { 'deepseek-v4-pro': { status: 200, body: {} }, }), }), @@ -330,12 +293,10 @@ describe('assertModelsServed', () => { it('accepts an unreported id only when the caller opts in', async () => { await expect( assertModelsServed({ - baseUrl: BASE, - apiKey: KEY, models: ['deepseek-v4-pro'], probe: true, allowUnreported: true, - fetchImpl: makeFetch(['deepseek-v4-pro'], { + request: makeRequest(['deepseek-v4-pro'], { 'deepseek-v4-pro': { status: 200, body: {} }, }), }), @@ -343,26 +304,22 @@ describe('assertModelsServed', () => { }) it('accepts a same-family swap only when the caller opts in', async () => { - const fetchImpl = makeFetch(['deepseek/deepseek-v3.2'], { + const request = makeRequest(['deepseek/deepseek-v3.2'], { 'deepseek/deepseek-v3.2': { status: 200, body: { model: 'deepseek-v4-flash' } }, }) await expect( assertModelsServed({ - baseUrl: BASE, - apiKey: KEY, models: ['deepseek/deepseek-v3.2'], probe: true, - fetchImpl, + request, }), ).rejects.toThrow(ModelsUnreachableError) await expect( assertModelsServed({ - baseUrl: BASE, - apiKey: KEY, models: ['deepseek/deepseek-v3.2'], probe: true, allowWithinFamily: true, - fetchImpl, + request, }), ).resolves.toHaveLength(1) }) @@ -370,11 +327,9 @@ describe('assertModelsServed', () => { it('treats a provider-prefixed request answered by the bare id as the same model', async () => { await expect( assertModelsServed({ - baseUrl: BASE, - apiKey: KEY, models: ['zai/glm-5.2'], probe: true, - fetchImpl: makeFetch(['zai/glm-5.2'], { + request: makeRequest(['zai/glm-5.2'], { 'zai/glm-5.2': { status: 200, body: { model: 'glm-5.2' } }, }), }), @@ -382,46 +337,42 @@ describe('assertModelsServed', () => { }) it('rethrows a network failure rather than reporting a partial pass', async () => { - const fetchImpl = (async () => { + const request: ModelEndpointRequest = async () => { throw new Error('ECONNREFUSED') - }) as typeof fetch - await expect( - assertModelsServed({ baseUrl: BASE, apiKey: KEY, models: ['claude-sonnet-4-6'], fetchImpl }), - ).rejects.toThrow(/ECONNREFUSED/) + } + await expect(assertModelsServed({ models: ['claude-sonnet-4-6'], request })).rejects.toThrow( + /ECONNREFUSED/, + ) }) }) describe('preflightModels — probe budget', () => { - /** Capture the max_tokens each probe requested. */ - function recordingFetch( + /** Capture the output-token budget each probe requested. */ + function recordingRequest( sent: number[], spec: Record, - ) { - return (async (input: RequestInfo | URL, init?: RequestInit) => { - const url = String(input) - if (url.endsWith('/models')) return listResponse(Object.keys(spec)) - const request = JSON.parse(String(init?.body)) - sent.push(request.max_tokens) - const outcome = spec[request.model as string] ?? { status: 200 } + ): ModelEndpointRequest { + return async (check) => { + if (check.kind === 'list-models') return listResponse(Object.keys(spec)) + sent.push(check.maxOutputTokens) + const outcome = spec[check.model] ?? { status: 200 } const body = outcome.body === undefined && outcome.status === 200 - ? { model: request.model } + ? { model: check.model } : (outcome.body ?? {}) return new Response(JSON.stringify(body), { status: outcome.status, headers: { 'content-type': 'application/json' }, }) - }) as typeof fetch + } } it('spends the shared probe budget, not a budget a reasoning model cannot answer within', async () => { const sent: number[] = [] await preflightModels({ - baseUrl: BASE, - apiKey: KEY, models: ['deepseek-v4-pro'], probe: true, - fetchImpl: recordingFetch(sent, { 'deepseek-v4-pro': { status: 200 } }), + request: recordingRequest(sent, { 'deepseek-v4-pro': { status: 200 } }), }) expect(sent).toEqual([PROBE_MAX_TOKENS]) expect(PROBE_MAX_TOKENS).toBeGreaterThanOrEqual(64) @@ -430,12 +381,10 @@ describe('preflightModels — probe budget', () => { it('honours an explicit probeMaxTokens', async () => { const sent: number[] = [] await preflightModels({ - baseUrl: BASE, - apiKey: KEY, models: ['deepseek-v4-pro'], probe: true, probeMaxTokens: 512, - fetchImpl: recordingFetch(sent, { 'deepseek-v4-pro': { status: 200 } }), + request: recordingRequest(sent, { 'deepseek-v4-pro': { status: 200 } }), }) expect(sent).toEqual([512]) }) @@ -444,12 +393,10 @@ describe('preflightModels — probe budget', () => { 'refuses probeMaxTokens %s instead of probing with a nonsense budget', async (probeMaxTokens) => { const out = await preflightModels({ - baseUrl: BASE, - apiKey: KEY, models: ['deepseek-v4-pro'], probe: true, probeMaxTokens, - fetchImpl: makeFetch(['deepseek-v4-pro']), + request: makeRequest(['deepseek-v4-pro']), }) expect(out.succeeded).toBe(false) expect(out.error).toMatch(/probeMaxTokens must be a positive integer/) @@ -463,12 +410,10 @@ describe('preflightModels — probe budget', () => { it('reports a reasoning model that ran out of budget as alive, not dead', async () => { const out = await preflightModels({ - baseUrl: BASE, - apiKey: KEY, models: ['deepseek-v4-pro'], probe: true, probeMaxTokens: 5, - fetchImpl: makeFetch(['deepseek-v4-pro'], { 'deepseek-v4-pro': exhausted }), + request: makeRequest(['deepseek-v4-pro'], { 'deepseek-v4-pro': exhausted }), }) expect(out.value?.[0]).toMatchObject({ model: 'deepseek-v4-pro', @@ -487,11 +432,9 @@ describe('preflightModels — probe budget', () => { 'upstream error: reasoning_budget_exhausted', ])('recognises the budget signature in %j', async (message) => { const out = await preflightModels({ - baseUrl: BASE, - apiKey: KEY, models: ['deepseek-v4-pro'], probe: true, - fetchImpl: makeFetch(['deepseek-v4-pro'], { + request: makeRequest(['deepseek-v4-pro'], { 'deepseek-v4-pro': { status: 503, body: { error: { message } } }, }), }) @@ -501,11 +444,9 @@ describe('preflightModels — probe budget', () => { it('leaves an ordinary 503 scored as dead', async () => { const out = await preflightModels({ - baseUrl: BASE, - apiKey: KEY, models: ['kimi-k2.6'], probe: true, - fetchImpl: makeFetch(['kimi-k2.6'], { + request: makeRequest(['kimi-k2.6'], { 'kimi-k2.6': { status: 503, body: { error: { message: 'No provider configured' } } }, }), }) @@ -514,12 +455,10 @@ describe('preflightModels — probe budget', () => { it('still blocks the run, naming the budget rather than declaring the model dead', async () => { const failure = await assertModelsServed({ - baseUrl: BASE, - apiKey: KEY, models: ['deepseek-v4-pro'], probe: true, probeMaxTokens: 5, - fetchImpl: makeFetch(['deepseek-v4-pro'], { 'deepseek-v4-pro': exhausted }), + request: makeRequest(['deepseek-v4-pro'], { 'deepseek-v4-pro': exhausted }), }).catch((err: unknown) => err) expect(failure).toBeInstanceOf(ModelsUnreachableError) @@ -532,13 +471,11 @@ describe('preflightModels — probe budget', () => { it('accepts a budget-exhausted probe when the caller allows unproven identity', async () => { await expect( assertModelsServed({ - baseUrl: BASE, - apiKey: KEY, models: ['deepseek-v4-pro'], probe: true, probeMaxTokens: 5, allowUnreported: true, - fetchImpl: makeFetch(['deepseek-v4-pro'], { 'deepseek-v4-pro': exhausted }), + request: makeRequest(['deepseek-v4-pro'], { 'deepseek-v4-pro': exhausted }), }), ).resolves.toHaveLength(1) }) diff --git a/src/integrity/preflight.ts b/src/integrity/preflight.ts index f97ae7d8..fa140060 100644 --- a/src/integrity/preflight.ts +++ b/src/integrity/preflight.ts @@ -4,14 +4,18 @@ * complement to `assertRealBackend` (which inspects RunRecords AFTER the run to * catch a stub/unconfigured backend). * + * The caller owns the endpoint. Agent Eval holds no base URL and no + * credential: it asks a caller-supplied `request` function for each check and + * reads the `Response` that comes back, so the status and the provider's own + * error text stay readable here. + * * Two checks, increasing in cost: - * - membership (free): GET `{baseUrl}/models` once; a model is `listed` when + * - membership (free): one `list-models` request; a model is `listed` when * its id is in the served set. - * - probe (spends a tiny number of tokens): POST `{baseUrl}/chat/completions` - * per model with a 1-message, `PROBE_MAX_TOKENS`-token request; `served` is - * whether the router reached a provider, with the HTTP `status` and the - * body's `error.message` captured in `detail`, and `servedModel` recording - * WHICH model answered. + * - probe (spends a tiny number of tokens): one `probe` request per model + * with a `maxOutputTokens` budget; `served` is whether the endpoint reached + * a provider, with the HTTP `status` and the body's `error.message` + * captured in `detail`, and `servedModel` recording WHICH model answered. * * A 2xx is not proof the requested model answered — a gateway can accept one * id and route to another. The probe therefore compares the echoed id against @@ -68,11 +72,27 @@ export interface ModelPreflight { substitution: ServedModelCheck | null } +/** One check Agent Eval asks the caller's endpoint to perform. */ +export type ModelEndpointCheck = + | { readonly kind: 'list-models' } + | { + readonly kind: 'probe' + readonly model: string + /** Output-token budget the probe completion may bill. */ + readonly maxOutputTokens: number + } + +/** + * Caller-owned request into the model endpoint. The caller binds the base URL + * and the credential and returns the raw `Response`; a `list-models` request + * answers with an OpenAI-compatible `{ data: [{ id }] }` body, and a `probe` + * request answers with one minimal chat completion for `model`. + */ +export type ModelEndpointRequest = (check: ModelEndpointCheck) => Promise + export interface PreflightModelsOptions { - /** Router base URL, e.g. `https://router.tangle.tools/v1`. Trailing slash tolerated. */ - baseUrl: string - /** Bearer token sent as `Authorization: Bearer `. */ - apiKey: string + /** Caller-owned endpoint request. Agent Eval issues no provider HTTP itself. */ + request: ModelEndpointRequest /** Model ids to check. */ models: string[] /** When true, additionally spend a small chat probe per model. Default false. */ @@ -83,8 +103,6 @@ export interface PreflightModelsOptions { * `budgetExhausted` instead of proving their identity. */ probeMaxTokens?: number - /** Injectable fetch for tests; defaults to the global. */ - fetchImpl?: typeof fetch } export interface PreflightOutcome { @@ -106,10 +124,6 @@ interface ChatProbeBody { model?: unknown } -function stripSlash(url: string): string { - return url.replace(/\/+$/, '') -} - /** Extract `error.message` (then top-level `message`) from a chat-completions error body. */ function errorMessage(body: unknown): string | null { if (body == null || typeof body !== 'object') return null @@ -130,9 +144,6 @@ function errorMessage(body: unknown): string | null { * unconfigured (a 401 `model_not_found` from the router) is caught. */ export async function preflightModels(opts: PreflightModelsOptions): Promise { - const fetchImpl = opts.fetchImpl ?? fetch - const baseUrl = stripSlash(opts.baseUrl) - const authHeaders = { authorization: `Bearer ${opts.apiKey}` } const maxTokens = opts.probeMaxTokens ?? PROBE_MAX_TOKENS if (!Number.isInteger(maxTokens) || maxTokens <= 0) { return { @@ -144,13 +155,13 @@ export async function preflightModels(opts: PreflightModelsOptions): Promise
   try {
-    const res = await fetchImpl(`${baseUrl}/models`, { method: 'GET', headers: authHeaders })
+    const res = await opts.request({ kind: 'list-models' })
     if (!res.ok) {
       const text = await res.text().catch(() => '')
       return {
         succeeded: false,
         value: null,
-        error: `preflightModels: GET ${baseUrl}/models → ${res.status} ${text.slice(0, 400)}`,
+        error: `preflightModels: list-models → ${res.status} ${text.slice(0, 400)}`,
       }
     }
     const body = (await res.json()) as ModelsListBody
@@ -160,7 +171,7 @@ export async function preflightModels(opts: PreflightModelsOptions): Promise
) {
+/** Caller-owned transport: agent-eval issues no provider request itself. */
+function answering(answers: Array): ChatClient {
   let call = 0
-  return (async () => {
-    const spec = bodies[Math.min(call, bodies.length - 1)]!
-    call++
-    if ('status' in spec && 'body' in spec) {
-      return new Response((spec as { body: string }).body, {
-        status: (spec as { status: number }).status,
-      })
-    }
-    return new Response(
-      JSON.stringify({
+  return createChatClient({
+    transport: 'custom',
+    defaultModel: 'mock',
+    maximumAttempts: 1,
+    chat: async () => {
+      const spec = answers[Math.min(call, answers.length - 1)]!
+      call++
+      if (spec instanceof Error) throw spec
+      return {
+        content: JSON.stringify(spec),
+        usage: { promptTokens: 30, completionTokens: 20, totalTokens: 50, captured: true },
+        costUsd: null,
         model: 'mock',
-        choices: [{ message: { content: JSON.stringify(spec) } }],
-        usage: { total_tokens: 50 },
-      }),
-      { status: 200, headers: { 'content-type': 'application/json' } },
-    )
-  }) as unknown as typeof fetch
+        servedModel: 'mock',
+        durationMs: 1,
+        raw: {},
+      }
+    },
+  })
 }
 
 describe('runIntentMatchJudge', () => {
   it('returns available=false when no input artifact', async () => {
-    const r = await runIntentMatchJudge({ userRequest: 'build a thing', sourceFiles: [] })
+    const r = await runIntentMatchJudge(
+      { userRequest: 'build a thing', sourceFiles: [] },
+      { chat: answering([{}]) },
+    )
     expect(r.available).toBe(false)
     expect(r.error).toBe('no input artifact')
     expect(r.score).toBe(0)
   })
 
-  it('returns score and evidence on a happy LLM call', async () => {
+  it('returns score and evidence on a happy model call', async () => {
     const costLedger = new CostLedger()
-    const fetch = mockFetch([
-      {
-        score: 0.92,
-        evidence: 'src/App.tsx renders  with mint-1/mint-5 buttons',
-      },
-    ])
     const r = await runIntentMatchJudge(
       {
         userRequest: 'build an NFT mint page',
@@ -51,7 +52,15 @@ describe('runIntentMatchJudge', () => {
           },
         ],
       },
-      { llm: { fetch }, costLedger },
+      {
+        chat: answering([
+          {
+            score: 0.92,
+            evidence: 'src/App.tsx renders  with mint-1/mint-5 buttons',
+          },
+        ]),
+        costLedger,
+      },
     )
 
     expect(r.available).toBe(true)
@@ -62,21 +71,19 @@ describe('runIntentMatchJudge', () => {
     ])
   })
 
-  it('soft-fails (available=false) on LLM 500', async () => {
-    const fetch = mockFetch([{ status: 500, body: 'upstream error' }])
+  it('soft-fails (available=false) when the transport throws', async () => {
     const r = await runIntentMatchJudge(
       { userRequest: 'x', sourceFiles: [{ path: 'a.ts', content: 'x' }] },
-      { llm: { fetch, maximumAttempts: 1 } },
+      { chat: answering([new Error('500 upstream error')]) },
     )
     expect(r.available).toBe(false)
     expect(r.error).toMatch(/500|upstream/i)
   })
 
   it('clamps score to [0, 1]', async () => {
-    const fetch = mockFetch([{ score: 1.5, evidence: 'overshoot' }])
     const r = await runIntentMatchJudge(
       { userRequest: 'x', sourceFiles: [{ path: 'a.ts', content: 'x' }] },
-      { llm: { fetch } },
+      { chat: answering([{ score: 1.5, evidence: 'overshoot' }]) },
     )
     expect(r.score).toBe(1)
   })
diff --git a/src/intent-match-judge.ts b/src/intent-match-judge.ts
index af76abfd..464d67d9 100644
--- a/src/intent-match-judge.ts
+++ b/src/intent-match-judge.ts
@@ -23,15 +23,15 @@
  * treat failure as "judge skipped."
  */
 
-import { CostLedger, type CostLedgerHandle, type CostReceipt } from './cost-ledger'
+import type { ChatClient } from './analyst/chat-client'
+import { paidJsonChat } from './chat-json-call'
 import {
-  callLlmJson,
-  costReceiptFromLlm,
-  costReceiptFromLlmError,
-  type LlmCallRequest,
-  type LlmClientOptions,
-  maximumChargeForLlmRequest,
-} from './llm-client'
+  CostLedger,
+  type CostLedgerHandle,
+  type CostReceipt,
+  type CustomTokenPricing,
+} from './cost-ledger'
+import type { LlmCallRequest } from './llm-client'
 
 export const INTENT_MATCH_JUDGE_VERSION = 'intent-match-judge-v1-2026-04-24'
 
@@ -61,13 +61,16 @@ export interface IntentMatchResult {
 }
 
 export interface IntentMatchOptions {
+  /** Caller-owned transport. Required: agent-eval executes no paid model. */
+  chat: ChatClient
   model?: string
   timeoutMs?: number
   maxTokens?: number
   maxSourceChars?: number
   maxPerFileChars?: number
   maxHtmlChars?: number
-  llm?: LlmClientOptions
+  /** Endpoint rates used when the transport reports no billed amount. */
+  pricing?: CustomTokenPricing
   costLedger?: CostLedgerHandle
   costPhase?: string
   costTags?: Record
@@ -96,7 +99,10 @@ function truncate(body: string, cap: number, label: string): string {
   return `${body.slice(0, cap)}\n… [truncated ${body.length - cap} chars of ${label}]`
 }
 
-function buildPrompt(input: IntentMatchInput, opts: Required): string {
+function buildPrompt(
+  input: IntentMatchInput,
+  opts: { maxPerFileChars: number; maxSourceChars: number; maxHtmlChars: number },
+): string {
   const sourceBlob = input.sourceFiles
     .filter((f) => f.content.length <= opts.maxPerFileChars)
     .map((f) => `--- FILE: ${f.path} ---\n${f.content}`)
@@ -145,17 +151,18 @@ Return STRICT JSON. No prose outside.`
  */
 export async function runIntentMatchJudge(
   input: IntentMatchInput,
-  options: IntentMatchOptions = {},
+  options: IntentMatchOptions,
 ): Promise {
   const start = Date.now()
-  const opts: Required = {
-    model: options.model ?? DEFAULT_MODEL,
+  const opts = {
+    chat: options.chat,
+    model: options.model ?? options.chat.defaultModel ?? DEFAULT_MODEL,
     timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT,
     maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
     maxSourceChars: options.maxSourceChars ?? DEFAULT_MAX_SOURCE,
     maxPerFileChars: options.maxPerFileChars ?? DEFAULT_MAX_PER_FILE,
     maxHtmlChars: options.maxHtmlChars ?? DEFAULT_MAX_HTML,
-    llm: options.llm ?? {},
+    ...(options.pricing ? { pricing: options.pricing } : {}),
     costLedger: options.costLedger ?? new CostLedger(),
     costPhase: options.costPhase ?? 'judge.intent-match',
     costTags: options.costTags ?? {},
@@ -192,26 +199,20 @@ export async function runIntentMatchJudge(
       maxTokens: opts.maxTokens,
       timeoutMs: opts.timeoutMs,
     } satisfies LlmCallRequest
-    const paid = await opts.costLedger.runPaidCall({
+    const paid = await paidJsonChat<{ score: number; evidence: string }>({
+      chat: opts.chat,
+      request,
+      ledger: opts.costLedger,
       channel: 'judge',
       phase: opts.costPhase,
       actor: 'intent-match',
-      model: opts.model,
-      ...(Object.keys(opts.costTags).length > 0 ? { tags: opts.costTags } : {}),
-      maximumCharge: maximumChargeForLlmRequest(request, opts.llm),
+      tags: opts.costTags,
       signal: opts.signal,
-      execute: (signal, callId) =>
-        callLlmJson<{ score: number; evidence: string }>(request, {
-          ...opts.llm,
-          signal,
-          idempotencyKey: callId,
-        }),
-      receipt: ({ result }) => costReceiptFromLlm(result),
-      receiptFromError: costReceiptFromLlmError,
+      ...(opts.pricing ? { pricing: opts.pricing } : {}),
     })
     receipt = paid.receipt
     if (!paid.succeeded) throw paid.error
-    const { value } = paid.value
+    const { value } = paid
 
     const score = Math.max(0, Math.min(1, Number(value?.score ?? 0)))
     return {
@@ -238,10 +239,10 @@ export async function runIntentMatchJudge(
 }
 
 /**
- * Factory: pin LLM options once, return a closure.
+ * Factory: pin the transport and options once, return a closure.
  */
 export function createIntentMatchJudge(
-  options: IntentMatchOptions = {},
+  options: IntentMatchOptions,
 ): (input: IntentMatchInput) => Promise {
   return (input) => runIntentMatchJudge(input, options)
 }
diff --git a/src/llm-client.test.ts b/src/llm-client.test.ts
index f1a9662a..16d7c520 100644
--- a/src/llm-client.test.ts
+++ b/src/llm-client.test.ts
@@ -1224,37 +1224,6 @@ describe('llm-client — callLlmJson + schema degrade', () => {
   })
 })
 
-describe('llm-client — probeLlm', () => {
-  it('returns ok=true + latency when LLM responds', async () => {
-    const fetch = mockFetch([
-      async () => mkOkResponse({ choices: [{ message: { content: 'pong' } }], usage: {} }),
-    ])
-    const { probeLlm } = await import('./llm-client')
-    const r = await probeLlm('m', { fetch })
-    expect(r.ok).toBe(true)
-    expect(r.error).toBeNull()
-    expect(r.latencyMs).toBeGreaterThanOrEqual(0)
-  })
-
-  it('returns ok=false with error message on 4xx', async () => {
-    const fetch = mockFetch([async () => mkErrResponse(401, 'Invalid Authentication')])
-    const { probeLlm } = await import('./llm-client')
-    const r = await probeLlm('m', { fetch, maximumAttempts: 1 })
-    expect(r.ok).toBe(false)
-    expect(r.error).toMatch(/401|Invalid Authentication/)
-  })
-
-  it('returns ok=false on network error', async () => {
-    const failingFetch: typeof globalThis.fetch = (async () => {
-      throw new Error('fetch failed')
-    }) as unknown as typeof globalThis.fetch
-    const { probeLlm } = await import('./llm-client')
-    const r = await probeLlm('m', { fetch: failingFetch, maximumAttempts: 1 })
-    expect(r.ok).toBe(false)
-    expect(r.error).toMatch(/fetch failed/)
-  })
-})
-
 describe('llm-client — LlmClient wrapper', () => {
   it('inherits default opts and allows per-call overrides', async () => {
     const fetch = vi.fn(async () =>
diff --git a/src/llm-client.ts b/src/llm-client.ts
index 2568feef..9e0b43ec 100644
--- a/src/llm-client.ts
+++ b/src/llm-client.ts
@@ -26,12 +26,10 @@ import {
   costForTokenPricing,
   type MaximumCharge,
 } from './cost-ledger'
-import { AgentEvalError, CaptureIntegrityError } from './errors'
+import { AgentEvalError } from './errors'
 import {
   type AssertServedModelOptions,
   assertServedModel as assertServedModelIdentity,
-  checkServedModel,
-  PROBE_MAX_TOKENS,
 } from './integrity/served-model'
 import {
   defaultProviderRedactor,
@@ -116,12 +114,23 @@ export interface LlmCallRequest {
  * Returns undefined when output or multimodal input is not bounded, causing a
  * capped CostLedger to reject the call before execution. Pass
  * `customTokenPricing` when package pricing does not cover the model or endpoint. */
+export interface LlmChargeBounds {
+  /** Total provider attempts the transport may make for this call. Default 3. */
+  maximumAttempts?: number
+  /** The transport sends JSON mode instead of a response schema. */
+  jsonSchemaTransport?: 'native' | 'json-object'
+  /** Default provider reasoning mode the transport applies. */
+  thinking?: LlmThinkingMode
+  /** Token rates used when the provider omits cost or package pricing does not cover the model. */
+  customTokenPricing?: CustomTokenPricing
+}
+
 export function maximumChargeForLlmRequest(
   request: Pick<
     LlmCallRequest,
     'model' | 'messages' | 'jsonSchema' | 'tools' | 'toolChoice' | 'maxTokens' | 'thinking'
   >,
-  options: LlmClientOptions = {},
+  options: LlmChargeBounds = {},
 ): MaximumCharge | undefined {
   if (request.maxTokens === undefined) return undefined
   if (!Number.isInteger(request.maxTokens) || request.maxTokens <= 0) {
@@ -308,7 +317,7 @@ export class LlmResponseError extends AgentEvalError {
   }
 }
 
-export interface LlmClientOptions {
+export interface LlmClientOptions extends LlmChargeBounds {
   /** Base URL (without trailing slash). Must end at the `/v1` prefix. */
   baseUrl?: string
   /** Bearer token — either `apiKey` or `bearer` populates `Authorization: Bearer ...`. */
@@ -335,25 +344,12 @@ export interface LlmClientOptions {
    * total attempts × `timeoutMs`.
    */
   deadlineMs?: number
-  /** Total provider attempts. Default 3. */
-  maximumAttempts?: number
-  /** Token rates used when the provider omits cost or package pricing does not cover the model. */
-  customTokenPricing?: CustomTokenPricing
-  /**
-   * Transport for requests that declare `jsonSchema`. `native` sends
-   * `response_format: json_schema`; `json-object` sends the broadly supported
-   * JSON mode and relies on the caller to include the schema in model-visible
-   * instructions. Default: `native`.
-   */
-  jsonSchemaTransport?: 'native' | 'json-object'
   /**
    * JSON payload parsing policy. `extract` accepts fenced or prose-prefixed JSON.
    * `exact` requires the complete response content to be one JSON value.
    * Default: `extract`.
    */
   jsonPayloadMode?: 'extract' | 'exact'
-  /** Default provider reasoning mode. A per-call request value takes precedence. */
-  thinking?: LlmThinkingMode
   /** Fetch implementation — defaults to global `fetch`. Override for custom transport (e.g. tests). */
   fetch?: typeof fetch
   /**
@@ -1198,176 +1194,6 @@ function parseJsonSafely(
   }
 }
 
-// ─── Route assertion ────────────────────────────────────────────────────
-
-export type LlmRouteAssertionReason =
-  | 'no_explicit_base_url'
-  | 'base_url_blocked'
-  | 'base_url_not_allowed'
-  | 'no_auth'
-  | 'wrong_provider'
-
-export class LlmRouteAssertionError extends CaptureIntegrityError {
-  constructor(
-    message: string,
-    public readonly reason: LlmRouteAssertionReason,
-    public readonly baseUrl: string,
-  ) {
-    super(message)
-  }
-}
-
-export interface LlmRouteRequirements {
-  /**
-   * Throw if `opts.baseUrl` is undefined, i.e. the call would fall back to
-   * `DEFAULT_BASE_URL`. Set this for evaluation runs where silently using
-   * the public/free-tier router is a defect — the launch reviewer needs to
-   * know exactly which provider answered.
-   */
-  requireExplicitBaseUrl?: boolean
-  /**
-   * Allowlist of acceptable base URLs. Strings match by prefix
-   * (case-insensitive); RegExps test against the full base URL.
-   */
-  allowedBaseUrls?: Array
-  /** Blocklist that takes precedence over `allowedBaseUrls`. */
-  blockedBaseUrls?: Array
-  /** Throw if no auth header / api key is configured. */
-  requireAuth?: boolean
-  /**
-   * Logical provider id the configured `baseUrl` is expected to match (via
-   * `providerFromBaseUrl`). Mainly useful when paired with `requireExplicitBaseUrl`.
-   */
-  expectedProvider?: string
-}
-
-/**
- * Fail-loud assertion that the configured LLM client points at the route
- * the caller intends. Designed for the matrix-runner preflight: invoke
- * once before any LLM call to catch misconfiguration before a sweep burns
- * dollars on the wrong provider.
- *
- * Throws `LlmRouteAssertionError`. Pure — no I/O — so it's safe to call
- * from constructors and CI gates.
- */
-export function assertLlmRoute(opts: LlmClientOptions, req: LlmRouteRequirements = {}): void {
-  const baseUrlExplicit = opts.baseUrl !== undefined
-  const baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, '')
-
-  if (req.requireExplicitBaseUrl && !baseUrlExplicit) {
-    throw new LlmRouteAssertionError(
-      `assertLlmRoute: requireExplicitBaseUrl set but opts.baseUrl is undefined; would fall back to ${DEFAULT_BASE_URL}.`,
-      'no_explicit_base_url',
-      baseUrl,
-    )
-  }
-
-  if (req.blockedBaseUrls?.some((p) => matchUrl(baseUrl, p))) {
-    throw new LlmRouteAssertionError(
-      `assertLlmRoute: baseUrl ${baseUrl} matches a blocked pattern.`,
-      'base_url_blocked',
-      baseUrl,
-    )
-  }
-
-  if (req.allowedBaseUrls && req.allowedBaseUrls.length > 0) {
-    const ok = req.allowedBaseUrls.some((p) => matchUrl(baseUrl, p))
-    if (!ok) {
-      throw new LlmRouteAssertionError(
-        `assertLlmRoute: baseUrl ${baseUrl} is not in the allowed list (${req.allowedBaseUrls.map(describePattern).join(', ')}).`,
-        'base_url_not_allowed',
-        baseUrl,
-      )
-    }
-  }
-
-  if (req.requireAuth && !opts.apiKey && !opts.bearer && !opts.authHeader) {
-    throw new LlmRouteAssertionError(
-      `assertLlmRoute: requireAuth set but no apiKey, bearer, or authHeader was supplied.`,
-      'no_auth',
-      baseUrl,
-    )
-  }
-
-  if (req.expectedProvider) {
-    const actual = opts.provider ?? providerFromBaseUrl(baseUrl)
-    if (actual !== req.expectedProvider) {
-      throw new LlmRouteAssertionError(
-        `assertLlmRoute: expected provider ${req.expectedProvider} but baseUrl ${baseUrl} resolves to ${actual}.`,
-        'wrong_provider',
-        baseUrl,
-      )
-    }
-  }
-}
-
-function matchUrl(url: string, pattern: string | RegExp): boolean {
-  if (pattern instanceof RegExp) return pattern.test(url)
-  return url.toLowerCase().startsWith(pattern.toLowerCase())
-}
-
-function describePattern(p: string | RegExp): string {
-  return p instanceof RegExp ? p.source : p
-}
-
-/**
- * Probe whether a model is reachable. Returns latency + null error on
- * success; `ok=false` + error message on any failure (HTTP, timeout,
- * network, parse). Designed for sweep preflights — fail loud at the
- * boundary before burning a 30-leaf run on a misconfigured router.
- *
- * Sends a tiny `ping` message with `maxTokens = PROBE_MAX_TOKENS`. Reasoning
- * models (glm-5.1, deepseek-v4) can burn the entire budget on internal
- * reasoning for short prompts, so don't tighten this further — the shared
- * constant keeps this probe and `preflightModels` on one answer. We don't
- * validate content.
- *
- * Reachability and identity are separate answers: `ok` means the route
- * answered, `servedModel` / `substituted` say WHICH model answered. A gateway
- * that serves another provider's model returns `ok: true` with
- * `substituted: true` — inspect both before treating the id as measured.
- */
-export async function probeLlm(
-  model: string,
-  opts: LlmClientOptions & { timeoutMs?: number } = {},
-): Promise<{
-  ok: boolean
-  latencyMs: number
-  error: string | null
-  /** Id echoed by the provider; `null` when it sent none or the probe failed. */
-  servedModel: string | null
-  /** True when the echoed id is a different model than `model` (or absent). */
-  substituted: boolean
-}> {
-  const start = Date.now()
-  try {
-    const result = await callLlm(
-      {
-        model,
-        messages: [{ role: 'user', content: 'ping' }],
-        maxTokens: PROBE_MAX_TOKENS,
-        timeoutMs: opts.timeoutMs ?? 30_000,
-      },
-      opts,
-    )
-    return {
-      ok: true,
-      latencyMs: Date.now() - start,
-      error: null,
-      servedModel: result.servedModel ?? null,
-      substituted: checkServedModel(model, result.servedModel).substituted,
-    }
-  } catch (err) {
-    return {
-      ok: false,
-      latencyMs: Date.now() - start,
-      error: err instanceof Error ? err.message : String(err),
-      servedModel: null,
-      substituted: false,
-    }
-  }
-}
-
 /**
  * Stateful client — construct once with defaults, call many times.
  * Thin wrapper around the free functions; exists for callers that want
diff --git a/src/multishot/cost.ts b/src/multishot/cost.ts
new file mode 100644
index 00000000..59e86bc8
--- /dev/null
+++ b/src/multishot/cost.ts
@@ -0,0 +1,30 @@
+// Per-model cost estimator for multishot legs whose transport reported token
+// usage but no billed amount.
+
+/**
+ * Rough per-model cost estimate from token counts. Underestimates Anthropic,
+ * overestimates open-weight models — accurate enough for a cost ceiling, and
+ * never presented as a billed amount: a leg metered from this table is
+ * recorded with `estimated` provenance.
+ */
+export function estimateMultishotCost(
+  model: string,
+  usage?: { prompt_tokens?: number; completion_tokens?: number },
+): number {
+  if (!usage) return 0
+  const inputTok = usage.prompt_tokens ?? 0
+  const outputTok = usage.completion_tokens ?? 0
+  let inPer1k = 0.003
+  let outPer1k = 0.015
+  if (model.includes('gpt-4o-mini')) {
+    inPer1k = 0.00015
+    outPer1k = 0.0006
+  } else if (model.includes('gpt-5.4') || model.includes('claude-sonnet')) {
+    inPer1k = 0.003
+    outPer1k = 0.015
+  } else if (model.includes('kimi') || model.includes('glm') || model.includes('deepseek')) {
+    inPer1k = 0.0005
+    outPer1k = 0.002
+  }
+  return (inputTok * inPer1k + outputTok * outPer1k) / 1000
+}
diff --git a/src/multishot/default-tools.ts b/src/multishot/default-tools.ts
index e439c102..b457512f 100644
--- a/src/multishot/default-tools.ts
+++ b/src/multishot/default-tools.ts
@@ -5,7 +5,7 @@
 // researcher system prompt at your domain's citation style and the coder
 // at your preferred language.
 
-import { estimateRouterCost, routerCompletion } from './router'
+import { estimateMultishotCost } from './cost'
 import type { MultishotToolDefinition, MultishotToolExecutor } from './types'
 
 const DEFAULT_RESEARCHER_MODEL = 'openai/gpt-4o-mini'
@@ -77,9 +77,7 @@ function createResearchExecutor(config: DefaultResearcherConfig = {}): Multishot
   return async (args, ctx) => {
     const question = String(args.question ?? '')
     const scope = args.scope ? String(args.scope) : undefined
-    const { message, usage } = await routerCompletion({
-      apiKey: ctx.apiKey,
-      baseUrl: ctx.baseUrl,
+    const { message, usage, costUsd } = await ctx.transport({
       model,
       temperature: 0.3,
       maxTokens: 1800,
@@ -89,7 +87,10 @@ function createResearchExecutor(config: DefaultResearcherConfig = {}): Multishot
       ],
       signal: ctx.signal,
     })
-    return { content: message.content ?? '', costUsd: estimateRouterCost(model, usage) }
+    return {
+      content: message.content ?? '',
+      costUsd: costUsd ?? estimateMultishotCost(model, usage),
+    }
   }
 }
 
@@ -99,9 +100,7 @@ function createCodeExecutor(config: DefaultCoderConfig = {}): MultishotToolExecu
   return async (args, ctx) => {
     const goal = String(args.goal ?? '')
     const language = args.language ? String(args.language) : 'TypeScript'
-    const { message, usage } = await routerCompletion({
-      apiKey: ctx.apiKey,
-      baseUrl: ctx.baseUrl,
+    const { message, usage, costUsd } = await ctx.transport({
       model,
       temperature: 0.2,
       maxTokens: 2000,
@@ -111,7 +110,10 @@ function createCodeExecutor(config: DefaultCoderConfig = {}): MultishotToolExecu
       ],
       signal: ctx.signal,
     })
-    return { content: message.content ?? '', costUsd: estimateRouterCost(model, usage) }
+    return {
+      content: message.content ?? '',
+      costUsd: costUsd ?? estimateMultishotCost(model, usage),
+    }
   }
 }
 
@@ -140,5 +142,3 @@ export function defaultDelegationTools(config: DefaultToolsConfig = {}): Default
       name === 'delegate_research' ? 'research' : name === 'delegate_code' ? 'code' : undefined,
   }
 }
-
-export { defaultRouterBaseUrl } from './router'
diff --git a/src/multishot/golden/golden.test.ts b/src/multishot/golden/golden.test.ts
index dac5c940..5baf0bc5 100644
--- a/src/multishot/golden/golden.test.ts
+++ b/src/multishot/golden/golden.test.ts
@@ -302,21 +302,6 @@ describe('the golden records are load-bearing', () => {
     }
   })
 
-  it('refuses a second matrix judge wire in the same process', () => {
-    const scenario = first(multishotMatrixGoldenScenarios(), 'matrix scenarios')
-    const held = scenario.build('/unused/golden-wire-a')
-    const restore = held.installJudgeWire()
-    try {
-      expect(() => scenario.build('/unused/golden-wire-b').installJudgeWire()).toThrow(
-        /run matrix checks serially within one process/,
-      )
-    } finally {
-      restore()
-    }
-    // Released again after the first check finishes.
-    scenario.build('/unused/golden-wire-c').installJudgeWire()()
-  })
-
   it('refuses an `only` id the catalog does not hold instead of greening zero scenarios', async () => {
     await expect(
       checkMultishotGolden({
diff --git a/src/multishot/golden/harness.ts b/src/multishot/golden/harness.ts
index 3305c2dc..1d2fbe03 100644
--- a/src/multishot/golden/harness.ts
+++ b/src/multishot/golden/harness.ts
@@ -178,13 +178,7 @@ export async function checkMultishotMatrixGoldenScenario(opts: {
   const records = opts.records ?? goldenRecords()
   const record = requireMatrixRecord(records, opts.scenario.id)
   const runCase = opts.scenario.build(opts.runDir)
-  const restore = runCase.installJudgeWire()
-  let matrix: RunMultishotMatrixResult
-  try {
-    matrix = await opts.engine(runCase.options)
-  } finally {
-    restore()
-  }
+  const matrix: RunMultishotMatrixResult = await opts.engine(runCase.options)
 
   const mismatches = [
     ...compareJson(record.matrix, stripVolatile(matrix.matrix), 'matrix'),
diff --git a/src/multishot/golden/matrix-scenarios.ts b/src/multishot/golden/matrix-scenarios.ts
index b04b92f7..a042de3e 100644
--- a/src/multishot/golden/matrix-scenarios.ts
+++ b/src/multishot/golden/matrix-scenarios.ts
@@ -26,9 +26,6 @@ export interface MultishotMatrixGoldenCase {
   requests: MultishotRecordedRequest[]
   /** Judge calls, filled while the case runs. Sorted before comparison. */
   judgeRequests: RecordedJudgeRequest[]
-  /** Installs the deterministic judge wire on `globalThis.fetch` and returns
-   *  the function that restores the previous one. */
-  installJudgeWire: () => () => void
 }
 
 export interface MultishotMatrixGoldenScenario {
@@ -37,8 +34,6 @@ export interface MultishotMatrixGoldenScenario {
   readonly build: (runDir: string) => MultishotMatrixGoldenCase
 }
 
-const JUDGE_BASE_URL = 'http://router.invalid/v1'
-
 const personas: MultishotPersona[] = [
   { id: 'retail-founder', ask: 'a launch brief' },
   { id: 'saas-operator', ask: 'a pricing page' },
@@ -160,23 +155,22 @@ const dimensions = [
   { key: 'specificity', description: 'Was it specific? (0-10)' },
 ]
 
-function judge(name: string, buildPrompt: (input: TInput) => string): JudgeConfig {
+function judge(
+  name: string,
+  buildPrompt: (input: TInput) => string,
+  transport: MultishotTransport,
+): JudgeConfig {
   return {
     name,
+    transport,
     model: 'test/judge-model',
     dimensions,
     systemPrompt: `JUDGE:${name}`,
     buildPrompt,
-    apiKey: 'golden-key',
-    baseUrl: JUDGE_BASE_URL,
   }
 }
 
-/** True while some case holds `globalThis.fetch`. Module scope, because the
- *  resource being guarded is the process's own fetch. */
-let judgeWireInstalled = false
-
-/** Scores keyed by judge name, so the wire is a pure function of the request. */
+/** Scores keyed by judge name, so the judge leg is a pure function of the request. */
 const JUDGE_SCORES: Record = {
   conversation: { usefulness: 8, specificity: 7 },
   'code-review': { usefulness: 6, specificity: 9 },
@@ -198,6 +192,33 @@ function buildMatrixCase(runDir: string): MultishotMatrixGoldenCase {
   const requests: MultishotRecordedRequest[] = []
   const judgeRequests: RecordedJudgeRequest[] = []
 
+  // The judge leg is scripted exactly like the agent and driver legs: a pure
+  // function of the request, so the recorded ledger is a property of the
+  // engine under test alone.
+  const judgeTransport: MultishotTransport = async (req) => {
+    judgeRequests.push(
+      recordJudgeRequest({
+        model: req.model,
+        temperature: req.temperature,
+        max_tokens: req.maxTokens,
+        messages: req.messages,
+      }),
+    )
+    const messages = req.messages as Array<{ role?: string; content?: string }>
+    const system = messages.find((m) => m.role === 'system')?.content ?? ''
+    const name = system.replace('JUDGE:', '')
+    const score = JUDGE_SCORES[name]
+    if (!score) {
+      throw new Error(`multishot golden judge transport: unknown judge system prompt ${system}`)
+    }
+    return {
+      message: { content: JSON.stringify({ ...score, notes: `${name} ok` }) },
+      usage: { prompt_tokens: 300, completion_tokens: 25 },
+      model: 'test/judge-model',
+      costUsd: 0.0007,
+    }
+  }
+
   const options: RunMultishotMatrixOptions = {
     profiles,
     personas,
@@ -207,15 +228,18 @@ function buildMatrixCase(runDir: string): MultishotMatrixGoldenCase {
         'conversation',
         (input: { transcript: unknown[] }) =>
           `Score this conversation of ${input.transcript.length} messages.`,
+        judgeTransport,
       ),
       codeReview: judge(
         'code-review',
         (input: { artifact: { content: string } }) => `Score this code: ${input.artifact.content}`,
+        judgeTransport,
       ),
       contentQuality: judge(
         'content-quality',
         (input: { artifact: { content: string } }) =>
           `Score this content: ${input.artifact.content}`,
+        judgeTransport,
       ),
     },
     tools,
@@ -227,8 +251,6 @@ function buildMatrixCase(runDir: string): MultishotMatrixGoldenCase {
     maxConcurrency: 1,
     agentModel: 'test/agent-model',
     driverModel: 'test/driver-model',
-    apiKey: 'golden-key',
-    baseUrl: JUDGE_BASE_URL,
     agentTransport: async (req) => {
       requests.push(recordRequest('agent', req))
       return agentTransport(req)
@@ -239,51 +261,5 @@ function buildMatrixCase(runDir: string): MultishotMatrixGoldenCase {
     },
   }
 
-  const installJudgeWire = (): (() => void) => {
-    // The wire is process-wide, so two matrix checks running at once in one
-    // process would cross their judge ledgers. Refuse the second one instead of
-    // recording a mixture: a golden check that silently reads another run's
-    // calls reports a mismatch nobody can explain.
-    if (judgeWireInstalled) {
-      throw new Error(
-        'multishot golden judge wire: another matrix check already holds globalThis.fetch — run matrix checks serially within one process',
-      )
-    }
-    judgeWireInstalled = true
-    const previous = globalThis.fetch
-    globalThis.fetch = (async (url: unknown, init?: { body?: string }) => {
-      // The judge leg is the ONLY call allowed to reach the wire; the agent
-      // and driver legs run on the scripted transports above. Anything else is
-      // a wiring defect in the engine under test, so fail loud.
-      if (String(url) !== `${JUDGE_BASE_URL}/chat/completions`) {
-        throw new Error(`multishot golden judge wire: unexpected request to ${String(url)}`)
-      }
-      const body = JSON.parse(init?.body ?? '{}') as Record
-      judgeRequests.push(recordJudgeRequest(body))
-      const messages = (body.messages ?? []) as Array<{ role: string; content: string }>
-      const system = messages.find((m) => m.role === 'system')?.content ?? ''
-      const name = system.replace('JUDGE:', '')
-      const score = JUDGE_SCORES[name]
-      if (!score) {
-        throw new Error(`multishot golden judge wire: unknown judge system prompt ${system}`)
-      }
-      return {
-        ok: true,
-        status: 200,
-        json: async () => ({
-          choices: [{ message: { content: JSON.stringify({ ...score, notes: `${name} ok` }) } }],
-          usage: { prompt_tokens: 300, completion_tokens: 25 },
-          model: 'test/judge-model',
-          _response_cost: 0.0007,
-        }),
-        text: async () => '',
-      }
-    }) as unknown as typeof globalThis.fetch
-    return () => {
-      globalThis.fetch = previous
-      judgeWireInstalled = false
-    }
-  }
-
-  return { options, requests, judgeRequests, installJudgeWire }
+  return { options, requests, judgeRequests }
 }
diff --git a/src/multishot/golden/scenarios.ts b/src/multishot/golden/scenarios.ts
index 00cae9ff..29681d29 100644
--- a/src/multishot/golden/scenarios.ts
+++ b/src/multishot/golden/scenarios.ts
@@ -190,8 +190,6 @@ function delegationCase(overrides: DelegationOverrides = {}): MultishotGoldenCas
     maxTurns: overrides.maxTurns ?? 3,
     agentModel: 'test/agent-model',
     driverModel: 'test/driver-model',
-    apiKey: 'golden-key',
-    baseUrl: 'http://router.invalid',
     agentTransport: ledgerTransport(requests, 'agent', overrides.agent ?? delegationAgent),
     driverTransport: ledgerTransport(requests, 'driver', overrides.driver ?? delegationDriver),
   }
@@ -384,8 +382,6 @@ function samplingCase(overrides: SamplingOverrides = {}): MultishotGoldenCase {
     agentModel: 'scripted/agent',
     driverModel: 'primary/driver',
     driverFallbackModels: ['fallback/driver'],
-    apiKey: 'golden-key',
-    baseUrl: 'http://router.invalid',
     agentTransport: ledgerTransport(
       requests,
       'agent',
diff --git a/src/multishot/index.ts b/src/multishot/index.ts
index 182be6de..542ee930 100644
--- a/src/multishot/index.ts
+++ b/src/multishot/index.ts
@@ -1,5 +1,7 @@
 // Multishot substrate — re-exports for `@tangle-network/agent-eval/multishot`.
 
+export { estimateMultishotCost } from './cost'
+
 export {
   DEFAULT_CODER_MODEL,
   type DefaultCoderConfig,
@@ -32,15 +34,6 @@ export {
   runMultishotMatrix,
 } from './matrix'
 export { type MultishotShot, type RunMultishotOptions, runMultishot } from './multishot'
-export {
-  defaultRouterBaseUrl,
-  estimateRouterCost,
-  type RouterCompletionRequest,
-  type RouterCompletionResponse,
-  type RouterToolCall,
-  requireRouterApiKey,
-  routerCompletion,
-} from './router'
 export {
   defaultMultishotDriverSystemPrompt,
   defaultMultishotOpener,
diff --git a/src/multishot/judges.ts b/src/multishot/judges.ts
index f7883e4a..4bf1d599 100644
--- a/src/multishot/judges.ts
+++ b/src/multishot/judges.ts
@@ -12,12 +12,7 @@ import type { JudgeScore } from '../campaign/types'
 import type { CostProvenance } from '../cost-ledger'
 import type { LlmCallMetadata, LlmUsage } from '../llm-client'
 import { estimateCost, isModelPriced } from '../metrics'
-import {
-  defaultRouterBaseUrl,
-  type RouterCompletionResponse,
-  requireRouterApiKey,
-  routerCompletion,
-} from './router'
+import type { MultishotTransport, MultishotTransportResponse } from './types'
 
 // Canonical declaration lives in campaign/types.ts. Multishot emits the same
 // shape on its producer-defined 0-10 scale.
@@ -35,6 +30,9 @@ export interface JudgeDimension {
 export interface JudgeConfig {
   /** Display name (for trace + log). */
   name: string
+  /** Caller-owned execution for this judge's call. agent-eval issues no
+   *  provider request and holds no credential. */
+  transport: MultishotTransport
   /** Model used for this judge. */
   model?: string
   /** 0-10 scored dimensions. */
@@ -44,9 +42,6 @@ export interface JudgeConfig {
   /** Build the user prompt from the typed input. Must include "Respond with
    *  ONLY this JSON: { ... }" listing each dimension key. */
   buildPrompt: (input: TInput) => string
-  /** Optional model + api overrides. */
-  apiKey?: string
-  baseUrl?: string
   /** Maximum output tokens for the judge response. Defaults to 1500. */
   maxTokens?: number
 }
@@ -62,18 +57,14 @@ export async function runJudge(
   judge: JudgeConfig,
   input: TInput,
 ): Promise {
-  const apiKey = judge.apiKey ?? requireRouterApiKey()
-  const baseUrl = judge.baseUrl ?? defaultRouterBaseUrl()
-  const model = judge.model ?? process.env.JUDGE_MODEL ?? DEFAULT_JUDGE_MODEL
+  const model = judge.model ?? DEFAULT_JUDGE_MODEL
   const prompt = judge.buildPrompt(input)
   let raw = ''
   let llmCall: LlmCallMetadata
   let cost: CostProvenance
   const startedAt = Date.now()
   try {
-    const response = await routerCompletion({
-      apiKey,
-      baseUrl,
+    const response = await judge.transport({
       model,
       temperature: 0,
       maxTokens: judge.maxTokens ?? 1500,
@@ -82,7 +73,7 @@ export async function runJudge(
         { role: 'user', content: prompt },
       ],
     })
-    const call = judgeCallMetadata(response)
+    const call = judgeCallMetadata(response, model, Date.now() - startedAt)
     llmCall = call.llmCall
     cost = call.cost
     raw = (response.message.content ?? '').trim()
@@ -140,31 +131,39 @@ export async function runJudge(
   }
 }
 
-function judgeCallMetadata(response: RouterCompletionResponse): {
+function judgeCallMetadata(
+  response: MultishotTransportResponse,
+  requestedModel: string,
+  durationMs: number,
+): {
   llmCall: LlmCallMetadata
   cost: CostProvenance
 } {
   const usage = canonicalUsage(response.usage)
+  // The transport reports the served identity when it observed one. Falling
+  // back to the requested id keeps attribution honest for a transport that
+  // reported none; it is not proof that model answered.
+  const model = response.model ?? requestedModel
   return {
     llmCall: {
       usage,
       costUsd: response.costUsd ?? null,
-      model: response.model,
-      durationMs: response.durationMs,
+      model,
+      durationMs,
     },
     cost:
       response.costUsd !== undefined
         ? { kind: 'observed', usd: response.costUsd }
-        : usage.captured === false || !isModelPriced(response.model)
+        : usage.captured === false || !isModelPriced(model)
           ? { kind: 'uncaptured', usd: null }
           : {
               kind: 'estimated',
-              usd: estimateCost(usage.promptTokens, usage.completionTokens, response.model),
+              usd: estimateCost(usage.promptTokens, usage.completionTokens, model),
             },
   }
 }
 
-function canonicalUsage(usage: RouterCompletionResponse['usage']): LlmUsage {
+function canonicalUsage(usage: MultishotTransportResponse['usage']): LlmUsage {
   const promptTokens = tokenCount(usage?.prompt_tokens)
   const completionTokens = tokenCount(usage?.completion_tokens)
   const captured = promptTokens !== undefined && completionTokens !== undefined
diff --git a/src/multishot/matrix.ts b/src/multishot/matrix.ts
index 33ec8566..3c39daf2 100644
--- a/src/multishot/matrix.ts
+++ b/src/multishot/matrix.ts
@@ -107,12 +107,15 @@ export interface RunMultishotMatrixOptions {
   driverMaxTokens?: number
   /** Maximum output tokens for each judge response. */
   judgeMaxTokens?: number
-  /** Execution seam for the agent leg of every cell — replaces the router
-   *  HTTP call when provided (see RunMultishotOptions.agentTransport).
-   *  Judges are unaffected; configure those via MultishotJudges. */
-  agentTransport?: MultishotTransport
-  /** Execution seam for the simulated-user driver leg of every cell. */
-  driverTransport?: MultishotTransport
+  /** Caller-owned execution for the agent leg of every cell (see
+   *  RunMultishotOptions.agentTransport). Judges are unaffected; each judge
+   *  carries its own transport in MultishotJudges. */
+  agentTransport: MultishotTransport
+  /** Caller-owned execution for the simulated-user driver leg of every cell. */
+  driverTransport: MultishotTransport
+  /** Caller-owned execution for the specialist leg the tool executors run in
+   *  every cell. Defaults to `agentTransport`. */
+  toolTransport?: MultishotTransport
   /** Conversation engine for every cell. Defaults to `runMultishot`.
    *
    *  The matrix owns everything around the shot — cell fan-out, concurrency,
@@ -128,9 +131,6 @@ export interface RunMultishotMatrixOptions {
    *  `MultishotShotResultError` for that cell. The default engine is never
    *  used as a fallback. */
   runShot?: MultishotShot
-  /** Pass-thru fields. */
-  apiKey?: string
-  baseUrl?: string
 }
 
 /** Per-cell output the multishot matrix records in `MatrixResult.cells`.
@@ -247,8 +247,7 @@ export async function runMultishotMatrix(
         driverMaxTokens: opts.driverMaxTokens,
         agentTransport: opts.agentTransport,
         driverTransport: opts.driverTransport,
-        apiKey: opts.apiKey,
-        baseUrl: opts.baseUrl,
+        toolTransport: opts.toolTransport,
       })
       // Everything from here on runs with the shot's spend already committed.
       // A throw past this line must carry it, or the money leaves the matrix's
diff --git a/src/multishot/multishot.ts b/src/multishot/multishot.ts
index 73785c73..44cac222 100644
--- a/src/multishot/multishot.ts
+++ b/src/multishot/multishot.ts
@@ -1,20 +1,18 @@
 // Multi-turn driver-agent simulation with inline tool execution.
 //
 // The driver = LLM acting as the persona (reactive, non-deterministic).
-// The agent = the product agent under test (router call by default, or an
-// injected transport — with profile's systemPrompt + the configured tools).
+// The agent = the product agent under test, executed by the caller-supplied
+// transport with the profile's systemPrompt + the configured tools.
 // Tool calls execute inline via the configured executors and feed back
 // into the agent's message log so the agent integrates the result.
+//
+// agent-eval owns no model transport: every leg runs on a MultishotTransport
+// the caller supplies, so no provider credential enters this package.
 
 import type { AgentProfile } from '@tangle-network/agent-interface'
 import { withCellSpend } from '../matrix'
+import { estimateMultishotCost } from './cost'
 import { defaultDelegationTools } from './default-tools'
-import {
-  defaultRouterBaseUrl,
-  estimateRouterCost,
-  requireRouterApiKey,
-  routerCompletion,
-} from './router'
 import { defaultShapeFromProfile } from './shape-defaults'
 import {
   type MultishotArtifact,
@@ -56,18 +54,16 @@ export interface RunMultishotOptions {
   driverMaxTokens?: number
   /** Maximum tool calls the agent may dispatch inside one assistant turn. */
   maxToolDispatches?: number
-  /** Execution seam for the agent leg. When provided, every agent inference
-   *  step goes through this function instead of the router HTTP call; the
-   *  string levers (agentModel, apiKey, baseUrl) stop applying to that leg.
-   *  apiKey/baseUrl are still resolved for tool executors and any leg
-   *  without an injected transport. */
-  agentTransport?: MultishotTransport
-  /** Execution seam for the simulated-user driver leg (symmetric to
-   *  agentTransport). Driver model fallback rotation still applies — the
-   *  transport receives each candidate model in turn. */
-  driverTransport?: MultishotTransport
-  apiKey?: string
-  baseUrl?: string
+  /** Caller-owned execution for the agent leg. Every agent inference step
+   *  runs through this function; `agentModel` names the model it receives. */
+  agentTransport: MultishotTransport
+  /** Caller-owned execution for the simulated-user driver leg. Driver model
+   *  fallback rotation still applies — the transport receives each candidate
+   *  model in turn. */
+  driverTransport: MultishotTransport
+  /** Caller-owned execution for the specialist leg the tool executors run.
+   *  Defaults to `agentTransport`. */
+  toolTransport?: MultishotTransport
   signal?: AbortSignal
 }
 
@@ -130,8 +126,6 @@ async function runShotTurns(
   opts: RunMultishotOptions,
   meter: ShotMeter,
 ): Promise {
-  const apiKey = opts.apiKey ?? requireRouterApiKey()
-  const baseUrl = opts.baseUrl ?? defaultRouterBaseUrl()
   const maxTurns = opts.maxTurns ?? 10
   const maxToolDispatches = opts.maxToolDispatches ?? 4
   const agentModel = opts.agentModel ?? 'openai/gpt-5.4'
@@ -153,9 +147,9 @@ async function runShotTurns(
   const executors = opts.toolExecutors ?? bundle.executors
   const artifactTypeFor = opts.artifactTypeFor ?? bundle.artifactTypeFor
 
-  const routerTransport: MultishotTransport = (req) => routerCompletion({ apiKey, baseUrl, ...req })
-  const agentTransport = opts.agentTransport ?? routerTransport
-  const driverTransport = opts.driverTransport ?? routerTransport
+  const agentTransport = opts.agentTransport
+  const driverTransport = opts.driverTransport
+  const toolTransport = opts.toolTransport ?? agentTransport
 
   const shape = defaultShapeFromProfile(opts.profile, opts.shape)
 
@@ -191,7 +185,7 @@ async function runShotTurns(
         maxTokens: dispatchesThisTurn === 0 ? agentMaxTokens : toolFollowupMaxTokens,
         signal: opts.signal,
       })
-      meter.costUsd += agentCostUsd ?? estimateRouterCost(agentModel, agentUsage)
+      meter.costUsd += agentCostUsd ?? estimateMultishotCost(agentModel, agentUsage)
       if (agentCostUsd === undefined && agentUsage === undefined) meter.uncaptured = true
 
       const agentText = (agentMsg.content ?? '').trim()
@@ -234,7 +228,7 @@ async function runShotTurns(
           if (!executor) {
             toolResult = JSON.stringify({ error: `unknown tool ${tc.name}` })
           } else {
-            const r = await executor(tc.args, { apiKey, baseUrl, signal: opts.signal })
+            const r = await executor(tc.args, { transport: toolTransport, signal: opts.signal })
             toolResult = r.content
             meter.costUsd += r.costUsd
             const artifactType = artifactTypeFor(tc.name)
@@ -326,7 +320,7 @@ async function driverTurn(opts: {
         maxTokens: opts.maxTokens,
         signal: opts.signal,
       })
-      opts.meter.costUsd += costUsd ?? estimateRouterCost(model, usage)
+      opts.meter.costUsd += costUsd ?? estimateMultishotCost(model, usage)
       if (costUsd === undefined && usage === undefined) opts.meter.uncaptured = true
       const content = (message.content ?? '').trim()
       if (content.length > 0) return { content }
diff --git a/src/multishot/router.ts b/src/multishot/router.ts
deleted file mode 100644
index e8f95d2e..00000000
--- a/src/multishot/router.ts
+++ /dev/null
@@ -1,112 +0,0 @@
-// Router fetch helper — single source of truth for OpenAI-compat calls
-// against the Tangle router. Used by the driver, agent, judges, and the
-// default tool executors.
-
-import type { MultishotToolDefinition } from './types'
-
-export interface RouterCompletionRequest {
-  apiKey: string
-  baseUrl: string
-  model: string
-  messages: Array>
-  tools?: MultishotToolDefinition[]
-  temperature?: number
-  maxTokens?: number
-  signal?: AbortSignal
-}
-
-export interface RouterToolCall {
-  id: string
-  type: 'function'
-  function: { name: string; arguments: string }
-}
-
-export interface RouterCompletionResponse {
-  message: { content?: string | null; tool_calls?: RouterToolCall[] }
-  usage?: { prompt_tokens?: number; completion_tokens?: number }
-  /** Provider-reported spend when the endpoint supplies it. */
-  costUsd?: number
-  /** Model echoed by the provider, falling back to the requested model. */
-  model: string
-  durationMs: number
-}
-
-export async function routerCompletion(
-  req: RouterCompletionRequest,
-): Promise {
-  const startedAt = Date.now()
-  const body: Record = {
-    model: req.model,
-    messages: req.messages,
-    temperature: req.temperature ?? 0.7,
-    max_tokens: req.maxTokens ?? 2000,
-  }
-  if (req.tools?.length) body.tools = req.tools
-  const url = `${req.baseUrl.replace(/\/+$/, '')}/chat/completions`
-  const res = await fetch(url, {
-    method: 'POST',
-    headers: { Authorization: `Bearer ${req.apiKey}`, 'Content-Type': 'application/json' },
-    body: JSON.stringify(body),
-    signal: req.signal,
-  })
-  if (!res.ok) {
-    const text = await res.text()
-    throw new Error(`router ${res.status}: ${text.slice(0, 300)}`)
-  }
-  const json = (await res.json()) as {
-    choices: Array<{ message: { content?: string | null; tool_calls?: RouterToolCall[] } }>
-    usage?: { prompt_tokens?: number; completion_tokens?: number }
-    model?: unknown
-    _response_cost?: unknown
-    cost_usd?: unknown
-  }
-  const choice = json.choices[0]
-  if (!choice) throw new Error(`router returned no choices: ${JSON.stringify(json).slice(0, 200)}`)
-  const rawCost = json._response_cost ?? json.cost_usd
-  const costUsd =
-    typeof rawCost === 'number' && Number.isFinite(rawCost) && rawCost >= 0 ? rawCost : undefined
-  return {
-    message: choice.message,
-    usage: json.usage,
-    ...(costUsd === undefined ? {} : { costUsd }),
-    model: typeof json.model === 'string' && json.model ? json.model : req.model,
-    durationMs: Date.now() - startedAt,
-  }
-}
-
-// Rough per-model cost estimator. Used for cost-ceiling enforcement.
-// Underestimates Anthropic, overestimates oss models — fine for ceilings.
-export function estimateRouterCost(
-  model: string,
-  usage?: { prompt_tokens?: number; completion_tokens?: number },
-): number {
-  if (!usage) return 0
-  const inputTok = usage.prompt_tokens ?? 0
-  const outputTok = usage.completion_tokens ?? 0
-  let inPer1k = 0.003
-  let outPer1k = 0.015
-  if (model.includes('gpt-4o-mini')) {
-    inPer1k = 0.00015
-    outPer1k = 0.0006
-  } else if (model.includes('gpt-5.4') || model.includes('claude-sonnet')) {
-    inPer1k = 0.003
-    outPer1k = 0.015
-  } else if (model.includes('kimi') || model.includes('glm') || model.includes('deepseek')) {
-    inPer1k = 0.0005
-    outPer1k = 0.002
-  }
-  return (inputTok * inPer1k + outputTok * outPer1k) / 1000
-}
-
-export function defaultRouterBaseUrl(): string {
-  return (process.env.TANGLE_ROUTER_BASE_URL ?? 'https://router.tangle.tools/v1').replace(
-    /\/+$/,
-    '',
-  )
-}
-
-export function requireRouterApiKey(): string {
-  const key = process.env.TANGLE_API_KEY
-  if (!key) throw new Error('multishot requires TANGLE_API_KEY (router-scoped sk-tan-* key)')
-  return key
-}
diff --git a/src/multishot/types.ts b/src/multishot/types.ts
index 32232688..0d231753 100644
--- a/src/multishot/types.ts
+++ b/src/multishot/types.ts
@@ -65,23 +65,31 @@ export interface MultishotTransportResponse {
   message: { content?: string | null; tool_calls?: MultishotTransportToolCall[] }
   usage?: { prompt_tokens?: number; completion_tokens?: number }
   /** Actual spend for this call. When omitted, the loop meters cost from
-   *  `usage` via the per-model router estimator (estimateRouterCost). */
+   *  `usage` via the per-model estimator (estimateMultishotCost). */
   costUsd?: number
+  /** Model identity the provider reported, when the transport observed one.
+   *  Omitted means unreported, and the requested model is used for
+   *  attribution — which is not proof that model answered. */
+  model?: string
 }
 
-/** Execution seam for one leg of the multishot loop. When provided, it
- *  replaces the internal router HTTP call for that leg — the loop still owns
- *  turn scheduling, tool dispatch, transcript capture, and cost metering.
- *  agent-eval has no dependency on agent-runtime; adapt agent-runtime's
- *  resolveAgentBackend (or any sandbox/cli-bridge/router client) into this
- *  signature product-side. */
+/** Execution seam for one leg of the multishot loop. The caller owns model
+ *  execution: agent-eval issues no provider request and holds no credential.
+ *  The loop still owns turn scheduling, tool dispatch, transcript capture, and
+ *  cost metering. agent-eval has no dependency on agent-runtime; adapt
+ *  agent-runtime's `profileChatClient` (or any sandbox, bridge, or router
+ *  client) into this signature product-side. */
 export type MultishotTransport = (
   req: MultishotTransportRequest,
 ) => Promise
 
 export type MultishotToolExecutor = (
   args: Record,
-  ctx: { apiKey: string; baseUrl: string; signal?: AbortSignal },
+  ctx: {
+    /** Caller-owned execution seam for the specialist leg this tool runs. */
+    transport: MultishotTransport
+    signal?: AbortSignal
+  },
 ) => Promise<{ content: string; costUsd: number }>
 
 export interface MultishotPersona {
diff --git a/src/reference-equivalence-judge.test.ts b/src/reference-equivalence-judge.test.ts
index 249f30ac..7e28287f 100644
--- a/src/reference-equivalence-judge.test.ts
+++ b/src/reference-equivalence-judge.test.ts
@@ -1,5 +1,10 @@
 import { afterEach, describe, expect, it, vi } from 'vitest'
-import { type ChatRequest, type ChatResponse, createChatClient } from './analyst/chat-client'
+import {
+  type ChatClient,
+  type ChatRequest,
+  type ChatResponse,
+  createChatClient,
+} from './analyst/chat-client'
 import { runCampaign } from './campaign/run-campaign'
 import { inMemoryCampaignStorage } from './campaign/storage'
 import { CostLedger } from './cost-ledger'
@@ -64,12 +69,13 @@ function mockChat(
   })
 }
 
-function directProvider() {
+/** A caller-owned transport: agent-eval issues no provider request itself. */
+function callerTransport(chat: ChatClient['chat']): ChatClient {
   return createChatClient({
-    transport: 'direct-provider',
-    baseUrl: 'https://provider.example/v1',
-    apiKey: 'test-key',
+    transport: 'custom',
     defaultModel: 'judge-model-2026-07-01',
+    maximumAttempts: 1,
+    chat,
   })
 }
 
@@ -162,52 +168,27 @@ describe('createReferenceEquivalenceJudge', () => {
 })
 
 describe('reference-equivalence transport and campaign integration', () => {
-  it('degrades provider 400 json_schema to json_object', async () => {
-    const bodies: Array> = []
-    const fetch = vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {
-      bodies.push(JSON.parse(String(init?.body)) as Record)
-      if (bodies.length === 1) return new Response('json_schema not supported', { status: 400 })
-      return new Response(
-        JSON.stringify({
-          model: 'judge-model-2026-07-01',
-          choices: [{ message: { content: JSON.stringify(verdict()) }, finish_reason: 'stop' }],
-          usage: { prompt_tokens: 120, completion_tokens: 24, total_tokens: 144 },
-          _response_cost: 0.0042,
-        }),
-        { status: 200 },
-      )
-    }) as unknown as typeof globalThis.fetch
-    vi.stubGlobal('fetch', fetch)
-
-    expect((await runReferenceEquivalenceJudge(INPUT, { chat: directProvider() })).score).toBe(0.9)
-    expect(bodies.map((body) => (body.response_format as { type: string }).type)).toEqual([
-      'json_schema',
-      'json_object',
-    ])
-  })
-
   it('propagates cancellation and records an incomplete receipt after transport termination', async () => {
-    let providerSignal: AbortSignal | null | undefined
+    let transportSignal: AbortSignal | undefined
     let markStarted!: () => void
     const started = new Promise((resolve) => {
       markStarted = resolve
     })
-    const fetch = vi.fn((_input: RequestInfo | URL, init?: RequestInit) => {
-      providerSignal = init?.signal
+    const chat = vi.fn((_req: ChatRequest, opts?: { signal?: AbortSignal }) => {
+      transportSignal = opts?.signal
       markStarted()
-      return new Promise((_resolve, reject) => {
-        providerSignal?.addEventListener(
+      return new Promise((_resolve, reject) => {
+        transportSignal?.addEventListener(
           'abort',
-          () => reject(new DOMException('provider request aborted', 'AbortError')),
+          () => reject(new DOMException('transport request aborted', 'AbortError')),
           { once: true },
         )
       })
-    }) as unknown as typeof globalThis.fetch
-    vi.stubGlobal('fetch', fetch)
+    })
     const controller = new AbortController()
     const ledger = new CostLedger()
     const pending = createReferenceEquivalenceJudge({
-      chat: directProvider(),
+      chat: callerTransport(chat),
       costLedger: ledger,
     }).score({
       artifact: INPUT.candidateOutput,
@@ -219,8 +200,8 @@ describe('reference-equivalence transport and campaign integration', () => {
     controller.abort(new DOMException('campaign cancelled', 'AbortError'))
 
     await expect(pending).rejects.toMatchObject({ name: 'AbortError' })
-    expect(fetch).toHaveBeenCalledOnce()
-    expect(providerSignal?.aborted).toBe(true)
+    expect(chat).toHaveBeenCalledOnce()
+    expect(transportSignal?.aborted).toBe(true)
     expect(ledger.list()).toEqual([
       expect.objectContaining({ costUnknown: true, usageUnknown: true, error: expect.any(String) }),
     ])
diff --git a/src/semantic-concept-judge.test.ts b/src/semantic-concept-judge.test.ts
index 0d7c0dc3..747971c3 100644
--- a/src/semantic-concept-judge.test.ts
+++ b/src/semantic-concept-judge.test.ts
@@ -1,26 +1,31 @@
 import { describe, expect, it } from 'vitest'
+
+import { type ChatClient, createChatClient } from './analyst/chat-client'
 import { CostLedger } from './cost-ledger'
 import { createSemanticConceptJudge, runSemanticConceptJudge } from './semantic-concept-judge'
 
-function mockFetch(bodies: Array) {
+/** Caller-owned transport: agent-eval issues no provider request itself. */
+function answering(answers: Array): ChatClient {
   let call = 0
-  return (async () => {
-    const spec = bodies[Math.min(call, bodies.length - 1)]!
-    call++
-    if ('status' in spec && 'body' in spec) {
-      return new Response((spec as { body: string }).body, {
-        status: (spec as { status: number }).status,
-      })
-    }
-    return new Response(
-      JSON.stringify({
+  return createChatClient({
+    transport: 'custom',
+    defaultModel: 'mock',
+    maximumAttempts: 1,
+    chat: async () => {
+      const spec = answers[Math.min(call, answers.length - 1)]!
+      call++
+      if (spec instanceof Error) throw spec
+      return {
+        content: JSON.stringify(spec),
+        usage: { promptTokens: 60, completionTokens: 40, totalTokens: 100, captured: true },
+        costUsd: null,
         model: 'mock',
-        choices: [{ message: { content: JSON.stringify(spec) } }],
-        usage: { total_tokens: 100 },
-      }),
-      { status: 200, headers: { 'content-type': 'application/json' } },
-    )
-  }) as unknown as typeof fetch
+        servedModel: 'mock',
+        durationMs: 1,
+        raw: {},
+      }
+    },
+  })
 }
 
 const BASE_INPUT = {
@@ -37,7 +42,7 @@ const BASE_INPUT = {
 describe('semantic-concept-judge', () => {
   it('parses a happy-path response + computes score from per-concept averages', async () => {
     const costLedger = new CostLedger()
-    const fetch = mockFetch([
+    const chat = answering([
       {
         summary: 'mint button wired, supply counter absent',
         concepts: [
@@ -58,7 +63,7 @@ describe('semantic-concept-judge', () => {
         ],
       },
     ])
-    const r = await runSemanticConceptJudge(BASE_INPUT, { llm: { fetch }, costLedger })
+    const r = await runSemanticConceptJudge(BASE_INPUT, { chat, costLedger })
     expect(r.available).toBe(true)
     expect(r.totalCount).toBe(2)
     expect(r.presentCount).toBe(1)
@@ -73,7 +78,7 @@ describe('semantic-concept-judge', () => {
   })
 
   it('clamps out-of-range scores to 0..10', async () => {
-    const fetch = mockFetch([
+    const chat = answering([
       {
         summary: 'out-of-range model response',
         concepts: [
@@ -88,13 +93,13 @@ describe('semantic-concept-judge', () => {
         ],
       },
     ])
-    const r = await runSemanticConceptJudge(BASE_INPUT, { llm: { fetch } })
+    const r = await runSemanticConceptJudge(BASE_INPUT, { chat })
     expect(r.findings[0]!.score).toBe(10)
     expect(r.findings[1]!.score).toBe(0)
   })
 
   it('coerces invalid severity to "info"', async () => {
-    const fetch = mockFetch([
+    const chat = answering([
       {
         summary: 's',
         concepts: [{ concept: 'x', present: true, score: 5, evidence: 'e', severity: 'nonsense' }],
@@ -102,22 +107,22 @@ describe('semantic-concept-judge', () => {
     ])
     const r = await runSemanticConceptJudge(
       { ...BASE_INPUT, expectedConcepts: [{ name: 'x' }] },
-      { llm: { fetch } },
+      { chat },
     )
     expect(r.findings[0]!.severity).toBe('info')
   })
 
   it('soft-fails available=false on malformed response (no concepts array)', async () => {
-    const fetch = mockFetch([{ summary: 'oops', concepts: 'not an array' }])
-    const r = await runSemanticConceptJudge(BASE_INPUT, { llm: { fetch } })
+    const chat = answering([{ summary: 'oops', concepts: 'not an array' }])
+    const r = await runSemanticConceptJudge(BASE_INPUT, { chat })
     expect(r.available).toBe(false)
     expect(r.error).toMatch(/malformed/)
     expect(r.score).toBe(0)
   })
 
-  it('soft-fails available=false on LLM 500', async () => {
-    const fetch = mockFetch([{ status: 500, body: 'upstream oops' }])
-    const r = await runSemanticConceptJudge(BASE_INPUT, { llm: { fetch, maximumAttempts: 1 } })
+  it('soft-fails available=false when the transport throws', async () => {
+    const chat = answering([new Error('500 upstream oops')])
+    const r = await runSemanticConceptJudge(BASE_INPUT, { chat })
     expect(r.available).toBe(false)
     expect(r.error).toMatch(/500/)
   })
@@ -125,7 +130,7 @@ describe('semantic-concept-judge', () => {
   it('returns available=false on empty expectedConcepts (no-op)', async () => {
     const r = await runSemanticConceptJudge(
       { ...BASE_INPUT, expectedConcepts: [] },
-      { llm: { fetch: mockFetch([]) } },
+      { chat: answering([{}]) },
     )
     expect(r.available).toBe(false)
     expect(r.totalCount).toBe(0)
@@ -133,7 +138,7 @@ describe('semantic-concept-judge', () => {
   })
 
   it('weightConcepts: complexity weights integrate concepts higher than render', async () => {
-    const fetch = mockFetch([
+    const chat = answering([
       {
         summary: 's',
         concepts: [
@@ -158,14 +163,14 @@ describe('semantic-concept-judge', () => {
           { name: 'wallet connect', complexity: 'integrate' },
         ],
       },
-      { llm: { fetch }, weightConcepts: 'complexity' },
+      { chat, weightConcepts: 'complexity' },
     )
     // weighted: (1.0*10 + 2.0*0) / (1.0 + 2.0) = 10/3 = 3.33 → /10 = 0.333
     expect(r.score).toBeCloseTo(0.333, 2)
   })
 
   it('weightConcepts: mean (default) gives equal weight (preserves 0.10 behavior)', async () => {
-    const fetch = mockFetch([
+    const chat = answering([
       {
         summary: 's',
         concepts: [
@@ -188,14 +193,14 @@ describe('semantic-concept-judge', () => {
           { name: 'wallet connect', complexity: 'integrate' },
         ],
       },
-      { llm: { fetch } },
+      { chat },
     )
     // mean: (10+0)/2 = 5 → /10 = 0.5
     expect(r.score).toBeCloseTo(0.5, 2)
   })
 
   it('weightConcepts: explicit weight overrides complexity-derived weight', async () => {
-    const fetch = mockFetch([
+    const chat = answering([
       {
         summary: 's',
         concepts: [
@@ -212,14 +217,14 @@ describe('semantic-concept-judge', () => {
           { name: 'b', complexity: 'integrate', weight: 1 },
         ],
       },
-      { llm: { fetch }, weightConcepts: 'complexity' },
+      { chat, weightConcepts: 'complexity' },
     )
     // (5*10 + 1*0) / (5 + 1) = 50/6 = 8.33 → /10 = 0.833
     expect(r.score).toBeCloseTo(0.833, 2)
   })
 
   it('createSemanticConceptJudge factory — closure over options', async () => {
-    const fetch = mockFetch([
+    const chat = answering([
       {
         summary: 's',
         concepts: [
@@ -239,7 +244,7 @@ describe('semantic-concept-judge', () => {
         ],
       },
     ])
-    const judge = createSemanticConceptJudge({ llm: { fetch }, model: 'x' })
+    const judge = createSemanticConceptJudge({ chat, model: 'x' })
     const a = await judge({ ...BASE_INPUT, expectedConcepts: [{ name: 'mint button' }] })
     const b = await judge({ ...BASE_INPUT, expectedConcepts: [{ name: 'supply counter' }] })
     expect(a.findings[0]!.concept).toBe('mint button')
diff --git a/src/semantic-concept-judge.ts b/src/semantic-concept-judge.ts
index ba305442..79c82f92 100644
--- a/src/semantic-concept-judge.ts
+++ b/src/semantic-concept-judge.ts
@@ -19,15 +19,15 @@
  * rather than "layer failed" in a multi-layer pipeline.
  */
 
-import { CostLedger, type CostLedgerHandle, type CostReceipt } from './cost-ledger'
+import type { ChatClient } from './analyst/chat-client'
+import { paidJsonChat } from './chat-json-call'
 import {
-  callLlmJson,
-  costReceiptFromLlm,
-  costReceiptFromLlmError,
-  type LlmCallRequest,
-  type LlmClientOptions,
-  maximumChargeForLlmRequest,
-} from './llm-client'
+  CostLedger,
+  type CostLedgerHandle,
+  type CostReceipt,
+  type CustomTokenPricing,
+} from './cost-ledger'
+import type { LlmCallRequest } from './llm-client'
 import type { Severity } from './multi-layer-verifier'
 
 // ─── Types ──────────────────────────────────────────────────────────────
@@ -131,8 +131,10 @@ export interface SemanticConceptJudgeOptions {
   maxPerFileChars?: number
   /** HTML cap. Default 30000. */
   maxHtmlChars?: number
-  /** LlmClient config (baseUrl, apiKey, authHeader, …). */
-  llm?: LlmClientOptions
+  /** Caller-owned transport. Required: agent-eval executes no paid model. */
+  chat: ChatClient
+  /** Endpoint rates used when the transport reports no billed amount. */
+  pricing?: CustomTokenPricing
   costLedger?: CostLedgerHandle
   costPhase?: string
   costTags?: Record
@@ -190,7 +192,7 @@ function truncate(body: string, cap: number, label: string): string {
 
 function buildPrompt(
   input: SemanticConceptJudgeInput,
-  opts: Required,
+  opts: { maxPerFileChars: number; maxSourceChars: number; maxHtmlChars: number },
 ): string {
   const sourceBlob = input.sourceFiles
     .filter((f) => f.content.length <= opts.maxPerFileChars)
@@ -249,7 +251,7 @@ Return STRICT JSON. No prose outside the JSON.`
  */
 export async function runSemanticConceptJudge(
   input: SemanticConceptJudgeInput,
-  options: SemanticConceptJudgeOptions = {},
+  options: SemanticConceptJudgeOptions,
 ): Promise {
   const start = Date.now()
   const totalCount = input.expectedConcepts.length
@@ -270,14 +272,15 @@ export async function runSemanticConceptJudge(
     }
   }
 
-  const opts: Required = {
-    model: options.model ?? DEFAULT_MODEL,
+  const opts = {
+    chat: options.chat,
+    model: options.model ?? options.chat.defaultModel ?? DEFAULT_MODEL,
     timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT,
     maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
     maxSourceChars: options.maxSourceChars ?? DEFAULT_MAX_SOURCE,
     maxPerFileChars: options.maxPerFileChars ?? DEFAULT_MAX_PER_FILE,
     maxHtmlChars: options.maxHtmlChars ?? DEFAULT_MAX_HTML,
-    llm: options.llm ?? {},
+    ...(options.pricing ? { pricing: options.pricing } : {}),
     costLedger: options.costLedger ?? new CostLedger(),
     costPhase: options.costPhase ?? 'judge.semantic-concept',
     costTags: options.costTags ?? {},
@@ -319,26 +322,20 @@ export async function runSemanticConceptJudge(
       maxTokens: opts.maxTokens,
       timeoutMs: opts.timeoutMs,
     } satisfies LlmCallRequest
-    const paid = await opts.costLedger.runPaidCall({
+    const paid = await paidJsonChat<{ summary: string; concepts: ConceptFinding[] }>({
+      chat: opts.chat,
+      request,
+      ledger: opts.costLedger,
       channel: 'judge',
       phase: opts.costPhase,
       actor: 'semantic-concept',
-      model: opts.model,
-      ...(Object.keys(opts.costTags).length > 0 ? { tags: opts.costTags } : {}),
-      maximumCharge: maximumChargeForLlmRequest(request, opts.llm),
+      tags: opts.costTags,
       signal: opts.signal,
-      execute: (signal, callId) =>
-        callLlmJson<{ summary: string; concepts: ConceptFinding[] }>(request, {
-          ...opts.llm,
-          signal,
-          idempotencyKey: callId,
-        }),
-      receipt: ({ result }) => costReceiptFromLlm(result),
-      receiptFromError: costReceiptFromLlmError,
+      ...(opts.pricing ? { pricing: opts.pricing } : {}),
     })
     receipt = paid.receipt
     if (!paid.succeeded) throw paid.error
-    const { value } = paid.value
+    const { value } = paid
 
     if (!value?.concepts || !Array.isArray(value.concepts)) {
       throw new Error('judge returned malformed response — expected array under "concepts"')
@@ -401,7 +398,7 @@ export async function runSemanticConceptJudge(
  * Convenient for pipelines that want to share a single LlmClient config.
  */
 export function createSemanticConceptJudge(
-  options: SemanticConceptJudgeOptions = {},
+  options: SemanticConceptJudgeOptions,
 ): (input: SemanticConceptJudgeInput) => Promise {
   return (input) => runSemanticConceptJudge(input, options)
 }
diff --git a/src/wire/handlers.ts b/src/wire/handlers.ts
index c70015f6..e83be7c5 100644
--- a/src/wire/handlers.ts
+++ b/src/wire/handlers.ts
@@ -10,19 +10,11 @@
  *   - Lets unexpected errors bubble — the transport maps them to 500.
  */
 
-import { CostLedger, type CostLedgerHandle } from '../cost-ledger'
+import type { ChatClient } from '../analyst/chat-client'
+import { paidJsonChat } from '../chat-json-call'
+import { CostLedger, type CostLedgerHandle, type CustomTokenPricing } from '../cost-ledger'
 import type { FeedbackTrajectoryStore } from '../feedback-trajectory'
-import {
-  assertLlmRoute,
-  callLlmJson,
-  costReceiptFromLlm,
-  costReceiptFromLlmError,
-  type LlmCallRequest,
-  type LlmClientOptions,
-  LlmRouteAssertionError,
-  type LlmRouteRequirements,
-  maximumChargeForLlmRequest,
-} from '../llm-client'
+import type { LlmCallRequest } from '../llm-client'
 import { packageVersion } from '../package-version'
 import type { TraceEvent as InternalTraceEvent } from '../trace/schema'
 import type { TraceStore } from '../trace/store'
@@ -194,9 +186,15 @@ const DEFAULT_JUDGE_MODEL = 'claude-sonnet-4-6'
 export interface HandleJudgeOptions {
   costLedger?: CostLedgerHandle
   costPhase?: string
-  llm?: LlmClientOptions
+  /**
+   * Caller-owned transport for the judge call. agent-eval holds no provider
+   * credential: the process that serves this endpoint binds its own client.
+   * Omitting it makes `/v1/judge` refuse with `llm_not_configured`.
+   */
+  chat?: ChatClient
   defaultModel?: string
-  routeRequirements?: LlmRouteRequirements
+  /** Endpoint rates used when the transport reports no billed amount. */
+  pricing?: CustomTokenPricing
   signal?: AbortSignal
 }
 
@@ -219,25 +217,17 @@ export async function handleJudge(
     throw new WireError('validation_error', 'Provide either `rubricName` or `rubric`.', 422)
   }
 
-  if (options.routeRequirements) {
-    try {
-      assertLlmRoute(options.llm ?? {}, options.routeRequirements)
-    } catch (error) {
-      if (!(error instanceof LlmRouteAssertionError)) throw error
-      const noEndpoint = error.reason === 'no_explicit_base_url'
-      throw new WireError(
-        noEndpoint ? 'llm_not_configured' : 'llm_route_rejected',
-        noEndpoint
-          ? 'No model endpoint is configured. Pass llm.baseUrl or configure the CLI provider environment variables.'
-          : error.message,
-        503,
-        { reason: error.reason },
-      )
-    }
+  if (!options.chat) {
+    throw new WireError(
+      'llm_not_configured',
+      'No model transport is configured. Pass a ChatClient, or configure the CLI provider environment variables.',
+      503,
+    )
   }
 
   const startedAt = Date.now()
-  const model = req.model ?? options.defaultModel ?? DEFAULT_JUDGE_MODEL
+  const model =
+    req.model ?? options.defaultModel ?? options.chat.defaultModel ?? DEFAULT_JUDGE_MODEL
 
   const request = {
     model,
@@ -250,25 +240,18 @@ export async function handleJudge(
     maxTokens: 4_000,
     timeoutMs: 60_000,
   } satisfies LlmCallRequest
-  const ledger = options.costLedger ?? new CostLedger()
-  const paid = await ledger.runPaidCall({
+  const paid = await paidJsonChat({
+    chat: options.chat,
+    request,
+    ledger: options.costLedger ?? new CostLedger(),
     channel: 'judge',
     phase: options.costPhase ?? 'wire.judge',
     actor: `wire.${req.rubricName ?? 'inline'}`,
-    model,
-    maximumCharge: maximumChargeForLlmRequest(request, options.llm),
-    signal: options.signal,
-    execute: (signal, callId) =>
-      callLlmJson(request, {
-        ...options.llm,
-        signal,
-        idempotencyKey: callId,
-      }),
-    receipt: ({ result }) => costReceiptFromLlm(result),
-    receiptFromError: costReceiptFromLlmError,
+    ...(options.signal ? { signal: options.signal } : {}),
+    ...(options.pricing ? { pricing: options.pricing } : {}),
   })
   if (!paid.succeeded) throw paid.error
-  const { value, result } = paid.value
+  const { value, response } = paid
 
   const output = validateJudgeOutput(value, rubric)
 
@@ -282,7 +265,7 @@ export async function handleJudge(
     wins: output.wins ?? [],
     rationale: output.rationale,
     rubricVersion: hashRubric(rubric),
-    model: result.model,
+    model: response.model,
     durationMs,
   }
 }
diff --git a/src/wire/rpc.ts b/src/wire/rpc.ts
index 747f405c..c793d39b 100644
--- a/src/wire/rpc.ts
+++ b/src/wire/rpc.ts
@@ -11,7 +11,7 @@
  * One request per process invocation. To pipeline many calls, the client
  * writes JSONL to stdin and reads JSONL from stdout — see batch mode below.
  */
-import type { LlmClientOptions, LlmRouteRequirements } from '../llm-client'
+import type { ChatClient } from '../analyst/chat-client'
 import { handleJudge, handleListRubrics, handleVersion, WireError } from './handlers'
 import { JudgeRequestSchema } from './schemas'
 
@@ -29,9 +29,9 @@ interface RpcError {
 }
 
 export interface RpcOptions {
-  llm?: LlmClientOptions
+  /** Caller-owned transport for `/v1/judge`. Without it the method refuses. */
+  chat?: ChatClient
   judgeModel?: string
-  llmRouteRequirements?: LlmRouteRequirements
 }
 
 export async function dispatchRpc(
@@ -53,9 +53,8 @@ export async function dispatchRpc(
         }
         return {
           result: await handleJudge(parsed.data, {
-            llm: options.llm,
+            chat: options.chat,
             defaultModel: options.judgeModel,
-            routeRequirements: options.llmRouteRequirements,
           }),
         }
       }
diff --git a/src/wire/server.ts b/src/wire/server.ts
index f705a32a..bb955711 100644
--- a/src/wire/server.ts
+++ b/src/wire/server.ts
@@ -16,7 +16,7 @@
 import { type ServerType, serve } from '@hono/node-server'
 import { Hono } from 'hono'
 import { cors } from 'hono/cors'
-import type { LlmClientOptions, LlmRouteRequirements } from '../llm-client'
+import type { ChatClient } from '../analyst/chat-client'
 import {
   handleFeedbackIngest,
   handleJudge,
@@ -34,12 +34,14 @@ const STARTED_AT = Date.now()
 export interface CreateAppOptions {
   /** Stores wired to the ingestion endpoints. */
   stores?: IngestionStores
-  /** Model provider used by `/v1/judge`. */
-  llm?: LlmClientOptions
+  /**
+   * Caller-owned transport used by `/v1/judge`. agent-eval holds no provider
+   * credential; without a client the endpoint refuses with
+   * `llm_not_configured`.
+   */
+  chat?: ChatClient
   /** Default judge model when a request does not provide one. */
   judgeModel?: string
-  /** Model route checks applied before every judge call. */
-  llmRouteRequirements?: LlmRouteRequirements
   /**
    * Bearer-token auth. When provided, every endpoint EXCEPT `/healthz`
    * and `/v1/version` requires `Authorization: Bearer `. The
@@ -121,9 +123,8 @@ export function createApp(opts: CreateAppOptions = {}) {
       )
     }
     const result = await handleJudge(parsed.data, {
-      llm: opts.llm,
+      chat: opts.chat,
       defaultModel: opts.judgeModel,
-      routeRequirements: opts.llmRouteRequirements,
     })
     return c.json(result)
   })
diff --git a/tests/consumer-contract.test.ts b/tests/consumer-contract.test.ts
index fde0f52d..57b64120 100644
--- a/tests/consumer-contract.test.ts
+++ b/tests/consumer-contract.test.ts
@@ -2,10 +2,7 @@ import { describe, expect, it } from 'vitest'
 import type {
   ChatCallOpts,
   ChatTransport,
-  CliBridgeTransportOpts,
   CustomTransportOpts,
-  DirectProviderTransportOpts,
-  RouterTransportOpts,
   SandboxSdkTransportOpts,
 } from '../src/analyst/chat-client'
 import * as builderEval from '../src/builder-eval/index'
@@ -60,8 +57,7 @@ const ROOT_RUNTIME_SYMBOLS = [
   // agent-runtime supervise surface imports these from the root
   'isToolSpan',
   'OUTPUT_VALUE',
-  // LLM client + retry
-  'callLlmJson',
+  // Caller-owned model transport + retry
   'withJudgeRetry',
   'createChatClient',
   // Verifier / review / campaign
@@ -232,13 +228,6 @@ describe('public-surface contract for consumers', () => {
     const createOpts: CreateChatClientOpts = mockOpts
     const client: ChatClient = agentEval.createChatClient(createOpts)
 
-    const routerOpts: RouterTransportOpts = { transport: 'router', apiKey: 'test' }
-    const cliBridgeOpts: CliBridgeTransportOpts = { transport: 'cli-bridge' }
-    const directProviderOpts: DirectProviderTransportOpts = {
-      transport: 'direct-provider',
-      baseUrl: 'https://example.invalid/v1',
-      apiKey: 'test',
-    }
     const sandboxSdkOpts: SandboxSdkTransportOpts = {
       transport: 'sandbox-sdk',
       chat: async () => response,
@@ -251,9 +240,6 @@ describe('public-surface contract for consumers', () => {
     }
     const custom = agentEval.createChatClient(customOpts)
 
-    expect(routerOpts.transport).toBe('router')
-    expect(cliBridgeOpts.transport).toBe('cli-bridge')
-    expect(directProviderOpts.transport).toBe('direct-provider')
     expect(sandboxSdkOpts.transport).toBe('sandbox-sdk')
     expect(await client.chat(request, callOpts)).toBe(response)
     expect(await custom.chat(request, callOpts)).toBe(response)
diff --git a/tests/eval-campaign.test.ts b/tests/eval-campaign.test.ts
index c8062bcf..cfe9c9ad 100644
--- a/tests/eval-campaign.test.ts
+++ b/tests/eval-campaign.test.ts
@@ -1,8 +1,8 @@
 import { describe, expect, it } from 'vitest'
 import { buildAgentProfileCell } from '../src/agent-profile-cell'
+import { createChatClient } from '../src/analyst/chat-client'
 import type { CampaignRunner, EvalCampaignOptions } from '../src/eval-campaign'
 import { runEvalCampaign } from '../src/eval-campaign'
-import { LlmRouteAssertionError } from '../src/llm-client'
 import { InMemoryRawProviderSink, NoopRawProviderSink } from '../src/trace/raw-provider-sink'
 import { InMemoryTraceStore } from '../src/trace/store'
 
@@ -10,6 +10,19 @@ interface VariantPayload {
   prompt: string
 }
 
+const EXECUTION_REF = 'https://api.test.local/v1'
+
+/** The caller owns execution; every runner here emits its own spans. */
+const chatFactory = () =>
+  createChatClient({
+    transport: 'custom',
+    defaultModel: 'test-model@2026-05-08',
+    maximumAttempts: 1,
+    chat: async () => {
+      throw new Error('no campaign test in this file calls the model')
+    },
+  })
+
 function baseOpts(
   overrides: Partial> = {},
 ): EvalCampaignOptions {
@@ -24,7 +37,8 @@ function baseOpts(
     scenarios: [{ scenarioId: 's1' }, { scenarioId: 's2' }],
     seeds: [0, 1],
     commitSha: 'cafebabe',
-    llmOpts: { baseUrl: 'https://api.test.local/v1', apiKey: 'sk-test' },
+    chatFactory,
+    executionRef: EXECUTION_REF,
     storeFactory: ({ runId }) => {
       const s = new InMemoryTraceStore()
       stores.set(runId, s)
@@ -58,7 +72,7 @@ const defaultRunner: CampaignRunner = async (ctx) => {
     provider: 'test',
     model: 'test-model@2026-05-08',
     endpoint: '/chat/completions',
-    baseUrl: ctx.llmOpts.baseUrl ?? '',
+    baseUrl: EXECUTION_REF,
     attemptIndex: 0,
     direction: 'request',
     timestamp: 1_000,
@@ -176,16 +190,6 @@ describe('runEvalCampaign — happy path', () => {
 })
 
 describe('runEvalCampaign — preflight', () => {
-  it('throws LlmRouteAssertionError when baseUrl is missing under the default policy', async () => {
-    await expect(
-      runEvalCampaign(
-        baseOpts({
-          llmOpts: { apiKey: 'sk-test' }, // no baseUrl
-        }),
-      ),
-    ).rejects.toBeInstanceOf(LlmRouteAssertionError)
-  })
-
   it('throws on duplicate variant ids', async () => {
     await expect(
       runEvalCampaign(
@@ -600,7 +604,7 @@ describe('runEvalCampaign — genuine-error containment (no orphaned workers)',
         provider: 'test',
         model: 'test-model@2026-05-08',
         endpoint: '/chat/completions',
-        baseUrl: ctx.llmOpts.baseUrl ?? '',
+        baseUrl: EXECUTION_REF,
         attemptIndex: 0,
         direction: 'request',
         timestamp: 1_000,
@@ -631,7 +635,8 @@ describe('runEvalCampaign — genuine-error containment (no orphaned workers)',
       scenarios: [{ scenarioId: 's1' }],
       seeds: [0],
       commitSha: 'cafebabe',
-      llmOpts: { baseUrl: 'https://api.test.local/v1', apiKey: 'sk-test' },
+      chatFactory,
+      executionRef: EXECUTION_REF,
       storeFactory: ({ runId }) => {
         const s = new InMemoryTraceStore()
         stores.set(runId, s)
@@ -704,7 +709,8 @@ describe('runEvalCampaign — abort failure is surfaced, not swallowed', () => {
       scenarios: [{ scenarioId: 's1' }],
       seeds: [0],
       commitSha: 'cafebabe',
-      llmOpts: { baseUrl: 'https://api.test.local/v1', apiKey: 'sk-test' },
+      chatFactory,
+      executionRef: EXECUTION_REF,
       storeFactory: () => new AbortFailingStore(),
       rawSinkFactory: () => new InMemoryRawProviderSink(),
       runner,
@@ -730,7 +736,8 @@ describe('runEvalCampaign — abort failure is surfaced, not swallowed', () => {
       scenarios: [{ scenarioId: 's1' }],
       seeds: [0],
       commitSha: 'cafebabe',
-      llmOpts: { baseUrl: 'https://api.test.local/v1', apiKey: 'sk-test' },
+      chatFactory,
+      executionRef: EXECUTION_REF,
       storeFactory: () => new InMemoryTraceStore(),
       rawSinkFactory: () => new InMemoryRawProviderSink(),
       runner,
diff --git a/tests/llm-route-assertion.test.ts b/tests/llm-route-assertion.test.ts
deleted file mode 100644
index 96aba5c1..00000000
--- a/tests/llm-route-assertion.test.ts
+++ /dev/null
@@ -1,74 +0,0 @@
-import { describe, expect, it } from 'vitest'
-import { assertLlmRoute, LlmRouteAssertionError } from '../src/llm-client'
-
-describe('assertLlmRoute', () => {
-  it('throws when requireExplicitBaseUrl is set and baseUrl is undefined', () => {
-    expect(() => assertLlmRoute({ apiKey: 'k' }, { requireExplicitBaseUrl: true })).toThrow(
-      LlmRouteAssertionError,
-    )
-  })
-
-  it('passes when baseUrl is explicit', () => {
-    expect(() =>
-      assertLlmRoute(
-        { baseUrl: 'https://api.openai.com/v1', apiKey: 'k' },
-        { requireExplicitBaseUrl: true },
-      ),
-    ).not.toThrow()
-  })
-
-  it('rejects URLs in the blocklist regardless of allowlist', () => {
-    expect(() =>
-      assertLlmRoute(
-        { baseUrl: 'https://router.tangle.tools/v1', apiKey: 'k' },
-        {
-          allowedBaseUrls: [/.*/],
-          blockedBaseUrls: ['https://router.tangle.tools'],
-        },
-      ),
-    ).toThrow(/blocked pattern/i)
-  })
-
-  it('requires a baseUrl in the allowlist', () => {
-    expect(() =>
-      assertLlmRoute(
-        { baseUrl: 'https://api.openai.com/v1', apiKey: 'k' },
-        { allowedBaseUrls: ['https://router.tangle.tools'] },
-      ),
-    ).toThrow(/not in the allowed list/)
-    expect(() =>
-      assertLlmRoute(
-        { baseUrl: 'https://api.openai.com/v1', apiKey: 'k' },
-        { allowedBaseUrls: [/api\.openai\.com/] },
-      ),
-    ).not.toThrow()
-  })
-
-  it('requires auth when requireAuth is set', () => {
-    expect(() => assertLlmRoute({ baseUrl: 'https://x' }, { requireAuth: true })).toThrow(
-      /no apiKey, bearer, or authHeader/,
-    )
-    expect(() =>
-      assertLlmRoute({ baseUrl: 'https://x', bearer: 'b' }, { requireAuth: true }),
-    ).not.toThrow()
-  })
-
-  it('checks expectedProvider against the resolved baseUrl', () => {
-    expect(() =>
-      assertLlmRoute({ baseUrl: 'https://api.openai.com/v1' }, { expectedProvider: 'anthropic' }),
-    ).toThrow(/expected provider anthropic/)
-    expect(() =>
-      assertLlmRoute({ baseUrl: 'https://api.openai.com/v1' }, { expectedProvider: 'openai' }),
-    ).not.toThrow()
-  })
-
-  it('exposes a structured reason for programmatic handling', () => {
-    try {
-      assertLlmRoute({}, { requireExplicitBaseUrl: true })
-    } catch (err) {
-      expect(err).toBeInstanceOf(LlmRouteAssertionError)
-      expect((err as LlmRouteAssertionError).reason).toBe('no_explicit_base_url')
-      expect((err as LlmRouteAssertionError).code).toBe('capture_integrity')
-    }
-  })
-})
diff --git a/tests/multishot/cell-cost-accounting.test.ts b/tests/multishot/cell-cost-accounting.test.ts
index fe603453..338bbb61 100644
--- a/tests/multishot/cell-cost-accounting.test.ts
+++ b/tests/multishot/cell-cost-accounting.test.ts
@@ -4,8 +4,8 @@
 //   1. `runMultishot` declares what the conversation spent when it throws,
 //      including every driver attempt that billed and returned nothing.
 //   2. `runMultishotMatrix` bills a cell whose shot result fails validation.
-//   3. A judge whose cost the router never reported marks the cell's total as
-//      a subtotal instead of presenting it as complete.
+//   3. A judge whose cost the transport never reported marks the cell's total
+//      as a subtotal instead of presenting it as complete.
 
 import { mkdtempSync, rmSync } from 'node:fs'
 import { tmpdir } from 'node:os'
@@ -39,19 +39,39 @@ const SHAPE: MultishotShape = {
   buildDriverSystemPrompt: (p) => `driver for ${p.name}`,
 }
 
-function conversationJudge(): JudgeConfig> {
+/** A leg the scenario must never reach — reaching it is the failure. */
+function unreachedTransport(leg: string): MultishotTransport {
+  return async () => {
+    throw new Error(`${leg} leg must not run in this scenario`)
+  }
+}
+
+/** Judge leg: a reply whose cost the transport reports, or withholds. */
+function judgeTransport(options: { reportCost: boolean }): MultishotTransport {
+  return async () => ({
+    message: { content: JSON.stringify({ quality: 8, notes: 'fine' }) },
+    usage: { prompt_tokens: 100, completion_tokens: 50 },
+    ...(options.reportCost ? { costUsd: 0.002 } : {}),
+  })
+}
+
+function conversationJudge(
+  transport: MultishotTransport,
+  model = 'openai/gpt-4o-mini',
+): JudgeConfig> {
   return {
     name: 'conversation',
-    model: 'openai/gpt-4o-mini',
+    transport,
+    model,
     dimensions: [{ key: 'quality', description: 'overall quality' }],
     systemPrompt: 'score the input',
     buildPrompt: (input) => `CONVERSATION turns=${input.transcript.length}`,
-    apiKey: 'judge-key',
-    baseUrl: 'http://judge.invalid/v1',
   }
 }
 
-const JUDGES: MultishotJudges = { conversation: conversationJudge() }
+function pricedJudges(): MultishotJudges {
+  return { conversation: conversationJudge(judgeTransport({ reportCost: true })) }
+}
 
 let tempDirs: string[] = []
 
@@ -61,30 +81,11 @@ function newRunDir(): string {
   return dir
 }
 
-/** Judge leg: a router reply whose cost is reported, or withheld by a model
- *  the ledger cannot price. */
-function stubJudgeFetch(options: { reportCost: boolean; model?: string }): void {
-  vi.stubGlobal(
-    'fetch',
-    vi.fn(async () => ({
-      ok: true,
-      status: 200,
-      json: async () => ({
-        choices: [{ message: { content: JSON.stringify({ quality: 8, notes: 'fine' }) } }],
-        model: options.model ?? 'openai/gpt-4o-mini',
-        usage: { prompt_tokens: 100, completion_tokens: 50 },
-        ...(options.reportCost ? { cost_usd: 0.002 } : {}),
-      }),
-    })),
-  )
-}
-
 beforeEach(() => {
   tempDirs = []
 })
 
 afterEach(() => {
-  vi.unstubAllGlobals()
   for (const dir of tempDirs) rmSync(dir, { recursive: true, force: true })
 })
 
@@ -111,8 +112,6 @@ describe('runMultishot — a throw declares the conversation spend', () => {
       driverFallbackModels: ['openai/gpt-4.1-mini'],
       agentTransport,
       driverTransport,
-      apiKey: 'agent-key',
-      baseUrl: 'http://agent.invalid/v1',
     }).then(
       () => undefined,
       (e: unknown) => e,
@@ -144,8 +143,6 @@ describe('runMultishot — a throw declares the conversation spend', () => {
       driverModel: 'openai/gpt-4o-mini',
       agentTransport,
       driverTransport,
-      apiKey: 'agent-key',
-      baseUrl: 'http://agent.invalid/v1',
     }).then(
       () => undefined,
       (e: unknown) => e,
@@ -160,7 +157,6 @@ describe('runMultishot — a throw declares the conversation spend', () => {
 
 describe('runMultishotMatrix — a failed cell keeps its spend', () => {
   it('bills the shot that spent before returning a malformed result', async () => {
-    stubJudgeFetch({ reportCost: true })
     const runShot: MultishotShot = async () =>
       ({
         transcript: 'not an array',
@@ -174,8 +170,10 @@ describe('runMultishotMatrix — a failed cell keeps its spend', () => {
       profiles: [{ id: 'p1', value: PROFILE }],
       personas: [PERSONA],
       shape: SHAPE,
-      judges: JUDGES,
+      judges: pricedJudges(),
       runDir: newRunDir(),
+      agentTransport: unreachedTransport('agent'),
+      driverTransport: unreachedTransport('driver'),
       runShot,
     })
 
@@ -188,7 +186,6 @@ describe('runMultishotMatrix — a failed cell keeps its spend', () => {
   })
 
   it('records the cell as uncaptured when the shot result carries no usable cost', async () => {
-    stubJudgeFetch({ reportCost: true })
     const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
     const runShot: MultishotShot = async () =>
       ({
@@ -203,8 +200,10 @@ describe('runMultishotMatrix — a failed cell keeps its spend', () => {
       profiles: [{ id: 'p1', value: PROFILE }],
       personas: [PERSONA],
       shape: SHAPE,
-      judges: JUDGES,
+      judges: pricedJudges(),
       runDir: newRunDir(),
+      agentTransport: unreachedTransport('agent'),
+      driverTransport: unreachedTransport('driver'),
       runShot,
     })
 
@@ -217,7 +216,6 @@ describe('runMultishotMatrix — a failed cell keeps its spend', () => {
   })
 
   it('stops scheduling once failed cells spend past the ceiling', async () => {
-    stubJudgeFetch({ reportCost: true })
     const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
     let shots = 0
     const runShot: MultishotShot = async () => {
@@ -240,10 +238,12 @@ describe('runMultishotMatrix — a failed cell keeps its spend', () => {
       ],
       personas: [PERSONA],
       shape: SHAPE,
-      judges: JUDGES,
+      judges: pricedJudges(),
       runDir: newRunDir(),
       maxConcurrency: 1,
       costCeiling: 0.5,
+      agentTransport: unreachedTransport('agent'),
+      driverTransport: unreachedTransport('driver'),
       runShot,
     })
 
@@ -258,7 +258,6 @@ describe('runMultishotMatrix — a failed cell keeps its spend', () => {
   it('marks a successful cell uncaptured when the SHOT priced a call at nothing', async () => {
     // Judges report their cost; only the agent leg is unpriced, so the cell can
     // only learn its total is a subtotal from the shot itself.
-    stubJudgeFetch({ reportCost: true })
     const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
     const agentTransport = vi.fn(async () => ({
       message: { content: 'agent answered' },
@@ -268,12 +267,11 @@ describe('runMultishotMatrix — a failed cell keeps its spend', () => {
       profiles: [{ id: 'p1', value: PROFILE }],
       personas: [PERSONA],
       shape: SHAPE,
-      judges: JUDGES,
+      judges: pricedJudges(),
       runDir: newRunDir(),
       maxTurns: 1,
       agentTransport,
-      apiKey: 'agent-key',
-      baseUrl: 'http://agent.invalid/v1',
+      driverTransport: unreachedTransport('driver'),
     })
 
     const run = matrix.cells[0]?.runs[0]
@@ -284,7 +282,6 @@ describe('runMultishotMatrix — a failed cell keeps its spend', () => {
   })
 
   it('reports a fully priced shot as a complete estimate', async () => {
-    stubJudgeFetch({ reportCost: true })
     const agentTransport = vi.fn(async () => ({
       message: { content: 'agent answered' },
       costUsd: 0.2,
@@ -294,12 +291,11 @@ describe('runMultishotMatrix — a failed cell keeps its spend', () => {
       profiles: [{ id: 'p1', value: PROFILE }],
       personas: [PERSONA],
       shape: SHAPE,
-      judges: JUDGES,
+      judges: pricedJudges(),
       runDir: newRunDir(),
       maxTurns: 1,
       agentTransport,
-      apiKey: 'agent-key',
-      baseUrl: 'http://agent.invalid/v1',
+      driverTransport: unreachedTransport('driver'),
     })
 
     const run = matrix.cells[0]?.runs[0]
@@ -308,7 +304,6 @@ describe('runMultishotMatrix — a failed cell keeps its spend', () => {
   })
 
   it('rejects a shot that claims an uncaptured provenance with a total', async () => {
-    stubJudgeFetch({ reportCost: true })
     const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
     const runShot: MultishotShot = async () =>
       ({
@@ -324,8 +319,10 @@ describe('runMultishotMatrix — a failed cell keeps its spend', () => {
       profiles: [{ id: 'p1', value: PROFILE }],
       personas: [PERSONA],
       shape: SHAPE,
-      judges: JUDGES,
+      judges: pricedJudges(),
       runDir: newRunDir(),
+      agentTransport: unreachedTransport('agent'),
+      driverTransport: unreachedTransport('driver'),
       runShot,
     })
 
@@ -338,7 +335,6 @@ describe('runMultishotMatrix — a failed cell keeps its spend', () => {
   })
 
   it('marks a successful cell uncaptured when a judge cost was never reported', async () => {
-    stubJudgeFetch({ reportCost: false, model: 'vendor/unpriced-model' })
     const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
     const runShot: MultishotShot = async () => ({
       transcript: [{ role: 'user', content: 'hello' }],
@@ -352,9 +348,16 @@ describe('runMultishotMatrix — a failed cell keeps its spend', () => {
       profiles: [{ id: 'p1', value: PROFILE }],
       personas: [PERSONA],
       // An unpriced model leaves the judge call with no reportable cost.
-      judges: { conversation: { ...conversationJudge(), model: 'vendor/unpriced-model' } },
+      judges: {
+        conversation: conversationJudge(
+          judgeTransport({ reportCost: false }),
+          'vendor/unpriced-model',
+        ),
+      },
       shape: SHAPE,
       runDir: newRunDir(),
+      agentTransport: unreachedTransport('agent'),
+      driverTransport: unreachedTransport('driver'),
       runShot,
     })
 
diff --git a/tests/multishot/judges.test.ts b/tests/multishot/judges.test.ts
index c5d95419..d58a78ae 100644
--- a/tests/multishot/judges.test.ts
+++ b/tests/multishot/judges.test.ts
@@ -1,8 +1,10 @@
-import { describe, expect, it, vi } from 'vitest'
+import { describe, expect, it } from 'vitest'
 import {
   computeCellComposite,
   type JudgeConfig,
   type JudgeScore as JudgeScoreShape,
+  type MultishotTransport,
+  type MultishotTransportRequest,
   renderDimensions,
   renderJsonFooter,
   runJudge,
@@ -13,48 +15,39 @@ const DIMS = [
   { key: 'specificity', description: 'concrete vs vague 0-10' },
 ] as const
 
-const JUDGE: JudgeConfig<{ text: string }> = {
-  name: 'test-judge',
-  dimensions: [...DIMS],
-  systemPrompt: 'You are a strict judge. JSON only.',
-  buildPrompt: ({ text }) => `Score:\n${text}\n${renderJsonFooter(DIMS)}`,
+function judgeWith(transport: MultishotTransport): JudgeConfig<{ text: string }> {
+  return {
+    name: 'test-judge',
+    transport,
+    dimensions: [...DIMS],
+    systemPrompt: 'You are a strict judge. JSON only.',
+    buildPrompt: ({ text }) => `Score:\n${text}\n${renderJsonFooter(DIMS)}`,
+  }
 }
 
-function stubFetch(responses: Array<{ ok?: boolean; body: unknown }>) {
-  let i = 0
-  return vi.fn(async () => {
-    const r = responses[i++]
-    if (!r) throw new Error('stub exhausted')
-    return {
-      ok: r.ok ?? true,
-      status: 200,
-      json: async () => r.body,
-      text: async () => JSON.stringify(r.body),
-    } as Response
+function replyTransport(reply: {
+  content: string
+  usage?: { prompt_tokens?: number; completion_tokens?: number }
+  costUsd?: number
+}): MultishotTransport {
+  return async () => ({
+    message: { content: reply.content },
+    usage: reply.usage,
+    costUsd: reply.costUsd,
   })
 }
 
 describe('runJudge', () => {
   it('parses dimensions + composite + notes', async () => {
-    const original = global.fetch
-    process.env.TANGLE_API_KEY = 'test-key'
-    global.fetch = stubFetch([
-      {
-        body: {
-          choices: [
-            {
-              message: {
-                content: '{"quality":8,"specificity":6,"notes":"good but vague at the end"}',
-              },
-            },
-          ],
-          usage: { prompt_tokens: 100, completion_tokens: 50 },
-          _response_cost: 0.0123,
-        },
-      },
-    ]) as unknown as typeof fetch
-
-    const { score, cost } = await runJudge(JUDGE, { text: 'test transcript' })
+    const judge = judgeWith(
+      replyTransport({
+        content: '{"quality":8,"specificity":6,"notes":"good but vague at the end"}',
+        usage: { prompt_tokens: 100, completion_tokens: 50 },
+        costUsd: 0.0123,
+      }),
+    )
+
+    const { score, cost } = await runJudge(judge, { text: 'test transcript' })
     expect(score.dimensions.quality).toBe(8)
     expect(score.dimensions.specificity).toBe(6)
     expect(score.composite).toBe(7)
@@ -70,36 +63,26 @@ describe('runJudge', () => {
       },
     })
     expect(cost).toEqual({ kind: 'observed', usd: 0.0123 })
-    global.fetch = original
   })
 
   it('clamps out-of-range values + defaults missing to 0', async () => {
-    const original = global.fetch
-    process.env.TANGLE_API_KEY = 'test-key'
-    global.fetch = stubFetch([
-      { body: { choices: [{ message: { content: '{"quality":15}' } }] } },
-    ]) as unknown as typeof fetch
+    const judge = judgeWith(replyTransport({ content: '{"quality":15}' }))
 
-    const { score } = await runJudge(JUDGE, { text: 'x' })
+    const { score } = await runJudge(judge, { text: 'x' })
     expect(score.dimensions.quality).toBe(10) // clamped from 15
     expect(score.dimensions.specificity).toBe(0) // missing → 0
     expect(score.composite).toBe(5)
-    global.fetch = original
   })
 
   it('marks non-JSON replies as failed (additive) with parse-failure note', async () => {
-    const original = global.fetch
-    process.env.TANGLE_API_KEY = 'test-key'
-    global.fetch = stubFetch([
-      {
-        body: {
-          choices: [{ message: { content: 'I cannot output JSON because reasons' } }],
-          usage: { prompt_tokens: 100, completion_tokens: 50 },
-        },
-      },
-    ]) as unknown as typeof fetch
-
-    const { score, cost } = await runJudge(JUDGE, { text: 'x' })
+    const judge = judgeWith(
+      replyTransport({
+        content: 'I cannot output JSON because reasons',
+        usage: { prompt_tokens: 100, completion_tokens: 50 },
+      }),
+    )
+
+    const { score, cost } = await runJudge(judge, { text: 'x' })
     expect(score.composite).toBe(0)
     expect(score.notes).toMatch(/non-JSON/)
     // failed:true lets aggregators exclude this score instead of meaning a zero.
@@ -115,21 +98,12 @@ describe('runJudge', () => {
     expect(score.llmCall?.costUsd).toBeNull()
     expect(cost.kind).toBe('estimated')
     expect(cost.usd).toBeCloseTo(0.000045, 10)
-    global.fetch = original
   })
 
   it('reports uncaptured usage and unknown cost instead of a zero', async () => {
-    const original = global.fetch
-    process.env.TANGLE_API_KEY = 'test-key'
-    global.fetch = stubFetch([
-      {
-        body: {
-          choices: [{ message: { content: '{"quality":8,"specificity":6}' } }],
-        },
-      },
-    ]) as unknown as typeof fetch
+    const judge = judgeWith(replyTransport({ content: '{"quality":8,"specificity":6}' }))
 
-    const { score, cost } = await runJudge(JUDGE, { text: 'x' })
+    const { score, cost } = await runJudge(judge, { text: 'x' })
     expect(score.llmCall).toMatchObject({
       costUsd: null,
       usage: {
@@ -140,17 +114,14 @@ describe('runJudge', () => {
       },
     })
     expect(cost).toEqual({ kind: 'uncaptured', usd: null })
-    global.fetch = original
   })
 
   it('preserves a failed score and uncaptured cost when the provider call fails', async () => {
-    const original = global.fetch
-    process.env.TANGLE_API_KEY = 'test-key'
-    global.fetch = stubFetch([
-      { ok: false, body: { error: 'unavailable' } },
-    ]) as unknown as typeof fetch
+    const judge = judgeWith(async () => {
+      throw new Error('unavailable')
+    })
 
-    const { score, cost } = await runJudge(JUDGE, { text: 'x' })
+    const { score, cost } = await runJudge(judge, { text: 'x' })
     expect(score).toMatchObject({
       composite: 0,
       failed: true,
@@ -161,40 +132,28 @@ describe('runJudge', () => {
     })
     expect(score.notes).toMatch(/call failed/)
     expect(cost).toEqual({ kind: 'uncaptured', usd: null })
-    global.fetch = original
   })
 
   it('strips ```json fences before parsing', async () => {
-    const original = global.fetch
-    process.env.TANGLE_API_KEY = 'test-key'
-    global.fetch = stubFetch([
-      {
-        body: {
-          choices: [{ message: { content: '```json\n{"quality":7,"specificity":5}\n```' } }],
-        },
-      },
-    ]) as unknown as typeof fetch
+    const judge = judgeWith(
+      replyTransport({ content: '```json\n{"quality":7,"specificity":5}\n```' }),
+    )
 
-    const { score } = await runJudge(JUDGE, { text: 'x' })
+    const { score } = await runJudge(judge, { text: 'x' })
     expect(score.dimensions.quality).toBe(7)
     expect(score.composite).toBe(6)
-    global.fetch = original
   })
 
   it('uses configured max token budget for judge calls', async () => {
-    const original = global.fetch
-    process.env.TANGLE_API_KEY = 'test-key'
-    const fetchStub = stubFetch([
-      { body: { choices: [{ message: { content: '{"quality":8,"specificity":8}' } }] } },
-    ])
-    global.fetch = fetchStub as unknown as typeof fetch
-
-    await runJudge({ ...JUDGE, maxTokens: 321 }, { text: 'x' })
+    const requests: MultishotTransportRequest[] = []
+    const judge = judgeWith(async (req) => {
+      requests.push(req)
+      return { message: { content: '{"quality":8,"specificity":8}' } }
+    })
 
-    const body = JSON.parse(String(fetchStub.mock.calls[0][1]?.body)) as Record
-    expect(body.max_tokens).toBe(321)
+    await runJudge({ ...judge, maxTokens: 321 }, { text: 'x' })
 
-    global.fetch = original
+    expect(requests[0]?.maxTokens).toBe(321)
   })
 })
 
diff --git a/tests/multishot/matrix-shot-seam.test.ts b/tests/multishot/matrix-shot-seam.test.ts
index 3cd64d5f..91bd14c8 100644
--- a/tests/multishot/matrix-shot-seam.test.ts
+++ b/tests/multishot/matrix-shot-seam.test.ts
@@ -99,18 +99,34 @@ function judgeScoreForPrompt(prompt: string): number {
   throw new Error(`unscripted judge prompt: ${prompt.slice(0, 60)}`)
 }
 
+let judgeCalls = 0
+
+const judgeTransport: MultishotTransport = async (req) => {
+  judgeCalls++
+  return {
+    message: {
+      content: JSON.stringify({
+        quality: judgeScoreForPrompt(String(req.messages[1]?.content ?? '')),
+        notes: 'ok',
+      }),
+    },
+    usage: { prompt_tokens: 10, completion_tokens: 5 },
+    model: 'openai/gpt-4o-mini',
+    costUsd: JUDGE_COST_USD,
+  }
+}
+
 function judgeConfig(
   name: string,
   buildPrompt: (input: TInput) => string,
 ): JudgeConfig {
   return {
     name,
+    transport: judgeTransport,
     model: 'openai/gpt-4o-mini',
     dimensions: [{ key: 'quality', description: 'overall quality' }],
     systemPrompt: 'score the input',
     buildPrompt,
-    apiKey: 'judge-key',
-    baseUrl: 'http://judge.invalid/v1',
   }
 }
 
@@ -152,7 +168,6 @@ const driverTransport = vi.fn(async () => ({
   costUsd: 0.001,
 }))
 
-let judgeCalls = 0
 let tempDirs: string[] = []
 
 function newRunDir(): string {
@@ -184,8 +199,6 @@ function baseOptions(runDir: string): RunMultishotMatrixOptions {
     judgeMaxTokens: 999,
     agentTransport,
     driverTransport,
-    apiKey: 'agent-key',
-    baseUrl: 'http://agent.invalid/v1',
   }
 }
 
@@ -194,37 +207,9 @@ beforeEach(() => {
   tempDirs = []
   agentTransport.mockClear()
   driverTransport.mockClear()
-  vi.stubGlobal(
-    'fetch',
-    vi.fn(async (_url: string, init?: RequestInit) => {
-      judgeCalls++
-      const body = JSON.parse(String(init?.body)) as {
-        messages: Array<{ role: string; content: string }>
-      }
-      const prompt = body.messages[1]?.content ?? ''
-      return {
-        ok: true,
-        status: 200,
-        json: async () => ({
-          choices: [
-            {
-              message: {
-                content: JSON.stringify({ quality: judgeScoreForPrompt(prompt), notes: 'ok' }),
-              },
-            },
-          ],
-          usage: { prompt_tokens: 10, completion_tokens: 5 },
-          model: 'openai/gpt-4o-mini',
-          _response_cost: JUDGE_COST_USD,
-        }),
-        text: async () => 'ok',
-      } as Response
-    }),
-  )
 })
 
 afterEach(() => {
-  vi.unstubAllGlobals()
   for (const dir of tempDirs) rmSync(dir, { recursive: true, force: true })
 })
 
@@ -455,8 +440,6 @@ describe('runMultishotMatrix shot seam', () => {
     expect(input.driverMaxTokens).toBe(321)
     expect(input.agentTransport).toBe(agentTransport)
     expect(input.driverTransport).toBe(driverTransport)
-    expect(input.apiKey).toBe('agent-key')
-    expect(input.baseUrl).toBe('http://agent.invalid/v1')
   })
 
   // `maxTurns: 1` never reaches the driver leg, so the case above compares two
@@ -508,6 +491,8 @@ describe('runMultishotMatrix shot seam', () => {
     const reference = await syntheticShot()({
       profile: PROFILES[0]!.value,
       persona: PERSONAS[0]!,
+      agentTransport,
+      driverTransport,
     })
     expect(readCellJson(runDir, 'p1', 'alice', 'transcript.json')).toEqual(reference.transcript)
     expect(readCellJson(runDir, 'p1', 'alice', 'artifacts.json')).toEqual(reference.artifacts)
diff --git a/tests/multishot/matrix-transport.test.ts b/tests/multishot/matrix-transport.test.ts
index 660a40bf..0726d0b3 100644
--- a/tests/multishot/matrix-transport.test.ts
+++ b/tests/multishot/matrix-transport.test.ts
@@ -1,16 +1,16 @@
-// Proves runMultishotMatrix plumbs the transport seams into every cell:
-// agent + driver legs run through the injected transports (no router HTTP),
-// judges keep using the router, and agent/driver/judge cost flows into the
-// matrix cost accounting.
+// Proves runMultishotMatrix plumbs the transport seams into every cell: agent,
+// driver, and judge legs each run on the transport the caller supplied, and
+// every leg's cost flows into the matrix cost accounting.
 
 import { mkdtempSync, readFileSync, rmSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import type { AgentProfile } from '@tangle-network/agent-interface'
-import { afterEach, describe, expect, it, vi } from 'vitest'
+import { describe, expect, it, vi } from 'vitest'
 import {
   type MultishotPersona,
   type MultishotShape,
+  type MultishotTransport,
   type MultishotTransportRequest,
   type MultishotTransportResponse,
   runMultishotMatrix,
@@ -31,36 +31,25 @@ const SHAPE: MultishotShape = {
   buildDriverSystemPrompt: (p) => `you are ${p.name}`,
 }
 
-const originalFetch = global.fetch
-
-afterEach(() => {
-  global.fetch = originalFetch
-})
+/** A leg the scenario must never reach — reaching it is the failure. */
+function unreachedTransport(leg: string): MultishotTransport {
+  return async () => {
+    throw new Error(`${leg} leg must not run in this scenario`)
+  }
+}
 
-function judgeOnlyFetch() {
-  // Serves the conversation judge; any other HTTP call is a seam leak.
-  return vi.fn(async (_url: string, init?: RequestInit) => {
-    const body = JSON.parse(String(init?.body)) as { messages: Array<{ content?: string }> }
-    if (!String(body.messages[0]?.content).includes('judge')) {
-      throw new Error('unexpected non-judge HTTP call — transport seam leaked')
-    }
-    return {
-      ok: true,
-      status: 200,
-      json: async () => ({
-        choices: [{ message: { content: '{"helpfulness":8,"notes":"fine"}' } }],
-        usage: { prompt_tokens: 10, completion_tokens: 10 },
-      }),
-      text: async () => 'ok',
-    } as Response
-  })
+function scoringJudgeTransport() {
+  return vi.fn(
+    async (_req: MultishotTransportRequest): Promise => ({
+      message: { content: '{"helpfulness":8,"notes":"fine"}' },
+      usage: { prompt_tokens: 10, completion_tokens: 10 },
+    }),
+  )
 }
 
 describe('runMultishotMatrix transport seam', () => {
   it('passes injected transports into each cell and totals agent, driver, and judge cost', async () => {
-    process.env.TANGLE_API_KEY = 'test-key'
-    const fetchStub = judgeOnlyFetch()
-    global.fetch = fetchStub as unknown as typeof fetch
+    const judgeTransport = scoringJudgeTransport()
 
     const agentTransport = vi.fn(
       async (_req: MultishotTransportRequest): Promise => ({
@@ -84,6 +73,7 @@ describe('runMultishotMatrix transport seam', () => {
         judges: {
           conversation: {
             name: 'conversation',
+            transport: judgeTransport,
             dimensions: [{ key: 'helpfulness', description: 'is it helpful' }],
             systemPrompt: 'you are a judge',
             buildPrompt: () => 'judge this transcript',
@@ -95,11 +85,10 @@ describe('runMultishotMatrix transport seam', () => {
         driverTransport,
       })
 
-      // 2 agent turns + 1 driver turn per cell.
+      // 2 agent turns + 1 driver turn + 1 judge call per cell.
       expect(agentTransport).toHaveBeenCalledTimes(2)
       expect(driverTransport).toHaveBeenCalledTimes(1)
-      // Judge ran over HTTP; the agent/driver legs did not.
-      expect(fetchStub).toHaveBeenCalledTimes(1)
+      expect(judgeTransport).toHaveBeenCalledTimes(1)
       // Transport costUsd (0.2*2 + 0.1) plus the judge's estimated usage cost
       // flows into the cell and matrix totals.
       expect(matrix.cells[0]?.runs[0]?.costUsd).toBeCloseTo(0.5000075, 10)
@@ -116,24 +105,18 @@ describe('runMultishotMatrix transport seam', () => {
   })
 
   it('counts conversation, code, and content judge calls, including parse failures', async () => {
-    process.env.TANGLE_API_KEY = 'test-key'
-    const fetchStub = vi.fn(async (_url: string, init?: RequestInit) => {
-      const body = JSON.parse(String(init?.body)) as { messages: Array<{ content?: string }> }
-      const systemPrompt = String(body.messages[0]?.content)
-      const content = systemPrompt.includes('content judge')
-        ? 'not json'
-        : '{"quality":8,"notes":"fine"}'
-      return {
-        ok: true,
-        status: 200,
-        json: async () => ({
-          choices: [{ message: { content } }],
+    const judgeTransport = vi.fn(
+      async (req: MultishotTransportRequest): Promise => {
+        const systemPrompt = String(req.messages[0]?.content)
+        const content = systemPrompt.includes('content judge')
+          ? 'not json'
+          : '{"quality":8,"notes":"fine"}'
+        return {
+          message: { content },
           usage: { prompt_tokens: 100, completion_tokens: 100 },
-        }),
-        text: async () => 'ok',
-      } as Response
-    })
-    global.fetch = fetchStub as unknown as typeof fetch
+        }
+      },
+    )
 
     const agentTransport = vi
       .fn<(req: MultishotTransportRequest) => Promise>()
@@ -169,18 +152,21 @@ describe('runMultishotMatrix transport seam', () => {
         judges: {
           conversation: {
             name: 'conversation',
+            transport: judgeTransport,
             dimensions: [{ key: 'quality', description: 'conversation quality' }],
             systemPrompt: 'conversation judge',
             buildPrompt: () => 'judge the conversation',
           },
           codeReview: {
             name: 'code',
+            transport: judgeTransport,
             dimensions: [{ key: 'quality', description: 'code quality' }],
             systemPrompt: 'code judge',
             buildPrompt: () => 'judge the code',
           },
           contentQuality: {
             name: 'content',
+            transport: judgeTransport,
             dimensions: [{ key: 'quality', description: 'content quality' }],
             systemPrompt: 'content judge',
             buildPrompt: () => 'judge the content',
@@ -212,13 +198,14 @@ describe('runMultishotMatrix transport seam', () => {
         runDir,
         maxTurns: 1,
         agentTransport,
+        driverTransport: unreachedTransport('driver'),
       })
 
       // Simulation: 2 agent calls + 2 tools = $0.50.
       // Judges: 3 * (100 input + 100 output tokens on gpt-4o-mini) = $0.000225.
       expect(matrix.cells[0]?.runs[0]?.costUsd).toBeCloseTo(0.500225, 10)
       expect(matrix.summary.totalCostUsd).toBeCloseTo(0.500225, 10)
-      expect(fetchStub).toHaveBeenCalledTimes(3)
+      expect(judgeTransport).toHaveBeenCalledTimes(3)
 
       const scores = JSON.parse(
         readFileSync(join(runDir, 'p1', 'alice', 'rep-0', 'scores.json'), 'utf8'),
@@ -246,9 +233,7 @@ describe('runMultishotMatrix transport seam', () => {
   })
 
   it('uses judge cost when deciding whether to schedule another cell', async () => {
-    process.env.TANGLE_API_KEY = 'test-key'
-    const fetchStub = judgeOnlyFetch()
-    global.fetch = fetchStub as unknown as typeof fetch
+    const judgeTransport = scoringJudgeTransport()
     const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
     const agentTransport = vi.fn(
       async (_req: MultishotTransportRequest): Promise => ({
@@ -270,6 +255,7 @@ describe('runMultishotMatrix transport seam', () => {
         judges: {
           conversation: {
             name: 'conversation',
+            transport: judgeTransport,
             dimensions: [{ key: 'helpfulness', description: 'is it helpful' }],
             systemPrompt: 'you are a judge',
             buildPrompt: () => 'judge this transcript',
@@ -280,13 +266,14 @@ describe('runMultishotMatrix transport seam', () => {
         maxConcurrency: 1,
         costCeiling: 0.000007,
         agentTransport,
+        driverTransport: unreachedTransport('driver'),
       })
 
       expect(matrix.summary.runsExecuted).toBe(1)
       expect(matrix.summary.cellsSkipped).toBe(2)
       expect(matrix.summary.totalCostUsd).toBeCloseTo(0.0000075, 10)
       expect(agentTransport).toHaveBeenCalledOnce()
-      expect(fetchStub).toHaveBeenCalledOnce()
+      expect(judgeTransport).toHaveBeenCalledOnce()
       expect(warn).toHaveBeenCalledWith('[matrix] cost ceiling reached')
     } finally {
       warn.mockRestore()
diff --git a/tests/multishot/multishot.test.ts b/tests/multishot/multishot.test.ts
index 2388bae8..cc10a8cd 100644
--- a/tests/multishot/multishot.test.ts
+++ b/tests/multishot/multishot.test.ts
@@ -5,6 +5,8 @@ import {
   MultishotFatalToolError,
   type MultishotPersona,
   type MultishotShape,
+  type MultishotToolDefinition,
+  type MultishotTransport,
   type MultishotTransportRequest,
   type MultishotTransportResponse,
   runMultishot,
@@ -27,45 +29,55 @@ const SHAPE: MultishotShape = {
 
 const PERSONA: TestPersona = { id: 'alice', name: 'Alice' }
 
-function makeFetchStub(
+const USAGE = { prompt_tokens: 10, completion_tokens: 20 }
+
+const CUSTOM_TOOL: MultishotToolDefinition = {
+  type: 'function',
+  function: {
+    name: 'my_custom_tool',
+    description: 'test',
+    parameters: { type: 'object', properties: {} },
+  },
+}
+
+function makeTransportStub(
   responses: Array<{
     content?: string
     toolCalls?: Array<{ name: string; args: Record }>
+    usage?: { prompt_tokens?: number; completion_tokens?: number }
+    costUsd?: number
   }>,
 ) {
   let i = 0
-  return vi.fn(async (_url: string, _init?: RequestInit) => {
+  return vi.fn(async (_req: MultishotTransportRequest): Promise => {
     const r = responses[i++]
-    if (!r) throw new Error(`fetch stub exhausted at call ${i}`)
-    const message: {
-      content: string | null
-      tool_calls?: Array<{
-        id: string
-        type: 'function'
-        function: { name: string; arguments: string }
-      }>
-    } = {
-      content: r.content ?? null,
-    }
-    if (r.toolCalls?.length) {
-      message.tool_calls = r.toolCalls.map((tc, idx) => ({
-        id: `call-${i}-${idx}`,
-        type: 'function' as const,
-        function: { name: tc.name, arguments: JSON.stringify(tc.args) },
-      }))
-    }
+    if (!r) throw new Error(`transport stub exhausted at call ${i}`)
     return {
-      ok: true,
-      status: 200,
-      json: async () => ({
-        choices: [{ message }],
-        usage: { prompt_tokens: 10, completion_tokens: 20 },
-      }),
-      text: async () => 'ok',
-    } as Response
+      message: {
+        content: r.content ?? null,
+        ...(r.toolCalls?.length
+          ? {
+              tool_calls: r.toolCalls.map((tc, idx) => ({
+                id: `t-${i}-${idx}`,
+                type: 'function' as const,
+                function: { name: tc.name, arguments: JSON.stringify(tc.args) },
+              })),
+            }
+          : {}),
+      },
+      usage: r.usage,
+      costUsd: r.costUsd,
+    }
   })
 }
 
+/** A leg the scenario must never reach — reaching it is the failure. */
+function unreachedTransport(leg: string): MultishotTransport {
+  return async () => {
+    throw new Error(`${leg} leg must not run in this scenario`)
+  }
+}
+
 describe('runMultishot', () => {
   it('sends the profile append prompt after the base system prompt', async () => {
     const agentRequests: MultishotTransportRequest[] = []
@@ -84,8 +96,7 @@ describe('runMultishot', () => {
         agentRequests.push(request)
         return { message: { content: 'done' }, costUsd: 0 }
       },
-      apiKey: 'test-key',
-      baseUrl: 'http://localhost:0',
+      driverTransport: unreachedTransport('driver'),
     })
 
     expect(agentRequests[0]?.messages[0]).toEqual({
@@ -97,28 +108,31 @@ describe('runMultishot', () => {
   })
 
   it('runs N turns, captures transcript + tool calls + cost', async () => {
-    const originalFetch = global.fetch
-    // Sequence per turn (maxTurns=2):
+    // Agent leg per turn (maxTurns=2):
     // t0 agent: tool_call delegate_research
-    // tool exec: research result (1 call)
+    // t0 tool leg: the research specialist, which defaults to the agent leg
     // t0 agent follow-up: text
-    // t0 driver: pushback
     // t1 agent: text
-    // (no driver turn after last)
-    global.fetch = makeFetchStub([
-      { toolCalls: [{ name: 'delegate_research', args: { question: 'who is alice?' } }] },
-      { content: '# Research Brief\n- Finding 1: alice exists [src: census]' },
-      { content: 'after research: hello Alice — based on the brief, you exist.' },
-      { content: 'great, but i need more specifics about MY situation' },
-      { content: 'specifically, you are user alice. final brief.' },
-    ]) as unknown as typeof fetch
-    process.env.TANGLE_API_KEY = 'test-key'
+    const agentTransport = makeTransportStub([
+      {
+        toolCalls: [{ name: 'delegate_research', args: { question: 'who is alice?' } }],
+        usage: USAGE,
+      },
+      { content: '# Research Brief\n- Finding 1: alice exists [src: census]', usage: USAGE },
+      { content: 'after research: hello Alice — based on the brief, you exist.', usage: USAGE },
+      { content: 'specifically, you are user alice. final brief.', usage: USAGE },
+    ])
+    const driverTransport = makeTransportStub([
+      { content: 'great, but i need more specifics about MY situation', usage: USAGE },
+    ])
 
     const result = await runMultishot({
       profile: PROFILE,
       persona: PERSONA,
       shape: SHAPE,
       maxTurns: 2,
+      agentTransport,
+      driverTransport,
     })
 
     expect(result.transcript.filter((m) => m.role === 'assistant').length).toBeGreaterThanOrEqual(2)
@@ -128,51 +142,50 @@ describe('runMultishot', () => {
     expect(result.artifacts[0].invocation.name).toBe('delegate_research')
     expect(result.costUsd).toBeGreaterThan(0)
     expect(result.durationMs).toBeGreaterThanOrEqual(0)
-
-    global.fetch = originalFetch
   })
 
   it('throws MultishotDriverEmptyError when driver returns empty twice', async () => {
-    const originalFetch = global.fetch
-    // t0 agent: text → t0 driver attempt 1: empty → driver attempt 2: empty → throws
-    global.fetch = makeFetchStub([
-      { content: 'agent turn 0 text' },
-      { content: '' },
-      { content: '' },
-    ]) as unknown as typeof fetch
-    process.env.TANGLE_API_KEY = 'test-key'
+    const agentTransport = makeTransportStub([{ content: 'agent turn 0 text', usage: USAGE }])
+    const driverTransport = makeTransportStub([
+      { content: '', usage: USAGE },
+      { content: '', usage: USAGE },
+    ])
 
     await expect(
-      runMultishot({ profile: PROFILE, persona: PERSONA, shape: SHAPE, maxTurns: 2 }),
+      runMultishot({
+        profile: PROFILE,
+        persona: PERSONA,
+        shape: SHAPE,
+        maxTurns: 2,
+        agentTransport,
+        driverTransport,
+      }),
     ).rejects.toBeInstanceOf(MultishotDriverEmptyError)
-
-    global.fetch = originalFetch
   })
 
   it('retries driver once and continues when retry produces content', async () => {
-    const originalFetch = global.fetch
-    global.fetch = makeFetchStub([
-      { content: 'agent t0' },
-      { content: '' }, // driver attempt 1 empty
-      { content: 'driver retry succeeded' },
-      { content: 'agent t1' },
-    ]) as unknown as typeof fetch
-    process.env.TANGLE_API_KEY = 'test-key'
+    const agentTransport = makeTransportStub([
+      { content: 'agent t0', usage: USAGE },
+      { content: 'agent t1', usage: USAGE },
+    ])
+    const driverTransport = makeTransportStub([
+      { content: '', usage: USAGE },
+      { content: 'driver retry succeeded', usage: USAGE },
+    ])
 
     const result = await runMultishot({
       profile: PROFILE,
       persona: PERSONA,
       shape: SHAPE,
       maxTurns: 2,
+      agentTransport,
+      driverTransport,
     })
     const driverTurns = result.transcript.filter((m) => m.role === 'user').slice(1) // skip opener
     expect(driverTurns[0].content).toBe('driver retry succeeded')
-
-    global.fetch = originalFetch
   })
 
   it('aborts cleanly when signal is set', async () => {
-    process.env.TANGLE_API_KEY = 'test-key'
     const ctl = new AbortController()
     ctl.abort()
     await expect(
@@ -181,104 +194,74 @@ describe('runMultishot', () => {
         persona: PERSONA,
         shape: SHAPE,
         maxTurns: 2,
+        agentTransport: unreachedTransport('agent'),
+        driverTransport: unreachedTransport('driver'),
         signal: ctl.signal,
       }),
     ).rejects.toThrow(/aborted/)
   })
 
   it('respects custom tools + executors', async () => {
-    const originalFetch = global.fetch
-    process.env.TANGLE_API_KEY = 'test-key'
-
     const customExecutor = vi.fn(async () => ({ content: 'custom tool result', costUsd: 0.001 }))
-
-    global.fetch = makeFetchStub([
-      { toolCalls: [{ name: 'my_custom_tool', args: { x: 1 } }] },
-      { content: 'agent after custom tool' },
-    ]) as unknown as typeof fetch
+    const agentTransport = makeTransportStub([
+      { toolCalls: [{ name: 'my_custom_tool', args: { x: 1 } }], usage: USAGE },
+      { content: 'agent after custom tool', usage: USAGE },
+    ])
 
     const result = await runMultishot({
       profile: PROFILE,
       persona: PERSONA,
       shape: SHAPE,
       maxTurns: 1,
-      tools: [
-        {
-          type: 'function',
-          function: {
-            name: 'my_custom_tool',
-            description: 'test',
-            parameters: { type: 'object', properties: {} },
-          },
-        },
-      ],
+      tools: [CUSTOM_TOOL],
       toolExecutors: { my_custom_tool: customExecutor },
       artifactTypeFor: (name) => (name === 'my_custom_tool' ? 'custom' : undefined),
+      agentTransport,
+      driverTransport: unreachedTransport('driver'),
     })
 
     expect(customExecutor).toHaveBeenCalledOnce()
     expect(result.artifacts).toHaveLength(1)
     expect(result.artifacts[0].type).toBe('custom')
     expect(result.artifacts[0].content).toBe('custom tool result')
-
-    global.fetch = originalFetch
   })
 
   it('keeps tools available across follow-up dispatch rounds', async () => {
-    const originalFetch = global.fetch
-    process.env.TANGLE_API_KEY = 'test-key'
-
     const customExecutor = vi.fn(async () => ({ content: 'custom tool result', costUsd: 0.001 }))
-    const fetchStub = makeFetchStub([
-      { toolCalls: [{ name: 'my_custom_tool', args: { x: 1 } }] },
-      { toolCalls: [{ name: 'my_custom_tool', args: { x: 2 } }] },
-      { content: 'agent after two custom tools' },
+    const agentTransport = makeTransportStub([
+      { toolCalls: [{ name: 'my_custom_tool', args: { x: 1 } }], usage: USAGE },
+      { toolCalls: [{ name: 'my_custom_tool', args: { x: 2 } }], usage: USAGE },
+      { content: 'agent after two custom tools', usage: USAGE },
     ])
-    global.fetch = fetchStub as unknown as typeof fetch
 
     const result = await runMultishot({
       profile: PROFILE,
       persona: PERSONA,
       shape: SHAPE,
       maxTurns: 1,
-      tools: [
-        {
-          type: 'function',
-          function: {
-            name: 'my_custom_tool',
-            description: 'test',
-            parameters: { type: 'object', properties: {} },
-          },
-        },
-      ],
+      tools: [CUSTOM_TOOL],
       toolExecutors: { my_custom_tool: customExecutor },
       artifactTypeFor: (name) => (name === 'my_custom_tool' ? 'custom' : undefined),
+      agentTransport,
+      driverTransport: unreachedTransport('driver'),
     })
 
     expect(customExecutor).toHaveBeenCalledTimes(2)
     expect(result.toolCalls).toBe(2)
     expect(result.artifacts).toHaveLength(2)
-    const requestBodies = fetchStub.mock.calls.map(
-      ([, init]) => JSON.parse(String(init?.body)) as Record,
-    )
-    expect(requestBodies[0]).toHaveProperty('tools')
-    expect(requestBodies[1]).toHaveProperty('tools')
-    expect(requestBodies[2]).toHaveProperty('tools')
-
-    global.fetch = originalFetch
+    expect(agentTransport).toHaveBeenCalledTimes(3)
+    for (const [req] of agentTransport.mock.calls) {
+      expect(req.tools).toEqual([CUSTOM_TOOL])
+    }
   })
 
   it('uses configured max token budgets for agent, tool follow-up, and driver calls', async () => {
-    const originalFetch = global.fetch
-    process.env.TANGLE_API_KEY = 'test-key'
-
-    const fetchStub = makeFetchStub([
-      { toolCalls: [{ name: 'my_custom_tool', args: { x: 1 } }] },
-      { content: 'agent after custom tool' },
-      { content: 'driver follow-up' },
-      { content: 'final agent answer' },
+    const agentTransport = makeTransportStub([
+      { toolCalls: [{ name: 'my_custom_tool', args: { x: 1 } }], usage: USAGE },
+      { content: 'agent after custom tool', usage: USAGE },
+      { content: 'final agent answer', usage: USAGE },
     ])
-    global.fetch = fetchStub as unknown as typeof fetch
+    const driverTransport = makeTransportStub([{ content: 'driver follow-up', usage: USAGE }])
 
     await runMultishot({
       profile: PROFILE,
@@ -288,41 +271,28 @@ describe('runMultishot', () => {
       agentMaxTokens: 111,
       toolFollowupMaxTokens: 222,
       driverMaxTokens: 333,
-      tools: [
-        {
-          type: 'function',
-          function: {
-            name: 'my_custom_tool',
-            description: 'test',
-            parameters: { type: 'object', properties: {} },
-          },
-        },
-      ],
+      tools: [CUSTOM_TOOL],
       toolExecutors: {
         my_custom_tool: async () => ({ content: 'custom tool result', costUsd: 0.001 }),
       },
+      agentTransport,
+      driverTransport,
     })
 
-    const requestBodies = fetchStub.mock.calls.map(
-      ([, init]) => JSON.parse(String(init?.body)) as Record,
-    )
-    expect(requestBodies.map((body) => body.max_tokens)).toEqual([111, 222, 333, 111])
-
-    global.fetch = originalFetch
+    expect(agentTransport.mock.calls.map(([req]) => req.maxTokens)).toEqual([111, 222, 111])
+    expect(driverTransport.mock.calls.map(([req]) => req.maxTokens)).toEqual([333])
   })
 
   it('tries driver fallback models after the primary driver returns empty twice', async () => {
-    const originalFetch = global.fetch
-    process.env.TANGLE_API_KEY = 'test-key'
-
-    const fetchStub = makeFetchStub([
-      { content: 'agent t0' },
-      { content: '' },
-      { content: '' },
-      { content: 'fallback driver response' },
-      { content: 'agent t1' },
+    const agentTransport = makeTransportStub([
+      { content: 'agent t0', usage: USAGE },
+      { content: 'agent t1', usage: USAGE },
+    ])
+    const driverTransport = makeTransportStub([
+      { content: '', usage: USAGE },
+      { content: '', usage: USAGE },
+      { content: 'fallback driver response', usage: USAGE },
     ])
-    global.fetch = fetchStub as unknown as typeof fetch
 
     const result = await runMultishot({
       profile: PROFILE,
@@ -331,77 +301,61 @@ describe('runMultishot', () => {
       maxTurns: 2,
       driverModel: 'primary-driver',
       driverFallbackModels: ['fallback-driver'],
+      agentTransport,
+      driverTransport,
     })
 
-    const requestBodies = fetchStub.mock.calls.map(
-      ([, init]) => JSON.parse(String(init?.body)) as Record,
-    )
-    expect(requestBodies.map((body) => body.model)).toEqual([
+    expect(agentTransport.mock.calls.map(([req]) => req.model)).toEqual([
       'openai/gpt-5.4',
+      'openai/gpt-5.4',
+    ])
+    expect(driverTransport.mock.calls.map(([req]) => req.model)).toEqual([
       'primary-driver',
       'primary-driver',
       'fallback-driver',
-      'openai/gpt-5.4',
     ])
     expect(
       result.transcript.some((message) => message.content === 'fallback driver response'),
     ).toBe(true)
-
-    global.fetch = originalFetch
   })
 
   it('does not send empty transcript messages to the driver after tool-only agent turns', async () => {
-    const originalFetch = global.fetch
-    process.env.TANGLE_API_KEY = 'test-key'
-
-    const fetchStub = makeFetchStub([
-      { toolCalls: [{ name: 'my_custom_tool', args: { x: 1 } }] },
-      { content: 'agent after custom tool' },
-      { content: 'driver saw the tool use and continues' },
-      { content: 'final agent answer' },
+    const agentTransport = makeTransportStub([
+      { toolCalls: [{ name: 'my_custom_tool', args: { x: 1 } }], usage: USAGE },
+      { content: 'agent after custom tool', usage: USAGE },
+      { content: 'final agent answer', usage: USAGE },
+    ])
+    const driverTransport = makeTransportStub([
+      { content: 'driver saw the tool use and continues', usage: USAGE },
     ])
-    global.fetch = fetchStub as unknown as typeof fetch
 
     await runMultishot({
       profile: PROFILE,
       persona: PERSONA,
       shape: SHAPE,
       maxTurns: 2,
-      tools: [
-        {
-          type: 'function',
-          function: {
-            name: 'my_custom_tool',
-            description: 'test',
-            parameters: { type: 'object', properties: {} },
-          },
-        },
-      ],
+      tools: [CUSTOM_TOOL],
       toolExecutors: {
         my_custom_tool: async () => ({ content: 'custom tool result', costUsd: 0.001 }),
       },
       artifactTypeFor: (name) => (name === 'my_custom_tool' ? 'custom' : undefined),
+      agentTransport,
+      driverTransport,
     })
 
-    const driverRequest = JSON.parse(String(fetchStub.mock.calls[2][1]?.body)) as {
-      messages: Array<{ role: string; content?: unknown }>
-    }
-    expect(driverRequest.messages.some((msg) => msg.content === '')).toBe(false)
-    expect(driverRequest.messages).toContainEqual({
+    const driverMessages = driverTransport.mock.calls[0][0].messages
+    expect(driverMessages.some((msg) => msg.content === '')).toBe(false)
+    expect(driverMessages).toContainEqual({
       role: 'user',
       content: 'Agent called tool: my_custom_tool.',
     })
-
-    global.fetch = originalFetch
   })
 
   it('fails loud when one assistant turn exceeds the tool dispatch cap', async () => {
-    const originalFetch = global.fetch
-    process.env.TANGLE_API_KEY = 'test-key'
-    global.fetch = makeFetchStub([
-      { toolCalls: [{ name: 'my_custom_tool', args: { x: 1 } }] },
-      { toolCalls: [{ name: 'my_custom_tool', args: { x: 2 } }] },
-    ]) as unknown as typeof fetch
+    const agentTransport = makeTransportStub([
+      { toolCalls: [{ name: 'my_custom_tool', args: { x: 1 } }], usage: USAGE },
+      { toolCalls: [{ name: 'my_custom_tool', args: { x: 2 } }], usage: USAGE },
+    ])
 
     await expect(
       runMultishot({
@@ -410,31 +364,20 @@ describe('runMultishot', () => {
         shape: SHAPE,
         maxTurns: 1,
         maxToolDispatches: 1,
-        tools: [
-          {
-            type: 'function',
-            function: {
-              name: 'my_custom_tool',
-              description: 'test',
-              parameters: { type: 'object', properties: {} },
-            },
-          },
-        ],
+        tools: [CUSTOM_TOOL],
         toolExecutors: {
           my_custom_tool: async () => ({ content: 'custom tool result', costUsd: 0.001 }),
         },
+        agentTransport,
+        driverTransport: unreachedTransport('driver'),
       }),
     ).rejects.toThrow(/tool dispatch cap exceeded/)
-
-    global.fetch = originalFetch
   })
 
   it('rethrows fatal tool errors instead of feeding them back to the agent', async () => {
-    const originalFetch = global.fetch
-    process.env.TANGLE_API_KEY = 'test-key'
-    global.fetch = makeFetchStub([
-      { toolCalls: [{ name: 'my_custom_tool', args: { x: 1 } }] },
-    ]) as unknown as typeof fetch
+    const agentTransport = makeTransportStub([
+      { toolCalls: [{ name: 'my_custom_tool', args: { x: 1 } }], usage: USAGE },
+    ])
 
     await expect(
       runMultishot({
@@ -442,100 +385,47 @@ describe('runMultishot', () => {
         persona: PERSONA,
         shape: SHAPE,
         maxTurns: 1,
-        tools: [
-          {
-            type: 'function',
-            function: {
-              name: 'my_custom_tool',
-              description: 'test',
-              parameters: { type: 'object', properties: {} },
-            },
-          },
-        ],
+        tools: [CUSTOM_TOOL],
         toolExecutors: {
           my_custom_tool: async () => {
             throw new MultishotFatalToolError('stop repeated tool loop')
           },
         },
+        agentTransport,
+        driverTransport: unreachedTransport('driver'),
       }),
     ).rejects.toBeInstanceOf(MultishotFatalToolError)
 
-    expect(global.fetch).toHaveBeenCalledTimes(1)
-    global.fetch = originalFetch
+    // The fatal error stops the turn: no follow-up call feeds the failure back.
+    expect(agentTransport).toHaveBeenCalledTimes(1)
   })
 })
 
-function makeTransportStub(
-  responses: Array<{
-    content?: string
-    toolCalls?: Array<{ name: string; args: Record }>
-    usage?: { prompt_tokens?: number; completion_tokens?: number }
-    costUsd?: number
-  }>,
-) {
-  let i = 0
-  return vi.fn(async (_req: MultishotTransportRequest): Promise => {
-    const r = responses[i++]
-    if (!r) throw new Error(`transport stub exhausted at call ${i}`)
-    return {
-      message: {
-        content: r.content ?? null,
-        ...(r.toolCalls?.length
-          ? {
-              tool_calls: r.toolCalls.map((tc, idx) => ({
-                id: `t-${i}-${idx}`,
-                type: 'function' as const,
-                function: { name: tc.name, arguments: JSON.stringify(tc.args) },
-              })),
-            }
-          : {}),
-      },
-      usage: r.usage,
-      costUsd: r.costUsd,
-    }
-  })
-}
-
-function forbiddenFetch() {
-  return vi.fn(async () => {
-    throw new Error('unexpected HTTP call — transport seam should have handled this leg')
-  }) as unknown as typeof fetch
-}
-
-describe('runMultishot transport seam', () => {
-  it('uses injected agentTransport instead of the router and meters returned costUsd', async () => {
-    const originalFetch = global.fetch
-    global.fetch = forbiddenFetch()
-    process.env.TANGLE_API_KEY = 'test-key'
-
-    const transport = makeTransportStub([{ content: 'hi from injected backend', costUsd: 0.123 }])
+describe('runMultishot cost metering', () => {
+  it('meters the cost the agent transport reports on the request it received', async () => {
+    const agentTransport = makeTransportStub([
+      { content: 'hi from injected backend', costUsd: 0.123 },
+    ])
     const result = await runMultishot({
       profile: PROFILE,
       persona: PERSONA,
       shape: SHAPE,
       maxTurns: 1,
-      agentTransport: transport,
+      agentTransport,
+      driverTransport: unreachedTransport('driver'),
     })
 
-    expect(transport).toHaveBeenCalledOnce()
-    const req = transport.mock.calls[0][0]
+    const req = agentTransport.mock.calls[0][0]
     expect(req.model).toBe('openai/gpt-5.4')
     expect(req.maxTokens).toBe(2500)
     expect(req.tools?.length).toBeGreaterThan(0)
     expect(req.messages[0]).toMatchObject({ role: 'system' })
     expect(result.transcript.at(-1)?.content).toBe('hi from injected backend')
     expect(result.costUsd).toBe(0.123)
-    expect(global.fetch).not.toHaveBeenCalled()
-
-    global.fetch = originalFetch
   })
 
-  it('meters transport usage via the router estimator when costUsd is omitted', async () => {
-    const originalFetch = global.fetch
-    global.fetch = forbiddenFetch()
-    process.env.TANGLE_API_KEY = 'test-key'
-
-    const transport = makeTransportStub([
+  it('meters transport usage through the estimator when costUsd is omitted', async () => {
+    const agentTransport = makeTransportStub([
       { content: 'usage-only', usage: { prompt_tokens: 1000, completion_tokens: 1000 } },
     ])
     const result = await runMultishot({
@@ -543,89 +433,42 @@ describe('runMultishot transport seam', () => {
       persona: PERSONA,
       shape: SHAPE,
       maxTurns: 1,
-      agentTransport: transport,
+      agentTransport,
+      driverTransport: unreachedTransport('driver'),
     })
 
     // gpt-5.4 estimator: (1000 * 0.003 + 1000 * 0.015) / 1000
     expect(result.costUsd).toBeCloseTo(0.018, 10)
-
-    global.fetch = originalFetch
   })
 
-  it('runs tool dispatch through the injected agentTransport while the driver stays on the router', async () => {
-    const originalFetch = global.fetch
-    process.env.TANGLE_API_KEY = 'test-key'
-
-    const driverFetch = makeFetchStub([{ content: 'driver pushback via router' }])
-    global.fetch = driverFetch as unknown as typeof fetch
-
+  it('sums agent, tool, and driver spend into one shot cost', async () => {
     const executor = vi.fn(async () => ({ content: 'tool output', costUsd: 0.002 }))
-    const transport = makeTransportStub([
+    const agentTransport = makeTransportStub([
       { toolCalls: [{ name: 'my_custom_tool', args: { x: 1 } }], costUsd: 0.01 },
       { content: 'agent after tool', costUsd: 0.01 },
       { content: 'final agent answer', costUsd: 0.01 },
     ])
+    const driverTransport = makeTransportStub([{ content: 'driver pushback', costUsd: 0.005 }])
 
     const result = await runMultishot({
       profile: PROFILE,
       persona: PERSONA,
       shape: SHAPE,
       maxTurns: 2,
-      agentTransport: transport,
-      tools: [
-        {
-          type: 'function',
-          function: {
-            name: 'my_custom_tool',
-            description: 'test',
-            parameters: { type: 'object', properties: {} },
-          },
-        },
-      ],
+      tools: [CUSTOM_TOOL],
       toolExecutors: { my_custom_tool: executor },
       artifactTypeFor: (name) => (name === 'my_custom_tool' ? 'custom' : undefined),
-    })
-
-    expect(transport).toHaveBeenCalledTimes(3)
-    expect(executor).toHaveBeenCalledOnce()
-    expect(result.artifacts).toHaveLength(1)
-    // Only the driver turn touches HTTP.
-    expect(driverFetch).toHaveBeenCalledTimes(1)
-    expect(result.transcript.some((m) => m.content === 'driver pushback via router')).toBe(true)
-    // 3 agent calls + tool executor cost + driver estimator cost (>0 from stub usage).
-    expect(result.costUsd).toBeGreaterThan(0.032)
-
-    global.fetch = originalFetch
-  })
-
-  it('uses injected driverTransport for the simulated user — zero HTTP with both seams', async () => {
-    const originalFetch = global.fetch
-    global.fetch = forbiddenFetch()
-    process.env.TANGLE_API_KEY = 'test-key'
-
-    const agentTransport = makeTransportStub([
-      { content: 'agent t0', costUsd: 0.01 },
-      { content: 'agent t1', costUsd: 0.01 },
-    ])
-    const driverTransport = makeTransportStub([{ content: 'driver via seam', costUsd: 0.005 }])
-
-    const result = await runMultishot({
-      profile: PROFILE,
-      persona: PERSONA,
-      shape: SHAPE,
-      maxTurns: 2,
       agentTransport,
       driverTransport,
     })
 
-    expect(driverTransport).toHaveBeenCalledOnce()
+    expect(agentTransport).toHaveBeenCalledTimes(3)
+    expect(executor).toHaveBeenCalledOnce()
     expect(driverTransport.mock.calls[0][0].model).toBe('openai/gpt-4o-mini')
-    expect(
-      result.transcript.some((m) => m.role === 'user' && m.content === 'driver via seam'),
-    ).toBe(true)
-    expect(result.costUsd).toBeCloseTo(0.025, 10)
-    expect(global.fetch).not.toHaveBeenCalled()
-
-    global.fetch = originalFetch
+    expect(result.artifacts).toHaveLength(1)
+    expect(result.transcript.some((m) => m.content === 'driver pushback')).toBe(true)
+    // 3 agent calls at 0.01, one tool executor at 0.002, one driver call at 0.005.
+    expect(result.costUsd).toBeCloseTo(0.037, 10)
+    expect(result.costProvenance?.kind).toBe('estimated')
   })
 })
diff --git a/tests/multishot/shape-defaults.test.ts b/tests/multishot/shape-defaults.test.ts
index 5b4f3d96..070b92b6 100644
--- a/tests/multishot/shape-defaults.test.ts
+++ b/tests/multishot/shape-defaults.test.ts
@@ -94,8 +94,6 @@ describe('runMultishot — pure-profile call (no shape)', () => {
       maxTurns: 2,
       agentTransport,
       driverTransport,
-      apiKey: 'test-key',
-      baseUrl: 'http://localhost:0',
     })
 
     // The derived opener is the first user message the agent sees.
diff --git a/tests/rl-rl-campaign.test.ts b/tests/rl-rl-campaign.test.ts
index 37262c7e..db02d337 100644
--- a/tests/rl-rl-campaign.test.ts
+++ b/tests/rl-rl-campaign.test.ts
@@ -1,4 +1,5 @@
 import { describe, expect, it } from 'vitest'
+import { createChatClient } from '../src/analyst/chat-client'
 import type { CampaignRunner } from '../src/eval-campaign'
 import { runRLCampaign } from '../src/rl/rl-campaign'
 import { InMemoryRawProviderSink } from '../src/trace/raw-provider-sink'
@@ -8,6 +9,19 @@ interface VariantPayload {
   prompt: string
 }
 
+const EXECUTION_REF = 'https://api.test/v1'
+
+/** The caller owns execution; every runner here emits its own spans. */
+const chatFactory = () =>
+  createChatClient({
+    transport: 'custom',
+    defaultModel: 'test-model@2026-05-08',
+    maximumAttempts: 1,
+    chat: async () => {
+      throw new Error('no RL campaign test in this file calls the model')
+    },
+  })
+
 const defaultRunner: CampaignRunner = async (ctx) => {
   await ctx.emitter.startRun({ scenarioId: ctx.scenarioId, layer: 'app-runtime' })
   const handle = await ctx.emitter.llm({
@@ -23,7 +37,7 @@ const defaultRunner: CampaignRunner = async (ctx) => {
     provider: 'test',
     model: 'test-model@2026-05-08',
     endpoint: '/chat/completions',
-    baseUrl: ctx.llmOpts.baseUrl ?? '',
+    baseUrl: EXECUTION_REF,
     attemptIndex: 0,
     direction: 'request',
     timestamp: 1_000,
@@ -55,7 +69,8 @@ describe('runRLCampaign', () => {
       ],
       scenarios: Array.from({ length: 8 }, (_, i) => ({ scenarioId: `task-${i}` })),
       seeds: [0, 1, 2],
-      llmOpts: { baseUrl: 'https://api.test/v1', apiKey: 'sk-test' },
+      chatFactory,
+      executionRef: EXECUTION_REF,
       storeFactory: () => new InMemoryTraceStore(),
       rawSinkFactory: () => new InMemoryRawProviderSink(),
       runner: defaultRunner,
@@ -110,7 +125,7 @@ describe('runRLCampaign', () => {
           provider: 'test',
           model: 'test-model@2026-05-08',
           endpoint: '/chat/completions',
-          baseUrl: ctx.llmOpts.baseUrl ?? '',
+          baseUrl: EXECUTION_REF,
           attemptIndex: 0,
           direction: 'request',
           timestamp: 1_000,
@@ -144,7 +159,8 @@ describe('runRLCampaign', () => {
         ],
         scenarios: Array.from({ length: 8 }, (_, i) => ({ scenarioId: `task-${i}` })),
         seeds: [0],
-        llmOpts: { baseUrl: 'https://api.test/v1', apiKey: 'sk-test' },
+        chatFactory,
+        executionRef: EXECUTION_REF,
         storeFactory: () => new InMemoryTraceStore(),
         rawSinkFactory: () => new InMemoryRawProviderSink(),
         runner: partialRunner(answersUpTo),
@@ -194,7 +210,8 @@ describe('runRLCampaign', () => {
       ],
       scenarios: Array.from({ length: 4 }, (_, i) => ({ scenarioId: `s-${i}` })),
       seeds: [0, 1],
-      llmOpts: { baseUrl: 'https://api.test/v1', apiKey: 'sk-test' },
+      chatFactory,
+      executionRef: EXECUTION_REF,
       storeFactory: () => new InMemoryTraceStore(),
       rawSinkFactory: () => new InMemoryRawProviderSink(),
       runner: defaultRunner,
@@ -234,7 +251,8 @@ describe('runRLCampaign', () => {
       ],
       scenarios: [{ scenarioId: 's-0' }],
       seeds: [0],
-      llmOpts: { baseUrl: 'https://api.test/v1', apiKey: 'sk-test' },
+      chatFactory,
+      executionRef: EXECUTION_REF,
       storeFactory: () => new InMemoryTraceStore(),
       rawSinkFactory: () => new InMemoryRawProviderSink(),
       runner,
@@ -258,7 +276,8 @@ describe('runRLCampaign', () => {
       variants: [{ id: 'only', payload: { prompt: 'x' } }],
       scenarios: [{ scenarioId: 's' }],
       seeds: [0],
-      llmOpts: { baseUrl: 'https://api.test/v1', apiKey: 'sk-test' },
+      chatFactory,
+      executionRef: EXECUTION_REF,
       storeFactory: () => new InMemoryTraceStore(),
       rawSinkFactory: () => new InMemoryRawProviderSink(),
       runner: defaultRunner,
@@ -279,7 +298,8 @@ describe('runRLCampaign', () => {
       seeds: [0],
       splitTag: 'holdout',
       preferences: { allowHeldOutTrainingData: true },
-      llmOpts: { baseUrl: 'https://api.test/v1', apiKey: 'sk-test' },
+      chatFactory,
+      executionRef: EXECUTION_REF,
       storeFactory: () => new InMemoryTraceStore(),
       rawSinkFactory: () => new InMemoryRawProviderSink(),
       runner: defaultRunner,
diff --git a/tests/wire/handlers.test.ts b/tests/wire/handlers.test.ts
index 0ce7a3c5..b7ac72a2 100644
--- a/tests/wire/handlers.test.ts
+++ b/tests/wire/handlers.test.ts
@@ -1,37 +1,27 @@
-/** `callLlmJson` is mocked: this file tests the wire handler's request
- *  validation, dispatch, and error mapping, which a real provider call would
- *  make non-deterministic and paid. Judge behaviour is tested against real
- *  transports elsewhere. */
-import { describe, expect, it, vi } from 'vitest'
+import { describe, expect, it } from 'vitest'
 
-const llmMock = vi.hoisted(() => ({
-  value: {
-    dimensions: { quality: 0.8 },
-    failureModes: [],
-    wins: [],
-    rationale: 'Clear enough.',
-  } as unknown,
-}))
+import { type ChatClient, createChatClient } from '../../src/analyst/chat-client'
+import { CostLedger } from '../../src/cost-ledger'
+import { handleJudge, type WireError } from '../../src/wire/handlers'
+import type { Rubric } from '../../src/wire/schemas'
 
-vi.mock('../../src/llm-client', async (importOriginal) => ({
-  ...(await importOriginal()),
-  callLlmJson: vi.fn(async () => ({
-    value: llmMock.value,
-    result: {
-      model: 'gpt-4o',
-      content: JSON.stringify(llmMock.value),
-      usage: { promptTokens: 10, completionTokens: 5, totalTokens: 15 },
+/** Caller-owned transport: the judge endpoint issues no provider request itself. */
+function answering(value: unknown): ChatClient {
+  return createChatClient({
+    transport: 'custom',
+    defaultModel: 'judge-model',
+    maximumAttempts: 1,
+    chat: async () => ({
+      content: JSON.stringify(value),
+      usage: { promptTokens: 10, completionTokens: 5, totalTokens: 15, captured: true },
       costUsd: 0.001,
-      finishReason: 'stop',
+      model: 'judge-model',
+      servedModel: 'judge-model',
       durationMs: 1,
       raw: {},
-    },
-  })),
-}))
-
-import { CostLedger } from '../../src/cost-ledger'
-import { handleJudge, type WireError } from '../../src/wire/handlers'
-import type { Rubric } from '../../src/wire/schemas'
+    }),
+  })
+}
 
 const rubric: Rubric = {
   name: 'test-rubric',
@@ -44,15 +34,19 @@ const rubric: Rubric = {
 
 describe('handleJudge output validation', () => {
   it('returns validated judge output', async () => {
-    llmMock.value = {
-      dimensions: { quality: 0.8 },
-      failureModes: ['bad'],
-      wins: ['good'],
-      rationale: 'Clear enough.',
-    }
-
     const costLedger = new CostLedger()
-    const result = await handleJudge({ rubric, content: 'hello' }, { costLedger })
+    const result = await handleJudge(
+      { rubric, content: 'hello' },
+      {
+        chat: answering({
+          dimensions: { quality: 0.8 },
+          failureModes: ['bad'],
+          wins: ['good'],
+          rationale: 'Clear enough.',
+        }),
+        costLedger,
+      },
+    )
 
     expect(result.composite).toBe(0.8)
     expect(result.failureModes).toEqual(['bad'])
@@ -63,13 +57,19 @@ describe('handleJudge output validation', () => {
     ])
   })
 
+  it('refuses when no ChatClient is configured', async () => {
+    await expect(handleJudge({ rubric, content: 'hello' })).rejects.toMatchObject<
+      Partial
+    >({
+      code: 'llm_not_configured',
+      status: 503,
+    })
+  })
+
   it('rejects malformed dimension scores before returning wire output', async () => {
-    llmMock.value = {
-      dimensions: { quality: Number.NaN },
-      rationale: 'nope',
-    }
+    const chat = answering({ dimensions: { quality: Number.NaN }, rationale: 'nope' })
 
-    await expect(handleJudge({ rubric, content: 'hello' })).rejects.toMatchObject<
+    await expect(handleJudge({ rubric, content: 'hello' }, { chat })).rejects.toMatchObject<
       Partial
     >({
       code: 'judge_error',
@@ -78,29 +78,32 @@ describe('handleJudge output validation', () => {
   })
 
   it('rejects unknown failure and win ids', async () => {
-    llmMock.value = {
+    const unknownFailure = answering({
       dimensions: { quality: 0.7 },
       failureModes: ['unknown-failure'],
       wins: [],
       rationale: 'bad id',
-    }
-    await expect(handleJudge({ rubric, content: 'hello' })).rejects.toThrow(/unknown failureModes/)
+    })
+    await expect(
+      handleJudge({ rubric, content: 'hello' }, { chat: unknownFailure }),
+    ).rejects.toThrow(/unknown failureModes/)
 
-    llmMock.value = {
+    const unknownWin = answering({
       dimensions: { quality: 0.7 },
       failureModes: [],
       wins: ['unknown-win'],
       rationale: 'bad id',
-    }
-    await expect(handleJudge({ rubric, content: 'hello' })).rejects.toThrow(/unknown wins/)
+    })
+    await expect(handleJudge({ rubric, content: 'hello' }, { chat: unknownWin })).rejects.toThrow(
+      /unknown wins/,
+    )
   })
 
   it('rejects missing rationale', async () => {
-    llmMock.value = {
-      dimensions: { quality: 0.7 },
-      rationale: '',
-    }
+    const chat = answering({ dimensions: { quality: 0.7 }, rationale: '' })
 
-    await expect(handleJudge({ rubric, content: 'hello' })).rejects.toThrow(/missing rationale/)
+    await expect(handleJudge({ rubric, content: 'hello' }, { chat })).rejects.toThrow(
+      /missing rationale/,
+    )
   })
 })
diff --git a/tests/wire/rpc.test.ts b/tests/wire/rpc.test.ts
index d6e5343c..0250c9c7 100644
--- a/tests/wire/rpc.test.ts
+++ b/tests/wire/rpc.test.ts
@@ -7,6 +7,7 @@
  */
 import { describe, expect, it, vi } from 'vitest'
 
+import { createChatClient } from '../../src/analyst/chat-client'
 import { dispatchRpc } from '../../src/wire/rpc'
 
 describe('dispatchRpc', () => {
@@ -44,52 +45,39 @@ describe('dispatchRpc', () => {
     }
   })
 
-  it('refuses CLI judge calls before a provider endpoint is configured', async () => {
-    const out = await dispatchRpc(
-      {
-        method: 'judge',
-        params: { rubricName: 'anti-slop', content: 'hello' },
-      },
-      { llmRouteRequirements: { requireExplicitBaseUrl: true } },
-    )
+  it('refuses CLI judge calls before a model transport is configured', async () => {
+    const out = await dispatchRpc({
+      method: 'judge',
+      params: { rubricName: 'anti-slop', content: 'hello' },
+    })
 
     expect(out).toEqual({
       error: {
         code: 'llm_not_configured',
         message:
-          'No model endpoint is configured. Pass llm.baseUrl or configure the CLI provider environment variables.',
-        details: { reason: 'no_explicit_base_url' },
+          'No model transport is configured. Pass a ChatClient, or configure the CLI provider environment variables.',
       },
     })
   })
 
-  it('forwards provider config and the default model to judge calls', async () => {
-    const fetch = vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {
-      const request = JSON.parse(String(init?.body)) as { model: string }
-      expect(request.model).toBe('configured-model')
-      const headers = init?.headers as Record | undefined
-      expect(headers?.Authorization).toBe('Bearer provider-key')
-      return new Response(
-        JSON.stringify({
-          model: 'configured-model',
-          choices: [
-            {
-              message: {
-                content: JSON.stringify({
-                  dimensions: { quality: 0.75 },
-                  failureModes: [],
-                  wins: [],
-                  rationale: 'Clear.',
-                }),
-              },
-              finish_reason: 'stop',
-            },
-          ],
-          usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 },
+  it("forwards the caller's ChatClient and the default model to judge calls", async () => {
+    const chat = vi.fn(async (req: { model?: string }) => {
+      expect(req.model).toBe('configured-model')
+      return {
+        content: JSON.stringify({
+          dimensions: { quality: 0.75 },
+          failureModes: [],
+          wins: [],
+          rationale: 'Clear.',
         }),
-        { status: 200, headers: { 'content-type': 'application/json' } },
-      )
-    }) as typeof globalThis.fetch
+        usage: { promptTokens: 10, completionTokens: 5, totalTokens: 15, captured: true },
+        costUsd: null,
+        model: 'configured-model',
+        servedModel: 'configured-model',
+        durationMs: 1,
+        raw: {},
+      }
+    })
 
     const out = await dispatchRpc(
       {
@@ -107,13 +95,12 @@ describe('dispatchRpc', () => {
         },
       },
       {
-        llm: { baseUrl: 'https://provider.example/v1', apiKey: 'provider-key', fetch },
+        chat: createChatClient({ transport: 'custom', maximumAttempts: 1, chat }),
         judgeModel: 'configured-model',
-        llmRouteRequirements: { requireExplicitBaseUrl: true },
       },
     )
 
-    expect(fetch).toHaveBeenCalledOnce()
+    expect(chat).toHaveBeenCalledOnce()
     expect(out).toMatchObject({
       result: { composite: 0.75, model: 'configured-model' },
     })

From 09fd67df0ab69793a035dda29522ec88bdf8ab6c Mon Sep 17 00:00:00 2001
From: drewstone 
Date: Thu, 20 Aug 2026 22:45:00 -0700
Subject: [PATCH 2/4] refactor(execution): require an explicit endpoint in the
 internal transport

---
 CHANGELOG.md                            |   1 +
 src/analyst/benchmark-implementation.ts |   2 +-
 src/integrity/single-backend.ts         |   6 +-
 src/llm-client.test.ts                  | 110 +++++++++++++++++-------
 src/llm-client.ts                       |  28 ++++--
 5 files changed, 107 insertions(+), 40 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 674b1ad2..6b0f42a3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,7 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-
   - `multishot/router.ts` is deleted with `routerCompletion`, `requireRouterApiKey`, and `defaultRouterBaseUrl`. `runMultishot`, `runMultishotMatrix`, and `runJudge` now require a caller-supplied `MultishotTransport`; `JudgeConfig.transport` is required and `JUDGE_MODEL` is no longer read from the environment. `MultishotToolExecutor` receives `{ transport, signal }` instead of `{ apiKey, baseUrl, signal }`, and the optional `toolTransport` names the leg the built-in delegate tools run on. `estimateRouterCost` is now `estimateMultishotCost` in `multishot/cost.ts`.
   - `preflightModels` and `assertModelsServed` take `request: ModelEndpointRequest` instead of `baseUrl` and `apiKey`. Agent Eval asks for a `list-models` or a `probe` check and reads the `Response`, so status, the provider's own `error.message`, `budgetExhausted`, and served-model substitution stay exactly as detectable as before.
   - `runIntentMatchJudge`, `runSemanticConceptJudge`, `handleJudge`, `dispatchRpc`, and `createApp` take `chat: ChatClient` (plus optional `pricing`) instead of `llm: LlmClientOptions`. `/v1/judge` refuses with `llm_not_configured` (503) when no transport is configured, which replaces the old route assertion.
+  - The internal OpenAI-compatible client has no default endpoint. `DEFAULT_BASE_URL = 'https://router.tangle.tools/v1'` is deleted and `baseUrl` is required, so a misconfigured caller fails loudly instead of silently billing the public router — the failure `assertLlmRoute` existed to catch, now unrepresentable.
   - `runEvalCampaign` takes `chatFactory: (wiring: CampaignChatWiring) => ChatClient` instead of `llmOpts`, and `CampaignRunContext.chat` replaces `ctx.llmOpts`. The campaign passes each run's `rawSink` and `runId` into the factory, so a transport that binds them still satisfies `assertRunCaptured`'s raw-coverage check. The campaign fingerprint now folds a caller-declared `executionRef` where it previously folded the base URL and provider it can no longer see.
 
 ### Added
diff --git a/src/analyst/benchmark-implementation.ts b/src/analyst/benchmark-implementation.ts
index db620474..324c4fa2 100644
--- a/src/analyst/benchmark-implementation.ts
+++ b/src/analyst/benchmark-implementation.ts
@@ -137,7 +137,7 @@ export const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([
 ])
 
 export const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 =
-  'd04c601555d7de691495312c69a7b8584467ae03d841f91e80a1530705ccf69d'
+  '8d215ad631cfee4e5691bb6291135fafc29e8549243d5debd0062289a21ae0a5'
 
 export function analystBenchmarkImplementationDigest() {
   return ANALYST_BENCHMARK_IMPLEMENTATION_SHA256
diff --git a/src/integrity/single-backend.ts b/src/integrity/single-backend.ts
index 7d9130ee..e7f39336 100644
--- a/src/integrity/single-backend.ts
+++ b/src/integrity/single-backend.ts
@@ -3,9 +3,9 @@
  * SAME backend config, so the judge can't silently re-route through a
  * different (often paid) backend than the agent.
  *
- * The bug class: `--backend cli-bridge` rewires the agent, but the judge still
- * reads `process.env.TANGLE_API_KEY` → router. Cost is billed against the
- * router, the eval reports the cli-bridge model, and the data is unusable.
+ * The bug class: a consumer rewires the agent onto one backend but leaves the
+ * judge bound to another. Cost is billed against the second backend, the eval
+ * reports the first backend's model, and the data is unusable.
  * Four consumers hand-roll this comparison (legal at `canonical.ts:702-795`);
  * this is the one substrate copy.
  *
diff --git a/src/llm-client.test.ts b/src/llm-client.test.ts
index 16d7c520..42327d0c 100644
--- a/src/llm-client.test.ts
+++ b/src/llm-client.test.ts
@@ -17,6 +17,9 @@ import {
 } from './llm-client'
 import { InMemoryRawProviderSink } from './trace/raw-provider-sink'
 
+/** The transport has no default endpoint; every call names one. */
+const TEST_BASE_URL = 'https://provider.test/v1'
+
 describe('maximumChargeForLlmRequest', () => {
   it('bounds the exact text request and its enforced output limit', () => {
     const maximum = maximumChargeForLlmRequest(
@@ -155,6 +158,7 @@ describe('costReceiptFromLlm', () => {
     const result = await callLlm(
       { model: 'gpt-4o', messages: [{ role: 'user', content: 'hello' }], maxTokens: 8 },
       {
+        baseUrl: TEST_BASE_URL,
         maximumAttempts: 1,
         customTokenPricing: {
           inputUsdPerMillion: 0.27,
@@ -191,6 +195,7 @@ describe('costReceiptFromLlm', () => {
         maxTokens: 8,
       },
       {
+        baseUrl: TEST_BASE_URL,
         maximumAttempts: 1,
         customTokenPricing: {
           inputUsdPerMillion: 0.27,
@@ -224,6 +229,7 @@ describe('costReceiptFromLlm', () => {
         maxTokens: 8,
       },
       {
+        baseUrl: TEST_BASE_URL,
         maximumAttempts: 1,
         customTokenPricing: {
           inputUsdPerMillion: 1,
@@ -258,6 +264,7 @@ describe('costReceiptFromLlm', () => {
     const result = await callLlm(
       { model: 'glm-4.5', messages: [{ role: 'user', content: 'test' }], maxTokens: 3_323 },
       {
+        baseUrl: TEST_BASE_URL,
         maximumAttempts: 1,
         fetch: async () =>
           mkOkResponse({
@@ -318,6 +325,7 @@ describe('costReceiptFromLlm', () => {
     const result = await callLlm(
       { model: 'gpt-4o', messages: [{ role: 'user', content: 'hello' }], maxTokens: 8 },
       {
+        baseUrl: TEST_BASE_URL,
         maximumAttempts: 1,
         fetch: async () =>
           mkOkResponse({
@@ -341,6 +349,7 @@ describe('costReceiptFromLlm', () => {
     const result = await callLlm(
       { model: 'gpt-4o', messages: [{ role: 'user', content: 'hello' }], maxTokens: 8 },
       {
+        baseUrl: TEST_BASE_URL,
         maximumAttempts: 1,
         fetch: async () =>
           mkOkResponse({
@@ -364,6 +373,7 @@ describe('costReceiptFromLlm', () => {
     const result = await callLlm(
       { model: 'gpt-4o', messages: [{ role: 'user', content: 'hello' }], maxTokens: 8 },
       {
+        baseUrl: TEST_BASE_URL,
         maximumAttempts: 1,
         fetch: async () =>
           mkOkResponse({
@@ -385,6 +395,7 @@ describe('costReceiptFromLlm', () => {
     const result = await callLlm(
       { model: 'gpt-4o', messages: [{ role: 'user', content: 'hello' }], maxTokens: 8 },
       {
+        baseUrl: TEST_BASE_URL,
         maximumAttempts: 1,
         fetch: async () =>
           mkOkResponse({
@@ -703,6 +714,7 @@ describe('llm-client — callLlm happy path', () => {
         thinking: 'enabled',
       },
       {
+        baseUrl: TEST_BASE_URL,
         fetch: fetch as unknown as typeof globalThis.fetch,
         thinking: 'disabled',
       },
@@ -719,7 +731,7 @@ describe('llm-client — callLlm happy path', () => {
     )
     await callLlm(
       { model: 'glm-5.2', messages: [{ role: 'user', content: 'x' }] },
-      { fetch: fetch as unknown as typeof globalThis.fetch },
+      { baseUrl: TEST_BASE_URL, fetch: fetch as unknown as typeof globalThis.fetch },
     )
 
     const call = (fetch.mock.calls[0] ?? []) as unknown as [string, RequestInit]
@@ -732,6 +744,7 @@ describe('llm-client — callLlm happy path', () => {
     await callLlm(
       { model: 'm', messages: [] },
       {
+        baseUrl: TEST_BASE_URL,
         fetch: fetch as unknown as typeof globalThis.fetch,
         apiKey: 'ignored',
         authHeader: { name: 'X-Custom-Auth', value: 'token-123' },
@@ -759,7 +772,7 @@ describe('llm-client — retry semantics', () => {
     ])
     const r = await callLlm(
       { model: 'm', messages: [{ role: 'user', content: 'x' }] },
-      { fetch, maximumAttempts: 3 },
+      { baseUrl: TEST_BASE_URL, fetch, maximumAttempts: 3 },
     )
     expect(r.content).toBe('ok')
     expect(calls).toEqual([429, 200])
@@ -772,7 +785,7 @@ describe('llm-client — retry semantics', () => {
     ])
     const r = await callLlm(
       { model: 'm', messages: [{ role: 'user', content: 'x' }] },
-      { fetch, maximumAttempts: 3 },
+      { baseUrl: TEST_BASE_URL, fetch, maximumAttempts: 3 },
     )
     expect(r.content).toBe('ok')
   })
@@ -782,7 +795,11 @@ describe('llm-client — retry semantics', () => {
     await expect(
       callLlm(
         { model: 'm', messages: [] },
-        { fetch: fetch as unknown as typeof globalThis.fetch, maximumAttempts: 3 },
+        {
+          baseUrl: TEST_BASE_URL,
+          fetch: fetch as unknown as typeof globalThis.fetch,
+          maximumAttempts: 3,
+        },
       ),
     ).rejects.toBeInstanceOf(LlmCallError)
     expect(fetch).toHaveBeenCalledOnce()
@@ -807,7 +824,7 @@ describe('llm-client — retry semantics', () => {
         messages: [{ role: 'user', content: 'x' }],
         temperature: 0.2,
       },
-      { fetch, maximumAttempts: 2 },
+      { baseUrl: TEST_BASE_URL, fetch, maximumAttempts: 2 },
     )
 
     expect(result.content).toBe('ok')
@@ -824,7 +841,11 @@ describe('llm-client — retry semantics', () => {
     await expect(
       callLlm(
         { model: 'm', messages: [], temperature: 3 },
-        { fetch: fetch as unknown as typeof globalThis.fetch, maximumAttempts: 2 },
+        {
+          baseUrl: TEST_BASE_URL,
+          fetch: fetch as unknown as typeof globalThis.fetch,
+          maximumAttempts: 2,
+        },
       ),
     ).rejects.toBeInstanceOf(LlmCallError)
     expect(fetch).toHaveBeenCalledOnce()
@@ -835,7 +856,11 @@ describe('llm-client — retry semantics', () => {
     await expect(
       callLlm(
         { model: 'm', messages: [] },
-        { fetch: fetch as unknown as typeof globalThis.fetch, maximumAttempts: 2 },
+        {
+          baseUrl: TEST_BASE_URL,
+          fetch: fetch as unknown as typeof globalThis.fetch,
+          maximumAttempts: 2,
+        },
       ),
     ).rejects.toBeInstanceOf(LlmCallError)
     expect(fetch).toHaveBeenCalledTimes(2)
@@ -852,7 +877,10 @@ describe('llm-client — retry semantics', () => {
       }
       return mkOkResponse({ choices: [{ message: { content: 'recovered' } }], usage: {} })
     }) as unknown as typeof globalThis.fetch
-    const r = await callLlm({ model: 'm', messages: [] }, { fetch, maximumAttempts: 3 })
+    const r = await callLlm(
+      { model: 'm', messages: [] },
+      { baseUrl: TEST_BASE_URL, fetch, maximumAttempts: 3 },
+    )
     expect(r.content).toBe('recovered')
   })
 
@@ -870,7 +898,10 @@ describe('llm-client — retry semantics', () => {
       }
       return mkOkResponse({ choices: [{ message: { content: 'recovered' } }], usage: {} })
     }) as unknown as typeof globalThis.fetch
-    const r = await callLlm({ model: 'm', messages: [] }, { fetch, maximumAttempts: 3 })
+    const r = await callLlm(
+      { model: 'm', messages: [] },
+      { baseUrl: TEST_BASE_URL, fetch, maximumAttempts: 3 },
+    )
     expect(r.content).toBe('recovered')
     expect(call).toBe(2)
   })
@@ -884,7 +915,10 @@ describe('llm-client — caller AbortSignal + cross-attempt deadline', () => {
     const controller = new AbortController()
     controller.abort()
     await expect(
-      callLlm({ model: 'm', messages: [] }, { fetch, signal: controller.signal }),
+      callLlm(
+        { model: 'm', messages: [] },
+        { baseUrl: TEST_BASE_URL, fetch, signal: controller.signal },
+      ),
     ).rejects.toThrow(/abort/i)
     expect(fetch as unknown as ReturnType).not.toHaveBeenCalled()
   })
@@ -908,7 +942,7 @@ describe('llm-client — caller AbortSignal + cross-attempt deadline', () => {
     await expect(
       callLlm(
         { model: 'm', messages: [] },
-        { fetch, signal: controller.signal, maximumAttempts: 3 },
+        { baseUrl: TEST_BASE_URL, fetch, signal: controller.signal, maximumAttempts: 3 },
       ),
     ).rejects.toThrow(/abort/i)
     expect(calls).toBe(1)
@@ -925,7 +959,10 @@ describe('llm-client — caller AbortSignal + cross-attempt deadline', () => {
         })
       })) as unknown as typeof globalThis.fetch
 
-    const p = callLlm({ model: 'm', messages: [] }, { fetch, signal: controller.signal })
+    const p = callLlm(
+      { model: 'm', messages: [] },
+      { baseUrl: TEST_BASE_URL, fetch, signal: controller.signal },
+    )
     controller.abort()
     await expect(p).rejects.toThrow(/abort/i)
   })
@@ -942,7 +979,10 @@ describe('llm-client — caller AbortSignal + cross-attempt deadline', () => {
     }) as unknown as typeof globalThis.fetch
 
     await expect(
-      callLlm({ model: 'm', messages: [] }, { fetch, maximumAttempts: 5, deadlineMs: 10 }),
+      callLlm(
+        { model: 'm', messages: [] },
+        { baseUrl: TEST_BASE_URL, fetch, maximumAttempts: 5, deadlineMs: 10 },
+      ),
     ).rejects.toBeInstanceOf(LlmCallError)
     // Without the deadline this would retry up to 5 times; the budget caps it at 1.
     expect(calls).toBe(1)
@@ -958,7 +998,7 @@ describe('llm-client — empty-content + finishReason signals', () => {
           usage: {},
         }),
     ])
-    const r = await callLlm({ model: 'm', messages: [] }, { fetch })
+    const r = await callLlm({ model: 'm', messages: [] }, { baseUrl: TEST_BASE_URL, fetch })
     expect(r.content).toBe('')
     expect(r.contentEmpty).toBe(true)
     expect(r.finishReason).toBe('length')
@@ -972,7 +1012,7 @@ describe('llm-client — empty-content + finishReason signals', () => {
           usage: {},
         }),
     ])
-    const r = await callLlm({ model: 'm', messages: [] }, { fetch })
+    const r = await callLlm({ model: 'm', messages: [] }, { baseUrl: TEST_BASE_URL, fetch })
     expect(r.contentEmpty).toBe(false)
     expect(r.finishReason).toBe('stop')
   })
@@ -981,7 +1021,7 @@ describe('llm-client — empty-content + finishReason signals', () => {
     const fetch = mockFetch([
       async () => mkOkResponse({ choices: [{ message: { content: '   \n ' } }], usage: {} }),
     ])
-    const r = await callLlm({ model: 'm', messages: [] }, { fetch })
+    const r = await callLlm({ model: 'm', messages: [] }, { baseUrl: TEST_BASE_URL, fetch })
     expect(r.contentEmpty).toBe(true)
     expect(r.finishReason).toBeNull()
   })
@@ -1039,7 +1079,7 @@ describe('llm-client — callLlmJson + schema degrade', () => {
     ])
     const { value } = await callLlmJson<{ foo: number }>(
       { model: 'm', messages: [{ role: 'user', content: 'x' }] },
-      { fetch },
+      { baseUrl: TEST_BASE_URL, fetch },
     )
     expect(value.foo).toBe(42)
   })
@@ -1060,7 +1100,7 @@ describe('llm-client — callLlmJson + schema degrade', () => {
         messages: [{ role: 'user', content: 'x' }],
         jsonSchema: { name: 's', schema: { type: 'object' } },
       },
-      { fetch },
+      { baseUrl: TEST_BASE_URL, fetch },
     )
 
     expect(value.ok).toBe(true)
@@ -1085,7 +1125,7 @@ describe('llm-client — callLlmJson + schema degrade', () => {
         messages: [{ role: 'user', content: 'Return the required JSON.' }],
         jsonSchema: { name: 's', schema: { type: 'object' } },
       },
-      { fetch, jsonSchemaTransport: 'json-object' },
+      { baseUrl: TEST_BASE_URL, fetch, jsonSchemaTransport: 'json-object' },
     )
 
     expect(value.ok).toBe(true)
@@ -1102,7 +1142,7 @@ describe('llm-client — callLlmJson + schema degrade', () => {
           messages: [],
           jsonSchema: { name: 's', schema: { type: 'object' } },
         },
-        { fetch: fetch as unknown as typeof globalThis.fetch },
+        { baseUrl: TEST_BASE_URL, fetch: fetch as unknown as typeof globalThis.fetch },
       ),
     ).rejects.toBeInstanceOf(LlmCallError)
     expect(fetch).toHaveBeenCalledOnce()
@@ -1114,7 +1154,10 @@ describe('llm-client — callLlmJson + schema degrade', () => {
         mkOkResponse({ choices: [{ message: { content: 'not json at all' } }], usage: {} }),
     ])
     await expect(
-      callLlmJson({ model: 'm', messages: [{ role: 'user', content: 'x' }] }, { fetch }),
+      callLlmJson(
+        { model: 'm', messages: [{ role: 'user', content: 'x' }] },
+        { baseUrl: TEST_BASE_URL, fetch },
+      ),
     ).rejects.toThrow(/non-JSON/)
   })
 
@@ -1129,7 +1172,10 @@ describe('llm-client — callLlmJson + schema degrade', () => {
     ])
 
     try {
-      await callLlmJson({ model: 'm', messages: [{ role: 'user', content: 'x' }] }, { fetch })
+      await callLlmJson(
+        { model: 'm', messages: [{ role: 'user', content: 'x' }] },
+        { baseUrl: TEST_BASE_URL, fetch },
+      )
       throw new Error('expected malformed JSON to fail')
     } catch (error) {
       expect(error).toBeInstanceOf(LlmResponseError)
@@ -1147,7 +1193,10 @@ describe('llm-client — callLlmJson + schema degrade', () => {
         }),
     ])
     await expect(
-      callLlmJson({ model: 'm', messages: [{ role: 'user', content: 'x' }] }, { fetch }),
+      callLlmJson(
+        { model: 'm', messages: [{ role: 'user', content: 'x' }] },
+        { baseUrl: TEST_BASE_URL, fetch },
+      ),
     ).rejects.toThrow(/non-JSON/)
   })
 
@@ -1160,7 +1209,10 @@ describe('llm-client — callLlmJson + schema degrade', () => {
         }),
     ])
     await expect(
-      callLlmJson({ model: 'm', messages: [{ role: 'user', content: 'x' }] }, { fetch }),
+      callLlmJson(
+        { model: 'm', messages: [{ role: 'user', content: 'x' }] },
+        { baseUrl: TEST_BASE_URL, fetch },
+      ),
     ).rejects.toThrow(/truncated JSON content.*finishReason=length/)
   })
 
@@ -1174,7 +1226,7 @@ describe('llm-client — callLlmJson + schema degrade', () => {
     ])
     const { value } = await callLlmJson<{ wrapped: boolean }>(
       { model: 'm', messages: [{ role: 'user', content: 'x' }] },
-      { fetch },
+      { baseUrl: TEST_BASE_URL, fetch },
     )
     expect(value.wrapped).toBe(true)
   })
@@ -1189,7 +1241,7 @@ describe('llm-client — callLlmJson + schema degrade', () => {
     ])
     const { value } = await callLlmJson<{ wrapped: boolean }>(
       { model: 'm', messages: [{ role: 'user', content: 'x' }] },
-      { fetch },
+      { baseUrl: TEST_BASE_URL, fetch },
     )
     expect(value.wrapped).toBe(true)
   })
@@ -1201,7 +1253,7 @@ describe('llm-client — callLlmJson + schema degrade', () => {
     ])
     const { value } = await callLlmJson<{ wrapped: boolean }>(
       { model: 'm', messages: [{ role: 'user', content: 'x' }] },
-      { fetch: exactFetch, jsonPayloadMode: 'exact' },
+      { baseUrl: TEST_BASE_URL, fetch: exactFetch, jsonPayloadMode: 'exact' },
     )
     expect(value.wrapped).toBe(true)
 
@@ -1217,7 +1269,7 @@ describe('llm-client — callLlmJson + schema degrade', () => {
       await expect(
         callLlmJson(
           { model: 'm', messages: [{ role: 'user', content: 'x' }] },
-          { fetch, jsonPayloadMode: 'exact' },
+          { baseUrl: TEST_BASE_URL, fetch, jsonPayloadMode: 'exact' },
         ),
       ).rejects.toBeInstanceOf(LlmResponseError)
     }
@@ -1229,7 +1281,7 @@ describe('llm-client — LlmClient wrapper', () => {
     const fetch = vi.fn(async () =>
       mkOkResponse({ choices: [{ message: { content: 'x' } }], usage: {} }),
     ) as unknown as typeof globalThis.fetch
-    const client = new LlmClient({ fetch, apiKey: 'default' })
+    const client = new LlmClient({ baseUrl: TEST_BASE_URL, fetch, apiKey: 'default' })
     await client.call({ model: 'm', messages: [] }, { apiKey: 'override' })
     const call = ((fetch as unknown as ReturnType).mock.calls[0] ??
       []) as unknown as [string, RequestInit]
diff --git a/src/llm-client.ts b/src/llm-client.ts
index 9e0b43ec..d11a3586 100644
--- a/src/llm-client.ts
+++ b/src/llm-client.ts
@@ -1,5 +1,18 @@
 /**
- * LLM client with graceful degrade.
+ * INTERNAL OpenAI-compatible client. NOT part of the published surface.
+ *
+ * agent-eval executes no paid model on behalf of a consumer: `createChatClient`
+ * binds a caller-supplied transport, and this module is not reachable through
+ * any export subpath. Two in-repo callers hold it:
+ *   - the `agent-eval` binary (`src/cli-config.ts`), a deployed server whose
+ *     caller is a JSON-RPC client in another language and which therefore
+ *     configures its own endpoint from its own environment;
+ *   - the loopback optimizer proxy path (`src/analyst/benchmark-public-model.ts`,
+ *     `src/analyst/dspy-rlm-engine.ts`), which targets `http://127.0.0.1:/v1`
+ *     with an ephemeral token and executes nothing itself.
+ *
+ * The canonical request/result TYPES it defines ARE public — they are the
+ * contract every caller-owned transport speaks.
  *
  * OpenAI-compatible `/v1/chat/completions` client with:
  *   - Exponential-backoff retry on 429 + 5xx gateway errors (502/503/504).
@@ -13,11 +26,9 @@
  * Usage:
  *   const { value, result } = await callLlmJson(
  *     { model: 'gpt-4o', messages: [...], jsonSchema: { name: 'x', schema: {...} } },
- *     { baseUrl: 'https://router.tangle.tools/v1', apiKey: process.env.KEY },
+ *     { baseUrl: process.env.AGENT_EVAL_LLM_BASE_URL, apiKey: process.env.AGENT_EVAL_LLM_API_KEY },
  *   )
  *
- * `createChatClient` wraps this implementation for provider-neutral package
- * entry points. Direct callers can use `callLlm` or `callLlmJson`.
  */
 
 import {
@@ -318,7 +329,7 @@ export class LlmResponseError extends AgentEvalError {
 }
 
 export interface LlmClientOptions extends LlmChargeBounds {
-  /** Base URL (without trailing slash). Must end at the `/v1` prefix. */
+  /** Base URL (without trailing slash), ending at the `/v1` prefix. Required: there is no default endpoint. */
   baseUrl?: string
   /** Bearer token — either `apiKey` or `bearer` populates `Authorization: Bearer ...`. */
   apiKey?: string
@@ -383,7 +394,6 @@ export interface LlmClientOptions extends LlmChargeBounds {
 
 // ─── Internals ──────────────────────────────────────────────────────────
 
-const DEFAULT_BASE_URL = 'https://router.tangle.tools/v1'
 // Flagship / reasoning models routinely take several minutes on large prompts (a
 // reflection over many failures, a long tool transcript). A tight cap aborts a
 // legitimately-slow but healthy call — and because every retry attempt re-uses
@@ -785,7 +795,11 @@ export async function callLlm(
   req: LlmCallRequest,
   opts: LlmClientOptions = {},
 ): Promise {
-  const baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, '')
+  // No default endpoint. This client is internal to the `agent-eval` binary
+  // and the loopback optimizer proxy; both name their endpoint explicitly, and
+  // a fallback would let a misconfigured caller bill an unintended provider.
+  if (!opts.baseUrl) throw new Error('callLlm: opts.baseUrl is required')
+  const baseUrl = opts.baseUrl.replace(/\/+$/, '')
   const url = `${baseUrl}/chat/completions`
   const endpoint = '/chat/completions'
   const timeoutMs = req.timeoutMs ?? opts.defaultTimeoutMs ?? DEFAULT_TIMEOUT_MS

From ece3a54918046dd7df9f52b379feebc936b3f2ed Mon Sep 17 00:00:00 2001
From: drewstone 
Date: Thu, 20 Aug 2026 22:48:20 -0700
Subject: [PATCH 3/4] fix(judges): keep the settled receipt when a model answer
 is not JSON

---
 CHANGELOG.md                   |  2 +-
 src/chat-json-call.ts          | 21 ++++++++++++++++-----
 src/intent-match-judge.test.ts | 19 ++++++++++++++++---
 3 files changed, 33 insertions(+), 9 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6b0f42a3..20b2c390 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -20,7 +20,7 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-
 
 ### Added
 
-- `paidJsonChat` collapses the four hand-rolled copies of "reserve the priced maximum, call the transport with a stable call id, settle the receipt, parse the JSON answer" that the two judges and the wire judge endpoint each carried.
+- `paidJsonChat` collapses the three hand-rolled copies of "reserve the priced maximum, call the transport with a stable call id, settle the receipt, parse the JSON answer" that the two judges and the wire judge endpoint each carried. A malformed answer keeps its settled receipt: the call completed and was billed, so the spend stays known rather than becoming unknown.
 - `LlmChargeBounds`: the narrow bound inputs `maximumChargeForLlmRequest` actually reads, so a caller can price a request without naming a transport options type.
 - `examples/_shared/openai-compatible-owner.ts` exposes one OpenAI-compatible endpoint two ways — `openAiCompatibleChatClient` for judges and workers, `openAiCompatibleExecutionOwner` for the optimizer surface — as the reference for what caller-owned execution looks like.
 
diff --git a/src/chat-json-call.ts b/src/chat-json-call.ts
index 274f06fd..151aa227 100644
--- a/src/chat-json-call.ts
+++ b/src/chat-json-call.ts
@@ -78,10 +78,21 @@ export async function paidJsonChat(input: PaidJsonChatInput): Promise(paid.value, input.actor),
-    response: paid.value,
-    receipt: paid.receipt,
+  // The call completed and was billed. A malformed answer is a contract
+  // failure AFTER the money was spent, so it keeps the settled receipt instead
+  // of reporting the spend as unknown.
+  try {
+    return {
+      succeeded: true,
+      value: parseJsonAnswer(paid.value, input.actor),
+      response: paid.value,
+      receipt: paid.receipt,
+    }
+  } catch (error) {
+    return {
+      succeeded: false,
+      error: error instanceof Error ? error : new Error(String(error)),
+      receipt: paid.receipt,
+    }
   }
 }
diff --git a/src/intent-match-judge.test.ts b/src/intent-match-judge.test.ts
index 7d11714c..40e946b0 100644
--- a/src/intent-match-judge.test.ts
+++ b/src/intent-match-judge.test.ts
@@ -5,7 +5,7 @@ import { CostLedger } from './cost-ledger'
 import { runIntentMatchJudge } from './intent-match-judge'
 
 /** Caller-owned transport: agent-eval issues no provider request itself. */
-function answering(answers: Array): ChatClient {
+function answering(answers: Array): ChatClient {
   let call = 0
   return createChatClient({
     transport: 'custom',
@@ -16,9 +16,9 @@ function answering(answers: Array): ChatClient {
       call++
       if (spec instanceof Error) throw spec
       return {
-        content: JSON.stringify(spec),
+        content: typeof spec === 'string' ? spec : JSON.stringify(spec),
         usage: { promptTokens: 30, completionTokens: 20, totalTokens: 50, captured: true },
-        costUsd: null,
+        costUsd: 0.004,
         model: 'mock',
         servedModel: 'mock',
         durationMs: 1,
@@ -80,6 +80,19 @@ describe('runIntentMatchJudge', () => {
     expect(r.error).toMatch(/500|upstream/i)
   })
 
+  it('keeps the settled cost when the model answer is not JSON', async () => {
+    const costLedger = new CostLedger()
+    const r = await runIntentMatchJudge(
+      { userRequest: 'x', sourceFiles: [{ path: 'a.ts', content: 'x' }] },
+      { chat: answering(['not json at all']), costLedger },
+    )
+    // The call completed and was billed; only the answer was unusable. Reporting
+    // the spend as unknown here would hide money the run actually cost.
+    expect(r.available).toBe(false)
+    expect(r.costUsd).toBe(0.004)
+    expect(costLedger.list()).toEqual([expect.objectContaining({ costUsd: 0.004 })])
+  })
+
   it('clamps score to [0, 1]', async () => {
     const r = await runIntentMatchJudge(
       { userRequest: 'x', sourceFiles: [{ path: 'a.ts', content: 'x' }] },

From 10602dcd9af2b96d191b59004d084126473cfb6a Mon Sep 17 00:00:00 2001
From: drewstone 
Date: Thu, 20 Aug 2026 22:53:50 -0700
Subject: [PATCH 4/4] refactor(examples): fail loud on a model-less optimizer
 request

---
 examples/_shared/openai-compatible-owner.ts | 3 ++-
 src/chat-json-call.ts                       | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/examples/_shared/openai-compatible-owner.ts b/examples/_shared/openai-compatible-owner.ts
index 1ae96391..a0c2b15d 100644
--- a/examples/_shared/openai-compatible-owner.ts
+++ b/examples/_shared/openai-compatible-owner.ts
@@ -222,8 +222,9 @@ function endpoint(
 }
 
 function wireBody(request: ChatRequest | ExternalOptimizerChatRequest): Record {
+  if (!request.model) throw new Error('openAiCompatibleExecutionOwner: request.model is required')
   const body: Record = {
-    ...(request.model === undefined ? {} : { model: request.model }),
+    model: request.model,
     messages: request.messages.map((message) =>
       message.role === 'tool'
         ? { role: 'tool', tool_call_id: message.toolCallId, content: message.content }
diff --git a/src/chat-json-call.ts b/src/chat-json-call.ts
index 151aa227..7e3146f7 100644
--- a/src/chat-json-call.ts
+++ b/src/chat-json-call.ts
@@ -43,7 +43,7 @@ export type PaidJsonChatResult =
   | { succeeded: false; error: Error; receipt?: CostReceipt }
 
 /** Parse a JSON answer out of a model response. The transport may fence it. */
-export function parseJsonAnswer(response: ChatResponse, actor: string): T {
+function parseJsonAnswer(response: ChatResponse, actor: string): T {
   try {
     return JSON.parse(extractJsonPayload(response.content)) as T
   } catch (error) {