diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b87575b..42b0b6ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,28 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- --- +## [0.149.0] — 2026-08-18 + +### Removed + +- Twelve modules and their tests: `reviewer`, `analyst/knowledge-capture`, `multi-toolchain-layer`, `dual-agent-bench`, `workspace-inspector`, `golden-matcher`, `ui-finding`, `slo`, `adapters/langchain`, `judge-runner`, `cost-report` and `worker-driver-seed`. + + 0.145.3 tiered the root barrel to consumer-imported symbols and documented front doors. These twelve lost their last export path in that pass and kept their source files. Each is unreachable from all 28 build entry points, so none has shipped in `dist` since 0.145.3, and the only importer each retained was its own test. + + **The public export surface does not move.** A build of 0.148.0 and a build of this release each declare 3,447 export entries, and the two lists are identical. No consumer of any published version can be importing a deleted name, because none of them was reachable to import. + +### Changed + +- Four doc comments named a deleted symbol and now describe the surviving behaviour: `multi-layer-verifier` (twice), `fuzz/types` and `contract/self-improve`, plus the `docs/feature-guide` feature map. `dist` loses 753 bytes, all of it that doc-comment text. + +### Migration + +Nothing to do. No symbol removed here has been importable since 0.145.3. + +One repository still names three of them. `starter-foundry` imports `runAssertions`, `WorkspaceAssertion` and `WorkspaceSnapshot` from this package and pins it to exactly `0.135.1`, where those symbols still exist, so it builds today and this release does not change that. The constraint it already carries is that it cannot move past 0.145.3 without porting them, and that predates this release by four versions. The workspace-assertion helpers were a thin projection over a snapshot it already builds itself. + +--- + ## [0.148.0] — 2026-08-18 ### Added diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 5fc262d1..22e0812f 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-eval-rpc" -version = "0.148.0" +version = "0.149.0" description = "Python RPC client, official optimizer bridge, and DSPy metric adapter for @tangle-network/agent-eval." readme = "README.md" requires-python = ">=3.10" diff --git a/clients/python/src/agent_eval_rpc/__init__.py b/clients/python/src/agent_eval_rpc/__init__.py index a65f377c..0c6dbbe5 100644 --- a/clients/python/src/agent_eval_rpc/__init__.py +++ b/clients/python/src/agent_eval_rpc/__init__.py @@ -53,7 +53,7 @@ try: __version__ = version("agent-eval-rpc") except PackageNotFoundError: - __version__ = "0.148.0" + __version__ = "0.149.0" __all__ = [ "Client", diff --git a/clients/python/uv.lock b/clients/python/uv.lock index d47028fd..4a1ff011 100644 --- a/clients/python/uv.lock +++ b/clients/python/uv.lock @@ -34,7 +34,7 @@ conflicts = [[ [[package]] name = "agent-eval-rpc" -version = "0.148.0" +version = "0.149.0" source = { editable = "." } dependencies = [ { name = "filelock" }, diff --git a/docs/feature-guide.md b/docs/feature-guide.md index bfd5c5b8..245ea0d7 100644 --- a/docs/feature-guide.md +++ b/docs/feature-guide.md @@ -152,7 +152,7 @@ Store as `FeedbackTrajectory`, then derive: | Area | Key exports | Best for | Notes | | --- | --- | --- | --- | | Judging | `llmJudge`, semantic judges, anti-slop, wire rubrics | Content, voice, semantic quality | Pair with objective checks when possible. | -| Verification | `MultiLayerVerifier`, `JudgeRunner`, sandbox harness | Code and multi-step gates | Do not let semantic judges override failed builds. | +| Verification | `MultiLayerVerifier`, sandbox harness | Code and multi-step gates | Do not let semantic judges override failed builds. | | Control | `runAgentControlLoop`, `objectiveEval`, `subjectiveEval` | Long-running agent tasks | Supports budgets, cost, stop policies, trace spans. | | Propose/review | `runProposeReview` | Iterative artifact repair | Good for code, docs, plans, briefs. | | Feedback data | `FeedbackTrajectory`, stores, converters | Human/environment labels | Domain adapters live in downstream repos. | diff --git a/package.json b/package.json index 5406d405..caf98014 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-eval", - "version": "0.148.0", + "version": "0.149.0", "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.", "homepage": "https://github.com/tangle-network/agent-eval#readme", "repository": { diff --git a/src/adapters/langchain.ts b/src/adapters/langchain.ts deleted file mode 100644 index ede22388..00000000 --- a/src/adapters/langchain.ts +++ /dev/null @@ -1,125 +0,0 @@ -/** - * # `@tangle-network/agent-eval/adapters/langchain` — wrap any LangChain - * Runnable as a `Dispatch` (or `JudgeConfig`). - * - * **Why structural, not pinned**: we don't depend on `@langchain/core` at - * install time. The adapter accepts anything with the canonical LangChain - * Runnable shape (`invoke(input, config?)`), so it works with their - * `Runnable`, `RunnableSequence`, `RunnableMap`, `RunnablePassthrough`, - * and any custom Runnable-shaped object. No version pin, no peer dep, - * no bundle-bloat risk. - * - * **Why this exists**: the most-asked question from foreign agent - * builders is "I'm already on LangChain — how do I plug in?". The answer - * is one function. Wrap your existing Runnable, pass the Dispatch into - * `runEval` / `runImprovementLoop`, ship. - */ - -import type { Dispatch, JudgeConfig, JudgeScore, Scenario } from '../contract' - -// ── Minimal structural type ────────────────────────────────────────── -// -// Whatever has `invoke(input, config?)` qualifies. We accept any -// config shape (LangChain's RunnableConfig has many optional fields) -// — the only thing we need is the AbortSignal seam, which LangChain's -// RunnableConfig already supports as `signal?: AbortSignal`. - -export interface RunnableLike { - invoke(input: TInput, config?: { signal?: AbortSignal; [key: string]: unknown }): Promise -} - -// ── Dispatch wrapper ──────────────────────────────────────────────── - -export interface LangchainDispatchOptions { - /** The Runnable (or RunnableSequence, or anything `.invoke`able). */ - runnable: RunnableLike - /** - * Optional config merged into every `invoke` call — tags, metadata, - * callbacks, runName. The substrate's per-cell `AbortSignal` is - * always merged in last (and so wins). - */ - config?: Record -} - -/** - * Wrap a LangChain Runnable as a `Dispatch`. The Runnable's input must - * accept the scenario (typically you'll shape it via - * `RunnableMap`/`RunnableLambda` upstream); its output is the artifact - * the engine + judges see. - * - * @example - * const chain = prompt.pipe(model).pipe(parser) - * const dispatch = langchainDispatch({ runnable: chain }) - * await runEval({ scenarios, dispatch, judges: [...], storage, runDir }) - */ -export function langchainDispatch( - opts: LangchainDispatchOptions, -): Dispatch { - return async (scenario, ctx) => { - return opts.runnable.invoke(scenario, { - ...opts.config, - signal: ctx.signal, - }) - } -} - -// ── Judge wrapper ─────────────────────────────────────────────────── - -export interface LangchainJudgeOptions { - /** Judge name; appears in `CampaignResult.aggregates.byJudge`. */ - name: string - /** - * Dimensions the judge scores. Used both for the judge's own prompt - * (if it reads them) and for the aggregator's `byJudge` rollup. - */ - dimensions: { key: string; description: string }[] - /** - * A Runnable that takes `{ artifact, scenario }` and returns a - * partial `JudgeScore` — the dimensions map at minimum. `composite` - * is computed by averaging `dimensions` when the Runnable doesn't - * provide it; `notes` defaults to an empty string. - */ - runnable: RunnableLike<{ artifact: TArtifact; scenario: TScenario }, Partial> - appliesTo?: (scenario: TScenario) => boolean -} - -/** - * Wrap a LangChain Runnable as a `JudgeConfig`. The Runnable can be any - * structured-output chain (e.g. `prompt.pipe(model).pipe(StructuredOutputParser)`) - * that returns a `Partial`. - * - * The substrate's invariant — throw on judge failure, never silently - * fold errors into a zero — is preserved: any error from the Runnable - * propagates and the substrate records a failed cell. - * - * @example - * const scorePrompt = ChatPromptTemplate.fromTemplate(`...`) - * const judgeChain = scorePrompt.pipe(judgeModel).pipe(jsonParser) - * const judge = langchainJudge({ - * name: 'marketing-quality', - * dimensions: [{ key: 'hook_strength', description: '...' }, ...], - * runnable: judgeChain, - * }) - */ -export function langchainJudge( - opts: LangchainJudgeOptions, -): JudgeConfig { - return { - name: opts.name, - dimensions: opts.dimensions, - appliesTo: opts.appliesTo, - async score({ artifact, scenario, signal }) { - const result = await opts.runnable.invoke({ artifact, scenario }, { signal }) - const dims = (result.dimensions ?? {}) as Record - const dimValues = Object.values(dims) - const composite = - result.composite ?? - (dimValues.length > 0 ? dimValues.reduce((a, b) => a + b, 0) / dimValues.length : 0) - return { - dimensions: dims, - composite, - notes: result.notes ?? '', - } - }, - } -} diff --git a/src/adapters/otel.ts b/src/adapters/otel.ts deleted file mode 100644 index bbdaf19a..00000000 --- a/src/adapters/otel.ts +++ /dev/null @@ -1,254 +0,0 @@ -/** - * # `@tangle-network/agent-eval/adapters/otel` — OTel→hosted bridge. - * - * Forwards OpenTelemetry-shaped spans into the hosted-tier ingest endpoint - * via `createHostedClient`. Works with anything that emits OTel - * `ReadableSpan`s — `@opentelemetry/sdk-trace-base` directly, OpenLLMetry, - * Phoenix's OTel exporter, TraceAI from future-agi, any OTel collector - * pipeline. - * - * **Pattern:** - * - * ```ts - * import { createHostedClient } from '@tangle-network/agent-eval/hosted' - * import { createOtelBridge } from '@tangle-network/agent-eval/adapters/otel' - * - * const client = createHostedClient({ endpoint, apiKey, tenantId }) - * const bridge = createOtelBridge({ client, defaultRunId: substrateRunId }) - * - * // Wherever your OTel SpanProcessor hands you a finished span: - * processor.onEnd = (span) => { void bridge.ingest([span]) } - * // …or in a SpanProcessor.onShutdown / batch flush: - * await bridge.ingest(batchedSpans) - * ``` - * - * No `@opentelemetry/*` dependency is declared here — the adapter accepts - * a structurally-typed `OtelLikeSpan`. This keeps the substrate dep graph - * lean while remaining compatible with any OTel SDK that produces - * `ReadableSpan`-shaped instances. If a consumer's span shape exposes - * `parentSpanId` as a top-level field rather than via `parentSpanContext()`, - * the adapter accepts both forms. - */ - -import type { HostedClient } from '../hosted/client' -import type { TraceSpanEvent } from '../hosted/types' - -// ── OTel-compatible structural types ───────────────────────────────── - -/** - * `[seconds, nanoseconds]` — the OTel SDK's `HrTime` shape. Spans emitted - * by the OTel SDK carry timestamps in this representation; we convert to - * a single unix-nano number for the wire format. - */ -export type HrTime = [number, number] - -/** Standard OTel `SpanStatusCode` numeric values: 0 = UNSET, 1 = OK, 2 = ERROR. */ -export const OTEL_STATUS_UNSET = 0 -export const OTEL_STATUS_OK = 1 -export const OTEL_STATUS_ERROR = 2 - -export type OtelAttributeValue = - | string - | number - | boolean - | null - | undefined - | Array - | Array - | Array - -/** - * Structural surface compatible with `@opentelemetry/sdk-trace-base`'s - * `ReadableSpan`. Consumers pass instances they get from their OTel SDK. - */ -export interface OtelLikeSpan { - spanContext: () => { traceId: string; spanId: string; traceFlags?: number } - /** Set on the span itself by some SDKs (OTLP-shape). Other SDKs expose - * the parent via `parentSpanContext()` instead — the adapter checks both. */ - parentSpanId?: string - parentSpanContext?: () => { spanId: string } | undefined - name: string - startTime: HrTime - endTime: HrTime - attributes: Record - events?: Array<{ - name: string - time: HrTime - attributes?: Record - }> - status?: { code: number; message?: string } -} - -// ── Conversion ─────────────────────────────────────────────────────── - -/** Convert `[seconds, nanoseconds]` to an exact base-10 Unix-nanosecond string. */ -export function hrTimeToUnixNano(hr: HrTime): string { - const [seconds, nanos] = hr - if (!Number.isSafeInteger(seconds) || !Number.isSafeInteger(nanos)) { - throw new RangeError('OTel HrTime values must be safe integers') - } - if (seconds < 0 || nanos < 0 || nanos >= 1_000_000_000) { - throw new RangeError('OTel HrTime must contain non-negative seconds and nanoseconds < 1e9') - } - return (BigInt(seconds) * 1_000_000_000n + BigInt(nanos)).toString() -} - -function statusCodeName(code: number | undefined): 'OK' | 'ERROR' | 'UNSET' { - if (code === OTEL_STATUS_OK) return 'OK' - if (code === OTEL_STATUS_ERROR) return 'ERROR' - return 'UNSET' -} - -/** - * Normalise OTel attributes to the scalar shape the wire format accepts. - * - * - null / undefined → dropped (no representation in the wire format) - * - string / number / boolean → passed through - * - Array → JSON-stringified (the wire format is scalar-only; - * dropping arrays would silently lose information, so we serialise - * them deterministically) - * - * OTel SDK / TraceAI emit array attributes routinely (e.g. `tool.names`, - * `http.request.header.set-cookie`). Stringification keeps the data flowing - * through to the dashboard; consumers parse with `JSON.parse` if they need - * the structured form back. - */ -function cleanAttributes( - attrs: Record | undefined, -): Record { - const out: Record = {} - if (!attrs) return out - for (const [k, v] of Object.entries(attrs)) { - if (v === null || v === undefined) continue - if (typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean') { - out[k] = v - continue - } - if (Array.isArray(v)) { - out[k] = JSON.stringify(v) - } - } - return out -} - -function readPivotString( - attrs: Record, - key: string, -): string | undefined { - const v = attrs[key] - return typeof v === 'string' ? v : undefined -} - -function readPivotNumber( - attrs: Record, - key: string, -): number | undefined { - const v = attrs[key] - return typeof v === 'number' ? v : undefined -} - -function resolveParentSpanId(span: OtelLikeSpan): string | undefined { - if (span.parentSpanId) return span.parentSpanId - const ctx = span.parentSpanContext?.() - return ctx?.spanId -} - -// ── Bridge ─────────────────────────────────────────────────────────── - -export interface OtelBridgeOptions { - /** Hosted client to forward spans to. */ - client: HostedClient - /** When set, spans missing a `tangle.runId` attribute receive this value - * on the way out. Useful when the OTel emitter doesn't know which - * substrate run it's serving. */ - defaultRunId?: string - /** Max spans per ingest call. Default 200. The hosted ingest endpoint - * caps at 5000 per call; we batch smaller by default to keep individual - * retries cheap. */ - batchSize?: number - /** Called when a batch fails to ingest. Defaults to a console.warn. Hook - * this when you need backpressure or to spill to a fallback. */ - onError?: (err: unknown, batch: TraceSpanEvent[]) => void | Promise -} - -export interface OtelBridge { - /** Convert + ingest a batch of OTel-shape spans. */ - ingest(spans: OtelLikeSpan[]): Promise - /** Convert one OTel span to the wire-format event. Useful for tests or - * custom batching pipelines. */ - spanToEvent(span: OtelLikeSpan): TraceSpanEvent -} - -export function createOtelBridge(opts: OtelBridgeOptions): OtelBridge { - const batchSize = opts.batchSize ?? 200 - const onError = - opts.onError ?? - ((err) => { - console.warn('[otel-bridge] ingest batch failed:', err) - }) - - function convert(span: OtelLikeSpan): TraceSpanEvent { - const ctx = span.spanContext() - const attributes = cleanAttributes(span.attributes) - // Pull pivot attributes off the cleaned attribute map so they round-trip - // through the wire format's first-class fields. They REMAIN in - // `attributes` as well so downstream OTel viewers see the same values. - const runId = readPivotString(attributes, 'tangle.runId') ?? opts.defaultRunId - const scenarioId = readPivotString(attributes, 'tangle.scenarioId') - const cellId = readPivotString(attributes, 'tangle.cellId') - const generation = readPivotNumber(attributes, 'tangle.generation') - - if (runId && !attributes['tangle.runId']) { - attributes['tangle.runId'] = runId - } - - const event: TraceSpanEvent = { - traceId: ctx.traceId, - spanId: ctx.spanId, - name: span.name, - startTimeUnixNano: hrTimeToUnixNano(span.startTime), - endTimeUnixNano: hrTimeToUnixNano(span.endTime), - attributes, - } - const parentSpanId = resolveParentSpanId(span) - if (parentSpanId) event.parentSpanId = parentSpanId - if (span.events && span.events.length > 0) { - event.events = span.events.map((e) => { - const eventAttrs = cleanAttributes(e.attributes) - const node: { - timeUnixNano: string - name: string - attributes?: Record - } = { - timeUnixNano: hrTimeToUnixNano(e.time), - name: e.name, - } - if (Object.keys(eventAttrs).length > 0) node.attributes = eventAttrs - return node - }) - } - if (span.status) { - event.status = { code: statusCodeName(span.status.code), message: span.status.message } - } - if (runId) event['tangle.runId'] = runId - if (scenarioId) event['tangle.scenarioId'] = scenarioId - if (cellId) event['tangle.cellId'] = cellId - if (generation !== undefined) event['tangle.generation'] = generation - return event - } - - async function ingest(spans: OtelLikeSpan[]): Promise { - if (spans.length === 0) return - const events = spans.map(convert) - for (let i = 0; i < events.length; i += batchSize) { - const batch = events.slice(i, i + batchSize) - try { - await opts.client.ingestTraces(batch) - } catch (err) { - await onError(err, batch) - } - } - } - - return { ingest, spanToEvent: convert } -} diff --git a/src/analyst/benchmark-implementation.ts b/src/analyst/benchmark-implementation.ts index 83282e6b..c8e5dac3 100644 --- a/src/analyst/benchmark-implementation.ts +++ b/src/analyst/benchmark-implementation.ts @@ -10,7 +10,7 @@ export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([ ]) export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = - '5c7e5561a2ecaa48aaffc52d8a44f582642ed434a92839ae4741ad5556fd9789' + '3e17b679d34215f636931e6d39cb5215d878cdaa501c38fd6cef513ce844677f' /** The published benchmark evidence was produced at this package version, by * the retired one-shot direct runner, before trace analysts moved to the diff --git a/src/analyst/knowledge-capture.test.ts b/src/analyst/knowledge-capture.test.ts deleted file mode 100644 index 65bfa633..00000000 --- a/src/analyst/knowledge-capture.test.ts +++ /dev/null @@ -1,281 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { - captureKnowledgeCandidates, - GROUNDING_EVIDENCE_KINDS, - KNOWLEDGE_CANDIDATE_TAG, -} from './knowledge-capture' -import { type AnalystFinding, type EvidenceRef, makeFinding } from './types' - -/** - * The observed session this module was built for: the agent filed a public - * issue naming the wrong root cause, then discovered mid-run that the - * platform grants a signup credit. The fact was learned and then lost. - */ -function signupCreditFinding( - overrides: Partial> = {}, -): AnalystFinding { - // Overrides are folded in BEFORE makeFinding so finding_id derives from the - // claim and subject the test actually uses; computing it first would give - // two different findings the same identity. - return makeFinding({ - analyst_id: 'knowledge-gap', - severity: 'high', - area: 'knowledge-gap', - claim: - 'The platform grants a $2 signup credit to new accounts; the agent did not know this and attributed the balance to a billing bug.', - rationale: - 'The agent filed issue #412 blaming a double-charge before reading the credit line in the account response.', - recommended_action: - 'Record the signup credit amount and its eligibility window on the billing wiki page.', - evidence_refs: [ - { - kind: 'span', - uri: 'otlp:span/2f9c1a4b', - excerpt: 'account.credits[0] = { source: "signup", amount_usd: 2 }', - }, - { kind: 'span', uri: 'otlp:span/7d31e880', excerpt: 'Actually, that $2 is a signup credit.' }, - ], - confidence: 0.9, - subject: 'agent-knowledge:wiki:platform-billing-signup-credit', - ...overrides, - }) -} - -describe('captureKnowledgeCandidates — routing', () => { - it('turns an agent-knowledge:wiki finding into a candidate page', () => { - const { candidates, dropped } = captureKnowledgeCandidates([signupCreditFinding()]) - - expect(dropped).toEqual([]) - expect(candidates).toHaveLength(1) - const candidate = candidates[0]! - expect(candidate.slug).toBe('platform-billing-signup-credit') - expect(candidate.heading).toBeUndefined() - expect(candidate.page.id).toBe('platform-billing-signup-credit') - expect(candidate.page.path).toBe('knowledge/platform-billing-signup-credit.md') - expect(candidate.page.title).toBe('Platform Billing Signup Credit') - expect(candidate.page.tags).toEqual([KNOWLEDGE_CANDIDATE_TAG]) - expect(candidate.analystId).toBe('knowledge-gap') - expect(candidate.severity).toBe('high') - expect(candidate.confidence).toBe(0.9) - expect(candidate.sourceFindingId).toBe(signupCreditFinding().finding_id) - }) - - it('ignores loci that belong to other runtime layers', () => { - const otherLayers = [ - 'system-prompt:account-access', - 'tool-doc:billing-api:credits', - 'memory:last-invoice', - 'skill:billing-triage', - 'websearch:outdated:stripe-fees', - 'agent-knowledge:claim:signup-credit', - 'agent-knowledge:raw:billing-docs-2026', - 'agent-knowledge:stale:platform-billing-signup-credit', - ] - const findings = otherLayers.map((subject) => signupCreditFinding({ subject })) - - const { candidates, dropped } = captureKnowledgeCandidates(findings) - - expect(candidates).toEqual([]) - expect(dropped.map((d) => d.reason)).toEqual(otherLayers.map(() => 'non-wiki-locus')) - expect(dropped.map((d) => d.subject)).toEqual(otherLayers) - }) - - it('drops a finding with no subject and one whose subject is not in the grammar', () => { - const noSubject = signupCreditFinding({ subject: undefined }) - const prose = signupCreditFinding({ subject: 'please add this to the wiki' }) - - const { candidates, dropped } = captureKnowledgeCandidates([noSubject, prose]) - - expect(candidates).toEqual([]) - expect(dropped).toEqual([ - { findingId: noSubject.finding_id, reason: 'no-subject' }, - { - findingId: prose.finding_id, - subject: 'please add this to the wiki', - reason: 'unparsed-subject', - }, - ]) - }) - - it('accounts for every input finding exactly once', () => { - const findings = [ - signupCreditFinding({ claim: 'kept: the signup credit is $2.' }), - signupCreditFinding({ claim: 'other layer', subject: 'system-prompt:account-access' }), - signupCreditFinding({ claim: 'ungrounded', evidence_refs: [] }), - signupCreditFinding({ claim: 'no locus', subject: undefined }), - ] - - const { candidates, dropped } = captureKnowledgeCandidates(findings) - - expect(candidates.length + dropped.length).toBe(findings.length) - const seen = [...candidates.map((c) => c.sourceFindingId), ...dropped.map((d) => d.findingId)] - expect(new Set(seen)).toEqual(new Set(findings.map((f) => f.finding_id))) - }) -}) - -describe('captureKnowledgeCandidates — grounding', () => { - it('drops a finding with no evidence refs rather than emitting empty anchors', () => { - const ungrounded = signupCreditFinding({ evidence_refs: [] }) - - const { candidates, dropped } = captureKnowledgeCandidates([ungrounded]) - - expect(candidates).toEqual([]) - expect(dropped).toEqual([ - { - findingId: ungrounded.finding_id, - subject: 'agent-knowledge:wiki:platform-billing-signup-credit', - reason: 'no-grounding-evidence', - }, - ]) - }) - - it('drops a finding anchored only to another finding or a metric', () => { - const refs: EvidenceRef[] = [ - { kind: 'finding', uri: 'f_1234567890abcdef1234' }, - { kind: 'metric', uri: 'metric:clarifying_questions' }, - ] - for (const kind of refs.map((r) => r.kind)) { - expect(GROUNDING_EVIDENCE_KINDS).not.toContain(kind) - } - - const { candidates, dropped } = captureKnowledgeCandidates([ - signupCreditFinding({ evidence_refs: refs }), - ]) - - expect(candidates).toEqual([]) - expect(dropped[0]!.reason).toBe('no-grounding-evidence') - }) - - it('drops a wiki-routed finding whose claim is blank', () => { - const { candidates, dropped } = captureKnowledgeCandidates([ - signupCreditFinding({ claim: ' ' }), - ]) - - expect(candidates).toEqual([]) - expect(dropped[0]!.reason).toBe('empty-claim') - }) - - it('anchors match the finding evidence URIs exactly, in order', () => { - const finding = signupCreditFinding() - const { candidates } = captureKnowledgeCandidates([finding]) - const candidate = candidates[0]! - - const expectedUris = finding.evidence_refs.map((r) => r.uri) - expect(candidate.anchors.map((a) => a.uri)).toEqual(expectedUris) - expect(candidate.page.sourceIds).toEqual(expectedUris) - expect(candidate.page.frontmatter.sources).toEqual(expectedUris) - expect(candidate.anchors).toEqual( - finding.evidence_refs.map((r) => ({ kind: r.kind, uri: r.uri, excerpt: r.excerpt })), - ) - }) - - it('keeps grounded refs while discarding blank, duplicate, and non-observed ones', () => { - const finding = signupCreditFinding({ - evidence_refs: [ - { kind: 'span', uri: 'otlp:span/2f9c1a4b' }, - { kind: 'span', uri: ' ' }, - { kind: 'span', uri: 'otlp:span/2f9c1a4b' }, - { kind: 'finding', uri: 'f_1234567890abcdef1234' }, - { kind: 'artifact', uri: 'artifact:issue-412.json' }, - ], - }) - - const { candidates } = captureKnowledgeCandidates([finding]) - - expect(candidates[0]!.anchors).toEqual([ - { kind: 'span', uri: 'otlp:span/2f9c1a4b' }, - { kind: 'artifact', uri: 'artifact:issue-412.json' }, - ]) - }) -}) - -describe('captureKnowledgeCandidates — page content', () => { - it('carries the claim, recommended action, and evidence excerpts, and nothing else', () => { - const finding = signupCreditFinding() - const { text } = captureKnowledgeCandidates([finding]).candidates[0]!.page - - expect(text).toContain(finding.claim) - expect(text).toContain(finding.rationale) - expect(text).toContain(finding.recommended_action) - for (const ref of finding.evidence_refs) { - expect(text).toContain(ref.uri) - expect(text).toContain(ref.excerpt) - } - // The frontmatter block belongs to the `frontmatter` field, matching how - // the knowledge base splits a page it loads from disk. - expect(text.startsWith('---')).toBe(false) - expect(text).toContain('# Platform Billing Signup Credit') - }) - - it('omits sections the finding did not supply', () => { - const { text } = captureKnowledgeCandidates([ - signupCreditFinding({ rationale: undefined, recommended_action: undefined }), - ]).candidates[0]!.page - - expect(text).not.toContain('Rationale') - expect(text).not.toContain('Recommended action') - expect(text).toContain('Sources') - }) - - it('never fabricates outgoing links', () => { - const { page } = captureKnowledgeCandidates([signupCreditFinding()]).candidates[0]! - - expect(page.outLinks).toEqual([]) - expect(page.text).not.toMatch(/\[\[/) - }) - - it('mirrors sourceIds and tags into frontmatter so a disk round-trip keeps them', () => { - const finding = signupCreditFinding() - const { page } = captureKnowledgeCandidates([finding]).candidates[0]! - - expect(page.frontmatter.sources).toEqual(page.sourceIds) - expect(page.frontmatter.tags).toEqual(page.tags) - expect(page.frontmatter.id).toBe(page.id) - expect(page.frontmatter.title).toBe(page.title) - expect(page.frontmatter.status).toBe('candidate') - expect(page.frontmatter.drafted_from_finding).toBe(finding.finding_id) - expect(page.frontmatter.analyst_id).toBe('knowledge-gap') - expect(page.frontmatter.severity).toBe('high') - expect(page.frontmatter.confidence).toBe(0.9) - expect(page.frontmatter.derived_from_judge).toBe(false) - }) - - it('flags a finding lifted from a judge verdict rather than read off a trace', () => { - const { page } = captureKnowledgeCandidates([signupCreditFinding({ derived_from_judge: true })]) - .candidates[0]! - - expect(page.frontmatter.derived_from_judge).toBe(true) - }) - - it('records a heading locus as a section body under the same page', () => { - const { candidates } = captureKnowledgeCandidates([ - signupCreditFinding({ - subject: 'agent-knowledge:wiki:platform-billing-signup-credit#eligibility', - }), - ]) - const candidate = candidates[0]! - - expect(candidate.heading).toBe('eligibility') - expect(candidate.slug).toBe('platform-billing-signup-credit') - expect(candidate.page.path).toBe('knowledge/platform-billing-signup-credit.md') - expect(candidate.page.text.startsWith('## Eligibility\n')).toBe(true) - expect(candidate.page.text).not.toContain('# Platform Billing Signup Credit') - }) - - it('preserves two findings that route to the same page instead of merging them', () => { - const first = signupCreditFinding() - const second = signupCreditFinding({ - claim: 'The signup credit expires 30 days after account creation.', - evidence_refs: [{ kind: 'span', uri: 'otlp:span/aa11bb22' }], - }) - - const { candidates } = captureKnowledgeCandidates([first, second]) - - expect(candidates).toHaveLength(2) - expect(candidates.map((c) => c.page.path)).toEqual([ - 'knowledge/platform-billing-signup-credit.md', - 'knowledge/platform-billing-signup-credit.md', - ]) - expect(candidates[0]!.sourceFindingId).not.toBe(candidates[1]!.sourceFindingId) - }) -}) diff --git a/src/analyst/knowledge-capture.ts b/src/analyst/knowledge-capture.ts deleted file mode 100644 index 774f6c5f..00000000 --- a/src/analyst/knowledge-capture.ts +++ /dev/null @@ -1,300 +0,0 @@ -/** - * Knowledge capture — knowledge-gap findings become CANDIDATE wiki pages. - * - * The gap this closes: the knowledge-gap analyst already detects facts the - * agent had to discover at run time. Observed case — an agent learned - * mid-session that the platform grants a signup credit, but only after it - * had filed a public issue naming the wrong root cause. It corrected - * itself, the session ended, and the fact died with it; the next session - * starts ignorant and can repeat the mistake. A detector existed - * (knowledge-gap) and a store existed (`@tangle-network/agent-knowledge`), - * and nothing joined them. This module is that join. - * - * These are CANDIDATES, never writes. The transform is pure: no file I/O, - * no knowledge-base handle, no network. That boundary is the point, not an - * omission. An analyst finding is a MODEL CLAIM about a trace, and writing - * model claims straight into a curated knowledge base is precisely the - * corruption the knowledge-poisoning analyst exists to detect — a KB that - * ingests its own analyst's guesses will later cite them as sources and - * launder a guess into a fact. So the output is a reviewable value: a human - * or a gated activation path decides what gets written. - * - * Routing is by subject prefix. Knowledge-gap findings carry a typed locus - * (`agent-knowledge:wiki:platform-billing-signup-credit`, - * `system-prompt:account-access`, `tool-doc:gh:auth`, ...), and the prefix - * says which layer owns the fix. Only `agent-knowledge:wiki:*` is a page; - * every other locus belongs to the improvement adapter or another layer and - * is dropped here with a stated reason rather than silently skipped. - * - * Caller responsibility this transform deliberately does NOT take on: - * `knowledge-poisoning` findings also emit `agent-knowledge:wiki:*` loci, - * but their claim describes a page that is WRONG — not knowledge to add. - * Every candidate carries `analystId` so a caller feeding mixed findings - * can filter; encoding that policy here would hide it from the operator. - * - * Structural contract: `KnowledgeCandidatePage` mirrors `KnowledgePage` - * from `@tangle-network/agent-knowledge` field for field, so a reviewed - * candidate can be handed to that package's page store unchanged. The shape - * is restated rather than imported because agent-eval does not depend on - * agent-knowledge — the dependency runs the other way (agent-knowledge - * imports `AnalystFinding` from agent-eval), and importing back would close - * a package cycle. - */ - -import { parseFindingSubject } from './finding-subject' -import type { AnalystFinding, AnalystSeverity, EvidenceRef } from './types' - -/** - * Evidence kinds admissible as page anchors: references that point at a - * recorded observation a reviewer can go re-read. - * - * `finding` and `metric` refs are excluded on purpose. A `finding` ref - * anchors one model claim to another model claim, so the resulting page - * would be grounded in nothing observable no matter how deep the chain is - * followed. A `metric` ref names a scalar reading with no retrievable text, - * so it cannot support the claim it would be cited for. - */ -export const GROUNDING_EVIDENCE_KINDS: ReadonlyArray = [ - 'span', - 'event', - 'artifact', -] - -/** - * Tag stamped on every candidate page. Survives into the knowledge base if - * a candidate is ever activated, so an unreviewed model claim that reached - * the store is still greppable after the fact. - */ -export const KNOWLEDGE_CANDIDATE_TAG = 'analyst-candidate' - -/** One admitted evidence reference, carried verbatim from the finding. */ -export interface KnowledgeCandidateAnchor { - kind: EvidenceRef['kind'] - uri: string - excerpt?: string -} - -/** - * A page value shaped exactly like agent-knowledge's `KnowledgePage`. - * - * Arrays are mutable, not `readonly`, because a `readonly string[]` is not - * assignable to that package's `string[]` fields — the whole point of the - * shape is that a reviewed candidate passes through without a translation - * step that could drop provenance. - * - * Invariant that the knowledge base's own loader depends on: `text` is the - * body WITHOUT the frontmatter block, and `sourceIds` / `tags` mirror - * `frontmatter.sources` / `frontmatter.tags`. That loader reconstructs the - * page by parsing the frontmatter off the markdown file, so a candidate - * whose fields disagree with its frontmatter would lose its source anchors - * the first time it round-tripped through disk. - */ -export interface KnowledgeCandidatePage { - id: string - path: string - title: string - text: string - frontmatter: Record - sourceIds: string[] - tags: string[] - outLinks: string[] -} - -/** A candidate page plus the provenance a reviewer needs to judge it. */ -export interface KnowledgeCandidate { - page: KnowledgeCandidatePage - /** Page slug from the finding's locus — the page identity in the KB. */ - slug: string - /** - * Section under `slug` when the locus named one. Present means `page.text` - * is a SECTION body to append under an existing page, not a whole file. - */ - heading?: string - sourceFindingId: string - analystId: string - severity: AnalystSeverity - confidence: number - anchors: KnowledgeCandidateAnchor[] -} - -export type KnowledgeCaptureDropReason = - /** Descriptive finding with no locus — nothing to route. */ - | 'no-subject' - /** Subject present but outside the finding-subject grammar. */ - | 'unparsed-subject' - /** A valid locus owned by another layer (system prompt, tool doc, ...). */ - | 'non-wiki-locus' - /** No evidence reference points at a recorded observation. */ - | 'no-grounding-evidence' - /** Claim text is empty, so the page would have no content. */ - | 'empty-claim' - -/** Why one finding produced no candidate. Drops are reported, never silent. */ -export interface KnowledgeCaptureDrop { - findingId: string - subject?: string - reason: KnowledgeCaptureDropReason -} - -export interface KnowledgeCaptureResult { - candidates: KnowledgeCandidate[] - /** - * Every input finding that produced no candidate, with its reason. An - * invisible skip is how routing tables rot: the caller can assert that - * `candidates.length + dropped.length === findings.length` and see which - * loci its analysts actually emit. - */ - dropped: KnowledgeCaptureDrop[] -} - -/** - * Turn findings into candidate wiki pages. - * - * Pure and total: every input finding appears in exactly one of - * `candidates` or `dropped`. Ordering follows the input. - * - * Several findings may route to the same slug, producing several candidates - * that share `page.id` and `page.path`. That collision is preserved rather - * than resolved — merging two claims about one page is an editorial - * decision, and picking a winner here would discard evidence the reviewer - * never saw. - * - * Confidence is recorded, never thresholded. Discarding a low-confidence - * finding is a review policy; applying it inside the transform would make - * the policy invisible to the operator who has to defend the KB's contents. - */ -export function captureKnowledgeCandidates( - findings: ReadonlyArray, -): KnowledgeCaptureResult { - const candidates: KnowledgeCandidate[] = [] - const dropped: KnowledgeCaptureDrop[] = [] - - for (const finding of findings) { - const outcome = captureOne(finding) - if ('reason' in outcome) dropped.push(outcome) - else candidates.push(outcome) - } - - return { candidates, dropped } -} - -function captureOne(finding: AnalystFinding): KnowledgeCandidate | KnowledgeCaptureDrop { - const drop = (reason: KnowledgeCaptureDropReason): KnowledgeCaptureDrop => ({ - findingId: finding.finding_id, - ...(finding.subject === undefined ? {} : { subject: finding.subject }), - reason, - }) - - if (finding.subject === undefined || finding.subject.trim().length === 0) { - return drop('no-subject') - } - - const subject = parseFindingSubject(finding.subject) - if (subject === null) return drop('unparsed-subject') - if (subject.kind !== 'knowledge.wiki') return drop('non-wiki-locus') - - const anchors = groundingAnchors(finding.evidence_refs) - if (anchors.length === 0) return drop('no-grounding-evidence') - - const claim = finding.claim.trim() - if (claim.length === 0) return drop('empty-claim') - - const title = humanize(subject.slug) - const frontmatter: Record = { - id: subject.slug, - title, - status: 'candidate', - drafted_from_finding: finding.finding_id, - analyst_id: finding.analyst_id, - severity: finding.severity, - confidence: finding.confidence, - // Distinguishes a claim read off a trace from one lifted out of another - // model's verdict; the second is a weaker basis for a curated page. - derived_from_judge: finding.derived_from_judge === true, - sources: anchors.map((a) => a.uri), - tags: [KNOWLEDGE_CANDIDATE_TAG], - } - - return { - page: { - id: subject.slug, - path: `knowledge/${subject.slug}.md`, - title, - text: renderBody({ finding, claim, title, heading: subject.heading, anchors }), - frontmatter, - sourceIds: anchors.map((a) => a.uri), - tags: [KNOWLEDGE_CANDIDATE_TAG], - // Never inferred. A link to a page that may not exist is fabricated - // structure, and the KB's linter reports it as a broken link. - outLinks: [], - }, - slug: subject.slug, - ...(subject.heading === undefined ? {} : { heading: subject.heading }), - sourceFindingId: finding.finding_id, - analystId: finding.analyst_id, - severity: finding.severity, - confidence: finding.confidence, - anchors, - } -} - -/** - * Admit references that anchor to observation, dropping blank URIs. Dedupes - * on URI so a finding that cited one span twice does not inflate the page's - * apparent source count. - */ -function groundingAnchors(refs: ReadonlyArray): KnowledgeCandidateAnchor[] { - const seen = new Set() - const anchors: KnowledgeCandidateAnchor[] = [] - for (const ref of refs) { - if (!GROUNDING_EVIDENCE_KINDS.includes(ref.kind)) continue - const uri = ref.uri.trim() - if (uri.length === 0 || seen.has(uri)) continue - seen.add(uri) - const excerpt = ref.excerpt?.trim() - anchors.push({ kind: ref.kind, uri, ...(excerpt ? { excerpt } : {}) }) - } - return anchors -} - -/** - * Render the page body. Every line originates in the finding: the claim, its - * rationale, its recommended action, and its evidence URIs with excerpts - * quoted verbatim. The only derived text is the title, a mechanical - * de-kebabing of the slug the analyst itself chose. - * - * The frontmatter block is NOT rendered here — it lives in the `frontmatter` - * field, matching how the knowledge base splits a page it loads from disk. - */ -function renderBody(input: { - finding: AnalystFinding - claim: string - title: string - heading?: string - anchors: ReadonlyArray -}): string { - const { finding, claim, title, heading, anchors } = input - const rationale = finding.rationale?.trim() - const action = finding.recommended_action?.trim() - - const lines = [heading ? `## ${humanize(heading)}` : `# ${title}`, '', claim, ''] - if (rationale) lines.push(heading ? '### Rationale' : '## Rationale', '', rationale, '') - if (action) - lines.push(heading ? '### Recommended action' : '## Recommended action', '', action, '') - - lines.push(heading ? '### Sources' : '## Sources', '') - for (const anchor of anchors) { - lines.push(`- \`${anchor.uri}\``) - // Blockquote per line rather than an inline quoted string: excerpts can - // contain quotes and newlines, and the excerpt must survive unedited. - if (anchor.excerpt) lines.push(...anchor.excerpt.split('\n').map((l) => ` > ${l}`)) - } - - return `${lines.join('\n')}\n` -} - -function humanize(slug: string): string { - const words = slug.split('-').filter((w) => w.length > 0) - if (words.length === 0) return slug - return words.map((w) => w[0]!.toUpperCase() + w.slice(1)).join(' ') -} diff --git a/src/auto-pr.ts b/src/auto-pr.ts deleted file mode 100644 index 138664e8..00000000 --- a/src/auto-pr.ts +++ /dev/null @@ -1,504 +0,0 @@ -/** - * Automated pull-request transports for the production loop. - * - * A consumer's `runProductionLoop` ships a promoted surface diff as a - * reviewable code change by handing one of these clients to its - * `ProductionShipConfig.client`. Each `AutoPrClient.proposeChange`: - * - * 1. Stage a branch off `baseBranch`. - * 2. Write each `fileChange` into the worktree. - * 3. Commit + push. - * 4. Open a PR via the GitHub API. - * - * (The campaign-loop `autoOnPromote: 'pr'` path uses `openAutoPr` in - * `src/campaign/auto-pr.ts`, a self-contained `gh pr create` shell-out.) - * - * Two transports ship in core: - * - * - `ghCliClient(opts)` — shells out to the `gh` CLI. No extra deps, - * re-uses the developer machine's `gh auth` state, works with both - * github.com and GitHub Enterprise. This is the recommended default. - * - `httpGithubClient(opts)` — direct `fetch` against `api.github.com` - * with a bearer token. Useful in CI where `gh` may not be installed. - * - * Both implement the small `AutoPrClient` interface, so tests substitute - * a fake without spinning a process or network. - */ - -import { ConfigError, ValidationError } from './errors' - -export interface FileChange { - /** Repo-relative path. Forward slashes; no `..`. */ - path: string - /** New file contents. UTF-8. */ - contents: string - /** Optional explanatory comment shown in the commit body. */ - rationale?: string -} - -export interface RepoRef { - owner: string - name: string -} - -export interface ProposeAutomatedPullRequestInput { - repo: RepoRef - /** Branch to base the PR on. Default `'main'`. */ - baseBranch?: string - /** New branch name. Use a prefix + a short stable id; no spaces. */ - branchName: string - fileChanges: FileChange[] - title: string - body: string - /** Optional GitHub usernames to request review from. */ - reviewers?: string[] - /** Optional labels to apply. */ - labels?: string[] - /** Commit author name. Default: derived from the GitHub client. */ - authorName?: string - /** Commit author email. Default: derived from the GitHub client. */ - authorEmail?: string - /** Dry-run — do not push or open a PR; just return the would-be plan. */ - dryRun?: boolean -} - -export interface ProposeAutomatedPullRequestResult { - prUrl: string - branchName: string - headSha: string - dryRun: boolean -} - -/** Pluggable transport for the auto-PR pipeline. */ -export interface AutoPrClient { - /** - * Create a branch from `baseBranch`, write file changes, commit, push, - * and open a PR. Returns the PR's HTML url and head SHA. - * - * Implementations must be idempotent on `branchName`: if the branch - * already exists with the same head SHA as the would-be commit, return - * the existing PR rather than failing. This makes the production loop - * safe to retry on transient errors. - */ - proposeChange(input: ProposeAutomatedPullRequestInput): Promise -} - -/** - * Validate a proposed change before any transport touches the network. Each - * `AutoPrClient.proposeChange` runs this at its boundary so a malformed input - * (path traversal, duplicate path, branch == base, empty title) fails loud - * locally instead of producing a broken commit on the remote. - */ -function validateProposeInput(input: ProposeAutomatedPullRequestInput): void { - if (!input.repo.owner.trim() || !input.repo.name.trim()) { - throw new ValidationError('proposeChange: repo.owner and repo.name required') - } - if (!input.branchName.trim() || /\s/.test(input.branchName)) { - throw new ValidationError( - 'proposeChange: branchName must be non-empty and contain no whitespace', - ) - } - if (input.branchName === (input.baseBranch ?? 'main')) { - throw new ValidationError('proposeChange: branchName must differ from baseBranch') - } - if (input.fileChanges.length === 0) { - throw new ValidationError('proposeChange: fileChanges must not be empty') - } - const seenPaths = new Set() - for (const change of input.fileChanges) { - if (!change.path.trim() || change.path.includes('..') || change.path.startsWith('/')) { - throw new ValidationError( - `proposeChange: invalid file path "${change.path}" (no '..' or leading '/')`, - ) - } - if (seenPaths.has(change.path)) { - throw new ValidationError(`proposeChange: duplicate file path "${change.path}"`) - } - seenPaths.add(change.path) - } - if (!input.title.trim()) { - throw new ValidationError('proposeChange: title must not be empty') - } -} - -// ── HTTP transport (uses `fetch` against api.github.com) ───────────── - -export interface HttpGithubClientOptions { - /** Personal access token, GitHub App token, or `GITHUB_TOKEN` from Actions. */ - token: string - /** Override for GitHub Enterprise. Default `'https://api.github.com'`. */ - apiBase?: string - /** Test seam — defaults to global `fetch`. */ - fetchImpl?: typeof fetch - /** Test seam — clock for commit timestamps. */ - now?: () => Date -} - -interface GhRef { - ref: string - object: { sha: string } -} - -interface GhCommit { - sha: string - tree: { sha: string } -} - -interface GhBlob { - sha: string -} - -interface GhTree { - sha: string -} - -interface GhPullRequest { - html_url: string - number: number -} - -/** - * Direct REST-API GitHub client. No external deps. - * - * Idempotency strategy: before creating refs/commits/PRs, check whether - * the branch already exists at the desired tree. If so, return the - * existing PR (or open one if missing). Errors from concurrent runs - * (`Reference already exists`) are caught and treated as success. - */ -export function httpGithubClient(opts: HttpGithubClientOptions): AutoPrClient { - const fetchImpl = opts.fetchImpl ?? fetch - const apiBase = (opts.apiBase ?? 'https://api.github.com').replace(/\/+$/, '') - const now = opts.now ?? (() => new Date()) - - async function api( - method: string, - path: string, - body?: unknown, - accept404 = false, - ): Promise { - const res = await fetchImpl(`${apiBase}${path}`, { - method, - headers: { - accept: 'application/vnd.github+json', - 'content-type': 'application/json', - authorization: `Bearer ${opts.token}`, - 'x-github-api-version': '2022-11-28', - }, - body: body === undefined ? undefined : JSON.stringify(body), - }) - if (accept404 && res.status === 404) return null - if (!res.ok) { - const text = await res.text().catch(() => '') - throw new ConfigError( - `proposeChange: GitHub ${method} ${path} → ${res.status} ${text.slice(0, 400)}`, - ) - } - return (await res.json()) as T - } - - return { - async proposeChange(input) { - validateProposeInput(input) - const baseBranch = input.baseBranch ?? 'main' - const repoPath = `/repos/${input.repo.owner}/${input.repo.name}` - - if (input.dryRun) { - return { - prUrl: `https://github.com/${input.repo.owner}/${input.repo.name}/compare/${baseBranch}...${input.branchName}`, - branchName: input.branchName, - headSha: 'dry-run', - dryRun: true, - } - } - - // 1. Find base SHA - const baseRef = await api('GET', `${repoPath}/git/ref/heads/${baseBranch}`) - if (!baseRef) { - throw new ConfigError(`proposeChange: base branch "${baseBranch}" not found`) - } - const baseSha = baseRef.object.sha - const baseCommit = await api('GET', `${repoPath}/git/commits/${baseSha}`) - if (!baseCommit) { - throw new ConfigError(`proposeChange: base commit ${baseSha} not found (race condition?)`) - } - - // 2. Create blobs for each file - const treeEntries = [] - for (const change of input.fileChanges) { - const blob = await api('POST', `${repoPath}/git/blobs`, { - content: change.contents, - encoding: 'utf-8', - }) - if (!blob) throw new ConfigError('proposeChange: blob creation returned null') - treeEntries.push({ - path: change.path, - mode: '100644', - type: 'blob', - sha: blob.sha, - }) - } - - // 3. Create tree - const tree = await api('POST', `${repoPath}/git/trees`, { - base_tree: baseCommit.tree.sha, - tree: treeEntries, - }) - if (!tree) throw new ConfigError('proposeChange: tree creation returned null') - - // 4. Create commit - const author = - input.authorName && input.authorEmail - ? { name: input.authorName, email: input.authorEmail, date: now().toISOString() } - : undefined - const commitMessage = renderCommitMessage(input) - const commit = await api('POST', `${repoPath}/git/commits`, { - message: commitMessage, - tree: tree.sha, - parents: [baseSha], - ...(author ? { author, committer: author } : {}), - }) - if (!commit) throw new ConfigError('proposeChange: commit creation returned null') - - // 5. Create or fast-forward branch ref (idempotent on existing branch). - const existing = await api( - 'GET', - `${repoPath}/git/ref/heads/${input.branchName}`, - undefined, - true, - ) - if (!existing) { - await api('POST', `${repoPath}/git/refs`, { - ref: `refs/heads/${input.branchName}`, - sha: commit.sha, - }) - } else if (existing.object.sha !== commit.sha) { - await api('PATCH', `${repoPath}/git/refs/heads/${input.branchName}`, { - sha: commit.sha, - force: true, - }) - } - - // 6. Open PR (or find an existing open one for the same branch). - const openPrs = await api( - 'GET', - `${repoPath}/pulls?state=open&head=${encodeURIComponent(`${input.repo.owner}:${input.branchName}`)}`, - ) - let pr: GhPullRequest - if (openPrs && openPrs.length > 0) { - pr = openPrs[0] as GhPullRequest - } else { - const created = await api('POST', `${repoPath}/pulls`, { - title: input.title, - body: input.body, - head: input.branchName, - base: baseBranch, - }) - if (!created) throw new ConfigError('proposeChange: PR creation returned null') - pr = created - } - - if (input.reviewers && input.reviewers.length > 0) { - await api( - 'POST', - `${repoPath}/pulls/${pr.number}/requested_reviewers`, - { reviewers: input.reviewers }, - true, - ).catch(() => { - /* reviewer assignment is best-effort */ - }) - } - if (input.labels && input.labels.length > 0) { - await api( - 'POST', - `${repoPath}/issues/${pr.number}/labels`, - { labels: input.labels }, - true, - ).catch(() => { - /* label assignment is best-effort */ - }) - } - - return { - prUrl: pr.html_url, - branchName: input.branchName, - headSha: commit.sha, - dryRun: false, - } - }, - } -} - -// ── gh CLI transport (no fetch needed, re-uses developer auth) ────── - -export interface GhCliClientOptions { - /** Override the CLI binary (`gh`). For testing. */ - bin?: string - /** Working directory containing a clone of `repo`. Default: process cwd. */ - cwd?: string - /** Test seam: process spawner. Default: node:child_process spawn. */ - exec?: ( - bin: string, - args: string[], - opts: { cwd: string; stdin?: string }, - ) => Promise<{ stdout: string; stderr: string; exitCode: number }> -} - -/** - * `gh` CLI transport. Requires: - * - `gh` installed and authenticated (`gh auth status`). - * - A local clone of the repo with a clean working tree. - * - `git` on PATH. - * - * Uses `gh api` for repo metadata and `gh pr create` for the PR. The - * actual commit lands via `git`, which keeps `gh`'s footprint minimal. - */ -export function ghCliClient(opts: GhCliClientOptions = {}): AutoPrClient { - const bin = opts.bin ?? 'gh' - const cwd = opts.cwd ?? process.cwd() - const exec = opts.exec ?? defaultExec - - async function run( - cmd: string, - args: string[], - stdin?: string, - ): Promise<{ stdout: string; stderr: string }> { - const r = await exec(cmd, args, { cwd, stdin }) - if (r.exitCode !== 0) { - throw new ConfigError( - `proposeChange: ${cmd} ${args.join(' ')} failed (${r.exitCode}): ${r.stderr.trim() || r.stdout.trim()}`, - ) - } - return r - } - - return { - async proposeChange(input) { - validateProposeInput(input) - const baseBranch = input.baseBranch ?? 'main' - if (input.dryRun) { - return { - prUrl: `https://github.com/${input.repo.owner}/${input.repo.name}/compare/${baseBranch}...${input.branchName}`, - branchName: input.branchName, - headSha: 'dry-run', - dryRun: true, - } - } - - // Ensure we're working in a clean tree on the base branch. - await run('git', ['fetch', 'origin', baseBranch]) - await run('git', ['checkout', baseBranch]) - await run('git', ['reset', '--hard', `origin/${baseBranch}`]) - - // Branch (idempotent: delete if exists, then re-create from base). - await exec('git', ['branch', '-D', input.branchName], { cwd }) - await run('git', ['checkout', '-b', input.branchName]) - - // Write file changes. - const { mkdir, writeFile } = await import('node:fs/promises') - const { dirname, join, resolve } = await import('node:path') - for (const change of input.fileChanges) { - const abs = resolve(cwd, change.path) - await mkdir(dirname(abs), { recursive: true }) - await writeFile(abs, change.contents, 'utf8') - await run('git', ['add', join(change.path)]) - } - - // Commit. - const env: Record = {} - if (input.authorName) env.GIT_AUTHOR_NAME = input.authorName - if (input.authorEmail) env.GIT_AUTHOR_EMAIL = input.authorEmail - if (input.authorName) env.GIT_COMMITTER_NAME = input.authorName - if (input.authorEmail) env.GIT_COMMITTER_EMAIL = input.authorEmail - const message = renderCommitMessage(input) - await run('git', ['commit', '-m', message]) - - const headRes = await run('git', ['rev-parse', 'HEAD']) - const headSha = headRes.stdout.trim() - - // Push. - await run('git', ['push', '-f', 'origin', input.branchName]) - - // Open PR (idempotent: `gh pr create` errors if one exists). - const existing = await exec( - bin, - [ - 'pr', - 'list', - '--state', - 'open', - '--head', - input.branchName, - '--json', - 'url,number', - '--limit', - '1', - ], - { cwd }, - ) - let prUrl = '' - if (existing.exitCode === 0 && existing.stdout.trim()) { - const parsed = JSON.parse(existing.stdout) as Array<{ url: string }> - if (parsed.length > 0 && parsed[0]) prUrl = parsed[0].url - } - if (!prUrl) { - const args = [ - 'pr', - 'create', - '--title', - input.title, - '--body', - input.body, - '--base', - baseBranch, - ] - if (input.reviewers && input.reviewers.length > 0) { - args.push('--reviewer', input.reviewers.join(',')) - } - if (input.labels && input.labels.length > 0) { - args.push('--label', input.labels.join(',')) - } - const r = await run(bin, args) - const match = r.stdout.match(/https?:\/\/\S+/) - prUrl = match ? match[0] : r.stdout.trim() - } - - return { prUrl, branchName: input.branchName, headSha, dryRun: false } - }, - } -} - -async function defaultExec( - bin: string, - args: string[], - opts: { cwd: string; stdin?: string }, -): Promise<{ stdout: string; stderr: string; exitCode: number }> { - const { spawn } = await import('node:child_process') - return new Promise((resolveExec) => { - const child = spawn(bin, args, { cwd: opts.cwd }) - let stdout = '' - let stderr = '' - child.stdout.on('data', (d) => { - stdout += d.toString() - }) - child.stderr.on('data', (d) => { - stderr += d.toString() - }) - if (opts.stdin) child.stdin.end(opts.stdin) - child.on('error', (err) => { - resolveExec({ stdout, stderr: `${stderr}${err.message}`, exitCode: 1 }) - }) - child.on('close', (code) => { - resolveExec({ stdout, stderr, exitCode: code ?? 1 }) - }) - }) -} - -function renderCommitMessage(input: ProposeAutomatedPullRequestInput): string { - const lines = [input.title, ''] - for (const change of input.fileChanges) { - if (change.rationale) lines.push(`- ${change.path}: ${change.rationale}`) - } - if (lines[lines.length - 1] !== '') lines.push('') - lines.push(input.body.trim()) - return lines.join('\n').trim() -} diff --git a/src/contract/self-improve.ts b/src/contract/self-improve.ts index d3a9497f..e0033d8c 100644 --- a/src/contract/self-improve.ts +++ b/src/contract/self-improve.ts @@ -141,8 +141,7 @@ export interface SelfImproveOptions { * these unless `budget.holdoutScenarios` is set explicitly. */ scenarios: TScenario[] - /** Judge that scores artifacts. Bring your own; use `langchainJudge` - * from `/adapters/langchain` for a Runnable-shaped one. */ + /** Judge that scores artifacts. Bring your own, or wrap `llmJudge`. */ judge: JudgeConfig /** Starting surface — system prompt, JSON config, anything `MutableSurface` diff --git a/src/cost-report.test.ts b/src/cost-report.test.ts deleted file mode 100644 index 9f4ccfac..00000000 --- a/src/cost-report.test.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { describe, expect, it } from 'vitest' -import type { CostReceipt, CostReceiptInput } from './cost-ledger' -import { CostLedger, costForUsage } from './cost-ledger' -import { attachCostToReport, costReport } from './cost-report' -import { ValidationError } from './errors' - -let receiptSequence = 0 - -function receipt(channel: 'agent' | 'judge', input: CostReceiptInput): CostReceipt { - const estimated = costForUsage(input.model, input) - return { - status: 'settled', - callId: `fixture-${receiptSequence++}`, - ...input, - channel, - phase: 'test', - actor: 'fixture', - costUsd: input.actualCostUsd ?? estimated.costUsd, - costUnknown: input.actualCostUsd === undefined && estimated.costUnknown, - timestamp: 1, - } -} - -function buildLedger(): CostLedger { - return new CostLedger({ - receipts: [ - receipt('agent', { model: 'gpt-4o', inputTokens: 1000, outputTokens: 1000 }), - receipt('judge', { model: 'gpt-4o', inputTokens: 2000, outputTokens: 0 }), - receipt('judge', { model: 'made-up-zzz', inputTokens: 1000, outputTokens: 1000 }), - ], - }) -} - -describe('costReport', () => { - it('projects per-channel, total, and per-model rollups from the ledger', () => { - const report = costReport(buildLedger()) - - expect(report.perChannel.map((c) => c.channel)).toEqual(['agent', 'judge']) - const judge = report.perChannel.find((c) => c.channel === 'judge') - expect(judge?.calls).toBe(2) - expect(judge?.unpricedCalls).toBe(1) - - expect(report.total.usd).toBeCloseTo(0.0125 + 0.005, 6) - expect(report.total.unknownEntries).toBe(1) - - expect(report.perModel.map((m) => m.model)).toEqual(['gpt-4o', 'made-up-zzz']) - expect(report.perModel[0]).toEqual({ - model: 'gpt-4o', - usd: 0.0175, - entries: 2, - unpriced: false, - }) - }) - - it('flags an unpriced model unpriced:true — its $0 is never a measured zero', () => { - const report = costReport(buildLedger()) - const unpriced = report.perModel.find((m) => m.model === 'made-up-zzz') - expect(unpriced).toEqual({ model: 'made-up-zzz', usd: 0, entries: 1, unpriced: true }) - }) - - it('an actualCostUsd override clears unpriced — observed dollars are real', () => { - const ledger = new CostLedger({ - receipts: [ - receipt('agent', { - model: 'made-up-zzz', - inputTokens: 100, - outputTokens: 100, - actualCostUsd: 0.42, - }), - ], - }) - const report = costReport(ledger) - expect(report.perModel[0]).toEqual({ - model: 'made-up-zzz', - usd: 0.42, - entries: 1, - unpriced: false, - }) - expect(report.total.unknownEntries).toBe(0) - }) - - it('an empty ledger projects to zeros, never throws', () => { - const report = costReport(new CostLedger()) - expect(report).toEqual({ - perChannel: [], - total: { usd: 0, unknownEntries: 0 }, - perModel: [], - }) - }) -}) - -describe('attachCostToReport', () => { - it('stamps the projection under cost and preserves the report fields', () => { - const stamped = attachCostToReport({ verdict: 'ship', lift: 0.04 }, buildLedger()) - expect(stamped.verdict).toBe('ship') - expect(stamped.lift).toBe(0.04) - expect(stamped.cost.total.unknownEntries).toBe(1) - expect(stamped.cost.perModel).toHaveLength(2) - }) - - it('refuses to overwrite an existing cost stamp', () => { - expect(() => attachCostToReport({ cost: 'already-stamped' }, new CostLedger())).toThrow( - ValidationError, - ) - }) -}) diff --git a/src/cost-report.ts b/src/cost-report.ts deleted file mode 100644 index cb372646..00000000 --- a/src/cost-report.ts +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Program cost report — a thin projection over `CostLedger.summary()` that - * adds the per-model rollup the summary lacks, plus `attachCostToReport`, the - * one way every artifact (capsules, campaign results, diagnose reports) gets - * its cost stamp. - * - * Honesty contract carried through from the ledger: `total.unknownEntries` - * and `perModel[].unpriced` surface the costUnknown axis — a $0 from an - * unpriced model is a lower bound, never a measured zero. - */ - -import type { ChannelRollup, CostLedgerHandle } from './cost-ledger' -import { ValidationError } from './errors' - -export interface ModelCostRollup { - model: string - usd: number - entries: number - /** ≥1 entry for this model was costUnknown — `usd` is a lower bound. An - * `actualCostUsd` override clears the flag for that entry (the dollars are - * observed, even when the model has no pricing). */ - unpriced: boolean -} - -export interface CostReport { - /** Per-channel breakdown — `CostLedgerSummary.byChannel` verbatim. */ - perChannel: ChannelRollup[] - total: { - usd: number - /** Entries whose cost was unknown — non-zero means `usd` is a lower bound. */ - unknownEntries: number - } - /** Per-model spend, sorted by model id. */ - perModel: ModelCostRollup[] -} - -/** Project a ledger into the program cost report. Pure — no I/O, no clock. */ -export function costReport(ledger: CostLedgerHandle): CostReport { - const summary = ledger.summary() - const perModel = new Map() - for (const entry of ledger.list()) { - const roll = perModel.get(entry.model) ?? { - model: entry.model, - usd: 0, - entries: 0, - unpriced: false, - } - roll.usd += entry.costUsd - roll.entries += 1 - if (entry.costUnknown) roll.unpriced = true - perModel.set(entry.model, roll) - } - return { - perChannel: summary.byChannel, - total: { - usd: summary.totalCostUsd, - unknownEntries: summary.byChannel.reduce((sum, c) => sum + c.unpricedCalls, 0), - }, - perModel: [...perModel.values()].sort((a, b) => a.model.localeCompare(b.model)), - } -} - -/** - * Stamp a report-shaped object with its cost projection under the `cost` key. - * Generic so capsules, campaign results, and diagnose reports all stamp the - * same way. Throws when the report already carries a `cost` key — silently - * overwriting an existing stamp would corrupt the artifact's provenance. - */ -export function attachCostToReport( - report: R, - ledger: CostLedgerHandle, -): R & { cost: CostReport } { - if ('cost' in report) { - throw new ValidationError( - "attachCostToReport: report already has a 'cost' key — refusing to overwrite an existing stamp", - ) - } - return { ...report, cost: costReport(ledger) } -} diff --git a/src/dual-agent-bench.ts b/src/dual-agent-bench.ts deleted file mode 100644 index 09025176..00000000 --- a/src/dual-agent-bench.ts +++ /dev/null @@ -1,163 +0,0 @@ -/** - * Dual-agent convergence bench. - * - * Pattern lifted from dual-worker review loops: two agents take turns until - * they converge on a consensus artifact. One proposes, the other critiques; - * the proposer revises; repeat until a score threshold is hit or max rounds. - * - * Generalized so any two "agents" (gateways, local functions, anything with - * `propose` + `critique`) compose in. Returns convergence rounds per - * scenario + whether convergence happened. - */ - -export interface DualAgentScenario { - id: string - initialPrompt: string - /** Optional context the agents can read (e.g. source documents). */ - context?: Record -} - -export interface DualAgentRound { - roundIndex: number - proposal: string - critique: string - convergenceScore: number // 0..1 — how close to convergence -} - -export interface DualAgentScenarioResult { - scenarioId: string - converged: boolean - roundsToConverge: number | null - finalProposal: string - history: DualAgentRound[] - finalScore: number -} - -export interface DualAgentBenchConfig { - scenarios: DualAgentScenario[] - maxRounds?: number - /** Convergence threshold in 0..1 (default 0.85). */ - convergenceThreshold?: number - /** - * Propose an answer given the scenario + the critic's prior critique (if any). - * Returns the proposal string. - */ - propose: (args: { - scenario: DualAgentScenario - roundIndex: number - priorProposal?: string - priorCritique?: string - }) => Promise - /** - * Critique the proposer's current output. Returns a structured critique - * (free text) plus a convergence score: how close the proposal is to - * acceptable. 1.0 = accept, 0.0 = totally off. - */ - critique: (args: { - scenario: DualAgentScenario - roundIndex: number - proposal: string - }) => Promise<{ critique: string; convergenceScore: number }> - /** Optional per-round hook for progress + tracing. */ - onRoundComplete?: (info: { scenarioId: string; round: DualAgentRound }) => void -} - -export interface DualAgentReport { - scenarios: DualAgentScenarioResult[] - aggregate: { - convergenceRate: number // fraction of scenarios that converged within maxRounds - avgRoundsToConverge: number | null // over scenarios that DID converge - avgFinalScore: number - } - config: { - maxRounds: number - convergenceThreshold: number - } -} - -export class DualAgentBench { - async run(config: DualAgentBenchConfig): Promise { - const maxRounds = config.maxRounds ?? 5 - const threshold = config.convergenceThreshold ?? 0.85 - - if (config.scenarios.length === 0) { - throw new Error('DualAgentBench requires at least 1 scenario') - } - - const results: DualAgentScenarioResult[] = [] - - for (const scenario of config.scenarios) { - const history: DualAgentRound[] = [] - let converged = false - let roundsToConverge: number | null = null - let finalProposal = '' - let lastScore = 0 - let priorCritique: string | undefined - - for (let r = 0; r < maxRounds; r++) { - const priorProposal = history[history.length - 1]?.proposal - const proposal = await config.propose({ - scenario, - roundIndex: r, - priorProposal, - priorCritique, - }) - const { critique, convergenceScore } = await config.critique({ - scenario, - roundIndex: r, - proposal, - }) - - if (!Number.isFinite(convergenceScore) || convergenceScore < 0 || convergenceScore > 1) { - throw new Error( - `critique must return convergenceScore in [0,1]; got ${convergenceScore} for scenario ${scenario.id} round ${r}`, - ) - } - - const round: DualAgentRound = { - roundIndex: r, - proposal, - critique, - convergenceScore, - } - history.push(round) - config.onRoundComplete?.({ scenarioId: scenario.id, round }) - - finalProposal = proposal - lastScore = convergenceScore - priorCritique = critique - - if (convergenceScore >= threshold) { - converged = true - roundsToConverge = r + 1 - break - } - } - - results.push({ - scenarioId: scenario.id, - converged, - roundsToConverge, - finalProposal, - history, - finalScore: lastScore, - }) - } - - const convergedResults = results.filter((r) => r.converged) - const convergenceRate = results.length ? convergedResults.length / results.length : 0 - const avgRoundsToConverge = convergedResults.length - ? convergedResults.reduce((acc, r) => acc + (r.roundsToConverge ?? 0), 0) / - convergedResults.length - : null - const avgFinalScore = results.length - ? results.reduce((acc, r) => acc + r.finalScore, 0) / results.length - : 0 - - return { - scenarios: results, - aggregate: { convergenceRate, avgRoundsToConverge, avgFinalScore }, - config: { maxRounds, convergenceThreshold: threshold }, - } - } -} diff --git a/src/fuzz/explorer-cost.test.ts b/src/fuzz/explorer-cost.test.ts index cf0b41f7..922eb504 100644 --- a/src/fuzz/explorer-cost.test.ts +++ b/src/fuzz/explorer-cost.test.ts @@ -1,6 +1,5 @@ import { describe, expect, it } from 'vitest' import { CostLedger } from '../cost-ledger' -import { costReport } from '../cost-report' import { ValidationError } from '../errors' import { renderCapsuleHtml } from './capsule' import { BehaviorExplorer } from './explorer' @@ -95,8 +94,7 @@ describe('BehaviorExplorer cost budget', () => { expect(entry.model).toBe('gpt-4o') expect(entry.tags?.target).toBe('cost-target') } - const report = costReport(ledger) - expect(report.perModel).toEqual([{ model: 'gpt-4o', usd: 1, entries: 2, unpriced: false }]) + expect(entries.reduce((sum, entry) => sum + entry.costUsd, 0)).toBe(1) }) it('labels ledger entries unattributed when costOf names no model', async () => { diff --git a/src/fuzz/types.ts b/src/fuzz/types.ts index f2f439d0..2f420273 100644 --- a/src/fuzz/types.ts +++ b/src/fuzz/types.ts @@ -306,7 +306,7 @@ export interface ExploreOptions { * Sink for per-run cost entries — each known `costOf` result is recorded * with channel 'agent' and `actualCostUsd` (token axes are zero: the * explorer only sees dollars). Pass the program's shared `CostLedger` so - * `costReport` stamps fuzz spend alongside judge/analyst spend. + * `CostLedger.summary()` counts fuzz spend alongside judge/analyst spend. */ ledger?: CostLedgerHandle /** Observer fired for every known-cost run recorded. */ diff --git a/src/golden-matcher.ts b/src/golden-matcher.ts deleted file mode 100644 index d80a847b..00000000 --- a/src/golden-matcher.ts +++ /dev/null @@ -1,159 +0,0 @@ -/** - * GoldenMatcher — fuzzy matcher for "did the agent produce the expected things?". - * - * Universal primitive across agent-eval consumers. Use it for: - * - Test suites: did the run hit the expected assertions? - * - Tool agents: did the agent emit the expected tool call sequence? - * - Judges: did the verdict include the expected concepts? - * - Design audits: did the auditor surface the planted defects? - * - * Match rule (per golden): - * - Any phrase in `golden.any` (case-insensitive substring) appears in the - * candidate's text fields, OR - * - Any pattern in `golden.anyRegex` (case-insensitive) matches. - * - * Recall is severity-weighted by default: critical=3, major=2, minor=1. - * Missing one critical hurts more than missing three minors. - */ - -export type GoldenSeverity = 'critical' | 'major' | 'minor' - -export interface GoldenSpec { - /** Stable identifier — survives across runs so consumers can grep by id. */ - id: string - /** Severity drives recall weighting. */ - severity: GoldenSeverity - /** - * Substring phrases (case-insensitive). A hit on ANY phrase counts as a - * match. Keep these SHORT (3-6 words) and SPECIFIC. - */ - any: string[] - /** Optional regex patterns. ORed with `any`. */ - anyRegex?: string[] - /** Free-form note — surfaces in reports for humans. */ - hint?: string - /** Optional category for grouping/filtering. */ - category?: string -} - -export interface MatchResult { - /** Same length as goldens; `true` when matched. */ - matches: boolean[] - /** Convenience: count of hits. */ - hits: number - /** Convenience: total goldens. */ - total: number -} - -/** - * Match each golden against `candidates`, where each candidate exposes one or - * more text fields the matcher should search. Defaults to searching all - * string-typed fields concatenated. - */ -export function matchGoldens( - goldens: GoldenSpec[], - candidates: T[], - options: { - /** - * Extract the searchable text for a candidate. Default: concatenate every - * top-level string field with a space. - */ - text?: (candidate: T) => string - } = {}, -): MatchResult { - const extract = options.text ?? defaultExtract - const haystacks = candidates.map((c) => extract(c).toLowerCase()) - const matches = goldens.map((golden) => goldenMatched(golden, haystacks)) - return { - matches, - hits: matches.filter(Boolean).length, - total: goldens.length, - } -} - -function defaultExtract(candidate: unknown): string { - if (typeof candidate === 'string') return candidate - if (candidate && typeof candidate === 'object') { - const parts: string[] = [] - for (const v of Object.values(candidate as Record)) { - if (typeof v === 'string') parts.push(v) - } - return parts.join(' ') - } - return String(candidate ?? '') -} - -function goldenMatched(golden: GoldenSpec, haystacks: string[]): boolean { - for (const phrase of golden.any) { - const needle = phrase.toLowerCase().trim() - if (!needle) continue - if (haystacks.some((h) => h.includes(needle))) return true - } - for (const pattern of golden.anyRegex ?? []) { - let re: RegExp - try { - re = new RegExp(pattern, 'i') - } catch { - continue - } - if (haystacks.some((h) => re.test(h))) return true - } - return false -} - -/** Severity weights — exposed so consumers can override (rare). */ -export const DEFAULT_SEVERITY_WEIGHTS: Record = { - critical: 3, - major: 2, - minor: 1, -} - -/** Severity-weighted recall over a MatchResult + the goldens that produced it. */ -export function weightedRecall( - goldens: GoldenSpec[], - result: MatchResult, - weights: Record = DEFAULT_SEVERITY_WEIGHTS, -): number { - if (goldens.length === 0) return 1 - const total = goldens.reduce((s, g) => s + (weights[g.severity] ?? 1), 0) - if (total === 0) return 1 - const hit = goldens.reduce( - (s, g, i) => s + (result.matches[i] ? (weights[g.severity] ?? 1) : 0), - 0, - ) - return hit / total -} - -/** - * Precision proxy: fraction of emitted candidates that match SOME golden. - * - * No human-labelled negatives means unmatched candidates are SOFT false - * positives — punishes verbose agents that pad with filler. Doesn't punish - * unknown-but-real findings; the way to tighten this is to grow the golden - * set, not to invent a stricter score. - */ -export function precision( - goldens: GoldenSpec[], - candidates: T[], - options: { text?: (candidate: T) => string } = {}, -): number { - if (candidates.length === 0) return 1 - const extract = options.text ?? defaultExtract - let matched = 0 - for (const cand of candidates) { - const haystack = extract(cand).toLowerCase() - const matchedAny = goldens.some( - (g) => - g.any.some((phrase) => phrase.length > 0 && haystack.includes(phrase.toLowerCase())) || - (g.anyRegex ?? []).some((pat) => { - try { - return new RegExp(pat, 'i').test(haystack) - } catch { - return false - } - }), - ) - if (matchedAny) matched++ - } - return matched / candidates.length -} diff --git a/src/judge-runner.test.ts b/src/judge-runner.test.ts deleted file mode 100644 index 539914f5..00000000 --- a/src/judge-runner.test.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { compilerJudge, runJudgeFleet } from './judge-runner' -import type { HarnessConfig, SandboxDriver, SandboxResult } from './sandbox-harness' - -/** - * Driver that records peak concurrent `exec` calls. Each exec yields to the - * event loop a few times so the pool's bound is observable: with unbounded - * fan-out every spec's subprocess is in-flight at once. - */ -function makeConcurrencyTrackingDriver(): SandboxDriver & { peak: number; active: number } { - const state = { id: 'tracking', peak: 0, active: 0 } as SandboxDriver & { - peak: number - active: number - } - state.exec = async ( - phase: SandboxResult['phase'], - _command: string, - _config: HarnessConfig, - ): Promise => { - state.active++ - if (state.active > state.peak) state.peak = state.active - // Yield repeatedly so other workers can start while we're "running". - for (let i = 0; i < 5; i++) await Promise.resolve() - state.active-- - return { phase, exitCode: 0, stdout: '', stderr: '', wallMs: 1 } - } - return state -} - -describe('runJudgeFleet bounded concurrency', () => { - it('caps concurrent subprocesses at the configured concurrency', async () => { - const driver = makeConcurrencyTrackingDriver() - const specs = Array.from({ length: 12 }, (_, i) => - compilerJudge(`judge-${i}`, { runCommand: 'true' } as HarnessConfig), - ) - - const results = await runJudgeFleet(specs, { driver, concurrency: 3 }) - - expect(results).toHaveLength(12) - // OLD behavior (Promise.all over all specs) would peak at 12. The bounded - // pool must never exceed the configured concurrency. - expect(driver.peak).toBeLessThanOrEqual(3) - expect(driver.peak).toBeGreaterThan(0) - }) - - it('preserves result order matching input spec order', async () => { - const driver = makeConcurrencyTrackingDriver() - const specs = Array.from({ length: 8 }, (_, i) => - compilerJudge(`spec-${i}`, { runCommand: 'true' } as HarnessConfig), - ) - const results = await runJudgeFleet(specs, { driver, concurrency: 2 }) - expect(results.map((r) => r.id)).toEqual(specs.map((s) => s.id)) - }) - - it('returns [] for an empty spec list without spawning workers', async () => { - const driver = makeConcurrencyTrackingDriver() - const results = await runJudgeFleet([], { driver }) - expect(results).toEqual([]) - expect(driver.peak).toBe(0) - }) -}) diff --git a/src/judge-runner.ts b/src/judge-runner.ts deleted file mode 100644 index da5f1479..00000000 --- a/src/judge-runner.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { cpus } from 'node:os' -import { - type HarnessConfig, - type SandboxDriver, - SandboxHarness, - type SandboxHarnessResult, - SubprocessSandboxDriver, -} from './sandbox-harness' -import { TraceEmitter } from './trace/emitter' -import { InMemoryTraceStore } from './trace/store' - -export type SandboxJudgeKind = 'compiler' | 'test' | 'linter' | 'security' - -export interface SandboxJudgeSpec { - id: string - kind: SandboxJudgeKind - config: HarnessConfig -} - -export interface SandboxJudgeResult { - id: string - kind: SandboxJudgeKind - passed: boolean - score: number - summary: string - detail: SandboxHarnessResult -} - -export interface JudgeFleetOptions { - driver?: SandboxDriver - parallel?: boolean - /** - * Max concurrent judge subprocesses. Each spec spawns its own subprocess - * via the driver, so unbounded fan-out exhausts file descriptors / PIDs. - * Defaults to the host CPU count. Ignored when `parallel === false`. - */ - concurrency?: number -} - -export class JudgeRunner { - private readonly driver: SandboxDriver - - constructor(driver: SandboxDriver = new SubprocessSandboxDriver()) { - this.driver = driver - } - - async run(spec: SandboxJudgeSpec): Promise { - const store = new InMemoryTraceStore() - const emitter = new TraceEmitter(store, { runId: `judge-${spec.id}` }) - await emitter.startRun({ - scenarioId: spec.id, - layer: 'meta', - projectId: 'judge-runner', - }) - const harness = new SandboxHarness(this.driver) - const detail = await harness.run(spec.config, emitter) - await emitter.endRun({ pass: detail.passed, score: detail.score, notes: `${spec.kind} judge` }) - return { - id: spec.id, - kind: spec.kind, - passed: detail.passed, - score: detail.score, - summary: renderJudgeSummary(spec.kind, detail), - detail, - } - } -} - -export async function runJudgeFleet( - specs: SandboxJudgeSpec[], - options: JudgeFleetOptions = {}, -): Promise { - const runner = new JudgeRunner(options.driver) - if (options.parallel === false) { - const results: SandboxJudgeResult[] = [] - for (const spec of specs) results.push(await runner.run(spec)) - return results - } - const concurrency = Math.max(1, options.concurrency ?? cpus().length) - const results: SandboxJudgeResult[] = new Array(specs.length) - let cursor = 0 - async function worker(): Promise { - while (true) { - const i = cursor++ - if (i >= specs.length) return - results[i] = await runner.run(specs[i]!) - } - } - await Promise.all(Array.from({ length: Math.min(concurrency, specs.length) }, () => worker())) - return results -} - -/** Build a `SandboxJudgeSpec` that scores whether the harness compiles without errors. */ -export function compilerJudge(id: string, config: HarnessConfig): SandboxJudgeSpec { - return { id, kind: 'compiler', config } -} - -/** Build a `SandboxJudgeSpec` that scores the harness by its test-suite pass rate. */ -export function testJudge(id: string, config: HarnessConfig): SandboxJudgeSpec { - return { id, kind: 'test', config } -} - -/** Build a `SandboxJudgeSpec` that scores the harness by linter rule violations. */ -export function linterJudge(id: string, config: HarnessConfig): SandboxJudgeSpec { - return { id, kind: 'linter', config } -} - -/** Build a `SandboxJudgeSpec` that scores the harness output for security issues via a security scanner. */ -export function securityJudge(id: string, config: HarnessConfig): SandboxJudgeSpec { - return { id, kind: 'security', config } -} - -function renderJudgeSummary(kind: SandboxJudgeKind, detail: SandboxHarnessResult): string { - if (!detail.passed) return `${kind} judge failed` - if (detail.test?.testsTotal) - return `${kind} judge passed ${detail.test.testsPassed}/${detail.test.testsTotal} tests` - return `${kind} judge passed` -} diff --git a/src/knowledge/index.ts b/src/knowledge/index.ts deleted file mode 100644 index f2c58091..00000000 --- a/src/knowledge/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './readiness' -export * from './types' diff --git a/src/multi-layer-verifier.ts b/src/multi-layer-verifier.ts index d1447c37..907bbf21 100644 --- a/src/multi-layer-verifier.ts +++ b/src/multi-layer-verifier.ts @@ -1,11 +1,9 @@ /** * Multi-layer verifier — ordered pipeline of verification layers. * - * Different contract from {@link JudgeRunner} (which runs parallel - * specs against a sandbox). MultiLayerVerifier is a DAG of layers - * (install → typecheck → build → lint → serve → semantic → …) with - * dependency-based skip, per-layer findings, soft-fail semantics, and - * an aggregated `blendedScore` across all passed layers. + * A DAG of layers (install → typecheck → build → lint → serve → semantic → …) + * with dependency-based skip, per-layer findings, soft-fail semantics, and an + * aggregated `blendedScore` across all passed layers. * * Use when you want: * - ordered stages where a failing upstream stage skips downstream ones @@ -13,13 +11,8 @@ * - a single composite score across stages with per-stage weights * - soft-fail stages whose failure doesn't abort the pipeline * - * Use {@link JudgeRunner} when you want: - * - N independent judges running in parallel against the same artifact - * - no inter-judge dependencies - * - boolean `passed` per judge + overall - * - * Both primitives compose — JudgeRunner can be invoked as a single - * layer inside a MultiLayerVerifier if that suits the caller. + * A layer body is an arbitrary async function, so independent checks that need + * no ordering run as one layer that resolves them in parallel. */ import { packageVersion } from './package-version' @@ -38,8 +31,8 @@ export interface Finding { /** Optional layer name the finding belongs to (set by the verifier if omitted). */ layer?: string /** - * Free-form structured payload — used by `multiToolchainLayer` to attach - * `{ adapter: 'pnpm' }`, by judges to attach evidence pointers, etc. + * Free-form structured payload — a layer attaches its toolchain + * (`{ adapter: 'pnpm' }`), a judge attaches evidence pointers, etc. * Renderers MAY interrogate; agent-eval primitives never assume shape. */ detail?: Record diff --git a/src/multi-toolchain-layer.test.ts b/src/multi-toolchain-layer.test.ts deleted file mode 100644 index bfc735f2..00000000 --- a/src/multi-toolchain-layer.test.ts +++ /dev/null @@ -1,196 +0,0 @@ -import { describe, expect, it } from 'vitest' -import type { LayerResult } from './multi-layer-verifier' -import { mergeLayerResults, multiToolchainLayer } from './multi-toolchain-layer' - -function mkResult( - status: LayerResult['status'], - score?: number, - findings: LayerResult['findings'] = [], -): LayerResult { - return { - layer: 'install', - status, - score, - durationMs: 100, - findings, - } -} - -describe('mergeLayerResults', () => { - it('skipped when no adapters', () => { - const r = mergeLayerResults('install', []) - expect(r.status).toBe('skipped') - expect(r.findings).toHaveLength(0) - expect(r.reason).toBe('no adapters') - }) - - it('passes through when single adapter (preserves findings + reason)', () => { - const r = mergeLayerResults('install', [ - { - adapter: 'pnpm', - result: { - ...mkResult('pass', 1), - findings: [{ severity: 'info', layer: 'install', message: 'pnpm install ok' }], - }, - }, - ]) - expect(r.status).toBe('pass') - expect(r.score).toBe(1) - expect(r.findings[0]!.detail).toEqual({ adapter: 'pnpm' }) - }) - - it('worst-of-parts status reduction', () => { - const r = mergeLayerResults('install', [ - { adapter: 'pnpm', result: mkResult('pass', 1) }, - { adapter: 'npm', result: mkResult('fail', 0) }, - ]) - expect(r.status).toBe('fail') - }) - - it('error > timeout > fail > skipped > pass', () => { - const cases: Array<[Array, LayerResult['status']]> = [ - [['pass', 'skipped'], 'skipped'], - [['pass', 'fail'], 'fail'], - [['fail', 'timeout'], 'timeout'], - [['timeout', 'error'], 'error'], - [['pass', 'pass'], 'pass'], - ] - for (const [statuses, expected] of cases) { - const r = mergeLayerResults( - 'x', - statuses.map((s, i) => ({ adapter: `a${i}`, result: mkResult(s) })), - ) - expect(r.status).toBe(expected) - } - }) - - it('weighted-mean score across numeric adapters; skips contribute null', () => { - const r = mergeLayerResults('build', [ - { adapter: 'pnpm', result: mkResult('pass', 0.8) }, - { adapter: 'npm', result: mkResult('pass', 1.0) }, - { adapter: 'forge', result: mkResult('skipped') }, - ]) - expect(r.score).toBeCloseTo(0.9, 2) - }) - - it('attributes findings with detail.adapter', () => { - const r = mergeLayerResults('typecheck', [ - { - adapter: 'pnpm', - result: mkResult('fail', 0, [ - { severity: 'major', layer: 'typecheck', message: 'tsc 4 errors' }, - ]), - }, - { - adapter: 'forge', - result: mkResult('pass', 1, [ - { severity: 'info', layer: 'typecheck', message: 'forge ok' }, - ]), - }, - ]) - expect(r.findings).toHaveLength(2) - expect(r.findings.find((f) => f.message === 'tsc 4 errors')?.detail).toMatchObject({ - adapter: 'pnpm', - }) - expect(r.findings.find((f) => f.message === 'forge ok')?.detail).toMatchObject({ - adapter: 'forge', - }) - }) - - it('reason concatenates adapter:status; durationMs is max-of-parts', () => { - const r = mergeLayerResults('install', [ - { adapter: 'pnpm', result: { ...mkResult('pass', 1), durationMs: 5000 } }, - { adapter: 'npm', result: { ...mkResult('skipped'), durationMs: 100 } }, - ]) - expect(r.reason).toBe('pnpm: pass · npm: skipped') - expect(r.durationMs).toBe(5000) - }) -}) - -describe('multiToolchainLayer', () => { - it('runs adapters in parallel + merges results', async () => { - const calls: string[] = [] - const layer = multiToolchainLayer({ - name: 'install', - adapters: [{ id: 'pnpm' }, { id: 'npm' }, { id: 'forge' }], - adapterName: (a) => a.id, - run: async (a) => { - calls.push(a.id) - return mkResult(a.id === 'forge' ? 'fail' : 'pass', a.id === 'forge' ? 0 : 1) - }, - }) - const r = await layer.run({ env: null, prior: {}, signal: new AbortController().signal }) - expect(r.status).toBe('fail') - expect(calls.sort()).toEqual(['forge', 'npm', 'pnpm']) - expect(r.detail).toMatchObject({ - adapters: expect.arrayContaining([ - expect.objectContaining({ adapter: 'pnpm', status: 'pass' }), - ]), - }) - }) - - it('catches per-adapter throws as status=error (rest of layer still completes)', async () => { - const layer = multiToolchainLayer({ - name: 'install', - adapters: ['pnpm', 'cursed'], - adapterName: (a) => a, - run: async (a) => { - if (a === 'cursed') throw new Error('boom') - return mkResult('pass', 1) - }, - }) - const r = await layer.run({ env: null, prior: {}, signal: new AbortController().signal }) - expect(r.status).toBe('error') // worst-of (pass + error) - const cursed = r.findings.find( - (f) => f.detail && (f.detail as Record).adapter === 'cursed', - ) - expect(cursed?.message).toBe('boom') - }) - - it('skipped result on zero adapters (no toolchain detected)', async () => { - const layer = multiToolchainLayer({ - name: 'install', - adapters: [], - adapterName: (a) => a, - run: async () => mkResult('pass', 1), - }) - const r = await layer.run({ env: null, prior: {}, signal: new AbortController().signal }) - expect(r.status).toBe('skipped') - expect(r.reason).toMatch(/no adapters detected/) - }) - - it('respects maxParallel — chunks calls', async () => { - let inFlight = 0 - let peak = 0 - const layer = multiToolchainLayer({ - name: 'x', - adapters: [1, 2, 3, 4, 5, 6, 7, 8], - adapterName: (a) => `a${a}`, - maxParallel: 2, - run: async () => { - inFlight++ - peak = Math.max(peak, inFlight) - await new Promise((r) => setTimeout(r, 10)) - inFlight-- - return mkResult('pass', 1) - }, - }) - await layer.run({ env: null, prior: {}, signal: new AbortController().signal }) - expect(peak).toBeLessThanOrEqual(2) - }) - - it('passes verify ctx through to adapter run fn', async () => { - const seen: unknown[] = [] - const layer = multiToolchainLayer<{ marker: number }, string>({ - name: 'x', - adapters: ['pnpm'], - adapterName: (a) => a, - run: async (_a, ctx) => { - seen.push(ctx.env.marker) - return mkResult('pass', 1) - }, - }) - await layer.run({ env: { marker: 42 }, prior: {}, signal: new AbortController().signal }) - expect(seen).toEqual([42]) - }) -}) diff --git a/src/multi-toolchain-layer.ts b/src/multi-toolchain-layer.ts deleted file mode 100644 index ccf3eb04..00000000 --- a/src/multi-toolchain-layer.ts +++ /dev/null @@ -1,266 +0,0 @@ -/** - * Multi-toolchain layer factory + merge helper. - * - * Some verification stages (install, typecheck, build, lint) run the - * SAME logical layer across multiple parallel adapters — pnpm AND npm - * AND cargo AND forge for a polyglot scaffold. The verifier presents - * one row per stage; the toolchain breakdown lives in `findings.detail`. - * - * This module provides the merge: take N independent `LayerResult`s - * (one per adapter) and reduce them to a single `LayerResult` whose - * status is the worst of the parts and whose findings cite the adapter - * that produced each one. Plus a {@link multiToolchainLayer} factory - * that runs the adapter calls in parallel + applies the reducer. - * - * Pure utility — composes with {@link MultiLayerVerifier}.{run}. - */ - -import type { - Layer, - LayerResult, - LayerStatus, - Severity, - VerifyContext, -} from './multi-layer-verifier' - -// ─── Status reduction ────────────────────────────────────────────────── - -const STATUS_RANK: Record = { - pass: 0, - skipped: 1, - fail: 2, - timeout: 3, - error: 4, -} - -function worst(a: LayerStatus, b: LayerStatus): LayerStatus { - return (STATUS_RANK[a] ?? 0) >= (STATUS_RANK[b] ?? 0) ? a : b -} - -const SEVERITY_RANK: Record = { - info: 0, - minor: 1, - major: 2, - critical: 3, -} - -function maxSeverity(findings: ReadonlyArray<{ severity: Severity }>): Severity { - let best: Severity = 'info' - for (const f of findings) { - if (SEVERITY_RANK[f.severity] > SEVERITY_RANK[best]) best = f.severity - } - return best -} - -// ─── Merge ────────────────────────────────────────────────────────────── - -export interface AdapterRun { - /** Identifier for the adapter (e.g. 'pnpm', 'npm', 'cargo', 'forge'). */ - adapter: string - result: LayerResult -} - -export interface MergeOptions { - /** - * How to combine per-adapter `durationMs`. Default `'max'` (parallel - * wall-clock). Set `'sum'` when reporting total work done across - * adapters rather than wall time. - */ - mergeDuration?: 'max' | 'sum' - /** - * Prefix finding messages with a per-adapter tag (e.g. `[pnpm] typecheck failed`). - * Default: no prefix (renderers read `detail.adapter` instead). - */ - messagePrefixer?: (adapter: string) => string - /** - * How to reduce per-adapter `LayerResult.diagnostics` into the merged - * result's diagnostics. `'max'` (default) — for each key, merged = - * max across adapters where value is non-null (matches "if ANY adapter - * saw N errors, merged saw N"). `'sum'` — sum non-null values. - */ - mergeDiagnostics?: 'max' | 'sum' -} - -/** - * Reduce N adapter runs to a single `LayerResult` for a logical layer. - * - * - status: worst of the parts (pass < skipped < fail < timeout < error) - * - score: weighted mean of numeric scores (skip = no contribution) - * - findings: union, each tagged with `detail.adapter` - * - durationMs: `mergeDuration` option (default 'max' for parallel wall-clock) - * - diagnostics: `mergeDiagnostics` option (default 'max' per key) - * - reason: " · "-joined `name: status` per adapter - */ -export function mergeLayerResults( - name: string, - perAdapter: AdapterRun[], - options: MergeOptions = {}, -): LayerResult { - const mergeDuration = options.mergeDuration ?? 'max' - const mergeDiagnostics = options.mergeDiagnostics ?? 'max' - const prefix = options.messagePrefixer - - if (perAdapter.length === 0) { - return { - layer: name, - status: 'skipped', - durationMs: 0, - findings: [], - reason: 'no adapters', - } - } - if (perAdapter.length === 1) { - const only = perAdapter[0]! - return { - ...only.result, - layer: name, - findings: only.result.findings.map((f) => ({ - ...f, - layer: name, - message: prefix ? `${prefix(only.adapter)} ${f.message}` : f.message, - detail: { ...(f.detail ?? {}), adapter: only.adapter }, - })), - reason: only.result.reason ?? `${only.adapter}: ${only.result.status}`, - } - } - - let status: LayerStatus = 'pass' - let weightedScoreSum = 0 - let weightCount = 0 - const findings: LayerResult['findings'] = [] - let durationMs = 0 - const reasonParts: string[] = [] - const diagnostics: Record = {} - - for (const { adapter, result } of perAdapter) { - status = worst(status, result.status) - if (typeof result.score === 'number') { - weightedScoreSum += result.score - weightCount += 1 - } - durationMs = - mergeDuration === 'sum' - ? durationMs + result.durationMs - : Math.max(durationMs, result.durationMs) - reasonParts.push(`${adapter}: ${result.status}`) - for (const f of result.findings) { - findings.push({ - ...f, - layer: name, - message: prefix ? `${prefix(adapter)} ${f.message}` : f.message, - detail: { ...(f.detail ?? {}), adapter }, - }) - } - for (const [k, v] of Object.entries(result.diagnostics ?? {})) { - if (typeof v !== 'number' || !Number.isFinite(v)) continue - const prev = diagnostics[k] - if (prev == null) diagnostics[k] = v - else diagnostics[k] = mergeDiagnostics === 'sum' ? prev + v : Math.max(prev, v) - } - } - - return { - layer: name, - status, - score: weightCount > 0 ? weightedScoreSum / weightCount : undefined, - durationMs, - findings, - reason: reasonParts.join(' · '), - diagnostics: Object.keys(diagnostics).length > 0 ? diagnostics : undefined, - detail: { - adapters: perAdapter.map(({ adapter, result }) => ({ - adapter, - status: result.status, - score: result.score ?? null, - })), - worstSeverity: maxSeverity(findings), - }, - } -} - -// ─── Layer factory ────────────────────────────────────────────────────── - -export interface MultiToolchainLayerConfig { - name: string - adapters: ReadonlyArray - /** Adapter identifier — used in findings + reason. */ - adapterName: (a: Adapter) => string - /** Run a single adapter against the verify context. */ - run: (a: Adapter, ctx: VerifyContext) => Promise | LayerResult - dependsOn?: string[] - weight?: number - failContributesToScore?: boolean - capMs?: number - /** - * Per-adapter parallel cap. Defaults to 8 — defense in depth against a - * caller passing 50 adapters and fanning out 50 simultaneous subprocesses. - * Adapters that need higher concurrency raise this explicitly. - */ - maxParallel?: number -} - -/** - * Build a {@link Layer} that fans the same logical stage across N adapters - * in parallel and merges via {@link mergeLayerResults}. - * - * Per-adapter throws are caught + converted to `status: 'error'` results - * so one bad adapter doesn't poison the whole layer. - */ -export function multiToolchainLayer( - config: MultiToolchainLayerConfig, -): Layer { - const maxParallel = Math.max(1, config.maxParallel ?? 8) - return { - name: config.name, - dependsOn: config.dependsOn, - weight: config.weight, - failContributesToScore: config.failContributesToScore, - capMs: config.capMs, - async run(ctx) { - if (config.adapters.length === 0) { - return { - layer: config.name, - status: 'skipped', - durationMs: 0, - findings: [], - reason: 'no adapters detected', - } - } - - const runOne = async (adapter: Adapter): Promise => { - const adapterName = config.adapterName(adapter) - try { - const r = await config.run(adapter, ctx) - return { adapter: adapterName, result: r } - } catch (err) { - return { - adapter: adapterName, - result: { - layer: config.name, - status: 'error', - durationMs: 0, - findings: [ - { - severity: 'major', - layer: config.name, - message: err instanceof Error ? err.message : String(err), - detail: { adapter: adapterName }, - }, - ], - reason: err instanceof Error ? err.message : String(err), - }, - } - } - } - - // Bounded parallelism — chunked into groups of size maxParallel. - const results: AdapterRun[] = [] - for (let i = 0; i < config.adapters.length; i += maxParallel) { - const chunk = config.adapters.slice(i, i + maxParallel) - const chunkResults = await Promise.all(chunk.map(runOne)) - results.push(...chunkResults) - } - return mergeLayerResults(config.name, results) - }, - } -} diff --git a/src/prm/builtin-rubrics.ts b/src/prm/builtin-rubrics.ts deleted file mode 100644 index 003e0104..00000000 --- a/src/prm/builtin-rubrics.ts +++ /dev/null @@ -1,134 +0,0 @@ -/** - * Built-in reference rubrics. Consumers combine these with domain - * rubrics. All are deterministic, rule-based — cheap to run + easy - * to unit-test. LLM-based rubrics are trivially authored by - * following the StepRubric contract. - */ - -import { hasCapturedToolArgs } from '../trace/query' -import type { LlmSpan, ToolSpan } from '../trace/schema' -import type { StepRubric } from './rubric' - -/** Penalize very short or very long assistant outputs. */ -export function outputLengthRubric( - args: { minChars?: number; maxChars?: number; weight?: number } = {}, -): StepRubric { - const min = args.minChars ?? 20 - const max = args.maxChars ?? 8000 - return { - id: 'output-length', - kinds: ['llm'], - weight: args.weight ?? 0.5, - async grade({ step }) { - const llm = step.span as LlmSpan - const len = (llm.output ?? '').length - if (len === 0) return { score: 0, rationale: 'empty output' } - if (len < min) - return { score: Math.max(0, len / min), rationale: `below min (${len} < ${min})` } - if (len > max) - return { - score: Math.max(0, 1 - (len - max) / max), - rationale: `above max (${len} > ${max})`, - } - return { score: 1, rationale: `${len} chars in bounds` } - }, - } -} - -/** Reward tool calls that succeeded (status='ok') with an informative result. */ -export function toolSuccessRubric(args: { weight?: number } = {}): StepRubric { - return { - id: 'tool-success', - kinds: ['tool'], - weight: args.weight ?? 1, - async grade({ step }) { - const tool = step.span as ToolSpan - if (tool.status === 'error') - return { score: 0, rationale: `error: ${tool.error ?? 'unknown'}` } - const r = tool.result - if (r === null || r === undefined) return { score: 0.3, rationale: 'empty result' } - const asText = typeof r === 'string' ? r : JSON.stringify(r) - if (asText.length < 4) return { score: 0.5, rationale: 'tiny result' } - return { score: 1, rationale: `${tool.toolName} ok` } - }, - } -} - -/** Penalize tool calls that duplicate a prior call with identical args. */ -export function toolNonRedundantRubric(args: { weight?: number } = {}): StepRubric { - const weight = args.weight ?? 0.5 - return { - id: 'tool-non-redundant', - kinds: ['tool'], - weight, - async grade({ step, prior }) { - const tool = step.span as ToolSpan - if (!hasCapturedToolArgs(tool)) return null - const priorMatches = prior.filter((p) => { - if (p.span.kind !== 'tool') return false - const pt = p.span as ToolSpan - return ( - hasCapturedToolArgs(pt) && - pt.toolName === tool.toolName && - stableStringify(pt.args) === stableStringify(tool.args) - ) - }) - if (priorMatches.length === 0) return { score: 1, rationale: 'novel call' } - return { - score: Math.max(0, 1 - priorMatches.length * 0.5), - rationale: `${priorMatches.length} duplicate(s)`, - } - }, - } -} - -/** Penalize LLM outputs that contain common refusal markers when a refusal - * is NOT expected (caller inverts weight for scenarios where refusal IS expected). */ -export function nonRefusalRubric(args: { markers?: RegExp[]; weight?: number } = {}): StepRubric { - const weight = args.weight ?? 1 - const markers = args.markers ?? [ - /\bi\s+(?:can(?:not|'t)|won't|will\s+not)\b/i, - /\b(?:as\s+an?\s+)?ai\b.*?\b(?:can't|cannot)\b/i, - ] - return { - id: 'non-refusal', - kinds: ['llm'], - weight, - async grade({ step }) { - const llm = step.span as LlmSpan - const out = llm.output ?? '' - const refused = markers.some((re) => re.test(out)) - return refused - ? { score: 0, rationale: 'refusal marker present' } - : { score: 1, rationale: 'no refusal' } - }, - } -} - -/** Reward outputs that invoke the next-step tool the trajectory actually uses - * (i.e. the LLM span announced "I will call X" and the following tool span IS X). */ -export function toolIntentAlignmentRubric(args: { weight?: number } = {}): StepRubric { - return { - id: 'tool-intent-alignment', - kinds: ['llm'], - weight: args.weight ?? 0.5, - async grade({ step, next }) { - const llm = step.span as LlmSpan - const nextTool = next.find((s) => s.span.kind === 'tool') - if (!nextTool) return null - const toolName = (nextTool.span as ToolSpan).toolName - const out = (llm.output ?? '').toLowerCase() - const mentioned = out.includes(toolName.toLowerCase()) - return mentioned - ? { score: 1, rationale: `mentioned "${toolName}" before calling it` } - : { score: 0.5, rationale: `called "${toolName}" without announcing it` } - }, - } -} - -function stableStringify(value: unknown): string { - if (value === null || typeof value !== 'object') return JSON.stringify(value) - if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]` - const keys = Object.keys(value as Record).sort() - return `{${keys.map((k) => `${JSON.stringify(k)}:${stableStringify((value as Record)[k])}`).join(',')}}` -} diff --git a/src/prm/index.ts b/src/prm/index.ts deleted file mode 100644 index 394f0d35..00000000 --- a/src/prm/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './builtin-rubrics' -export * from './inference' -export * from './rubric' -export * from './training-export' diff --git a/src/prm/inference.test.ts b/src/prm/inference.test.ts deleted file mode 100644 index ac1e045e..00000000 --- a/src/prm/inference.test.ts +++ /dev/null @@ -1,113 +0,0 @@ -import { describe, expect, it } from 'vitest' -import type { TraceStore } from '../trace/store' -import { prmBestOfN, prmEnsembleBestOfN } from './inference' -import type { PrmGradedTrace, PrmGrader } from './rubric' - -const fakeStore = {} as TraceStore - -function tracedTrace(runId: string, score: number): PrmGradedTrace { - return { runId, steps: [], aggregateScore: score, gradedCount: 0, ungradedCount: 0 } -} - -interface ConcurrencyMeter { - peak: number - active: number -} - -/** - * Grader stub that records peak concurrent `grade` calls against a shared - * meter and assigns a score per runId. A shared meter lets multiple graders - * report the true simultaneous in-flight count across the whole pool. Each - * call yields a few times so concurrency is observable. - */ -function makeTrackingGrader( - scores: Record, - meter: ConcurrencyMeter = { peak: 0, active: 0 }, -): PrmGrader & { meter: ConcurrencyMeter } { - const state = { meter } as PrmGrader & { meter: ConcurrencyMeter } - state.grade = async (_store: TraceStore, runId: string): Promise => { - meter.active++ - if (meter.active > meter.peak) meter.peak = meter.active - for (let i = 0; i < 5; i++) await Promise.resolve() - meter.active-- - return tracedTrace(runId, scores[runId] ?? 0) - } - return state -} - -/** Grader whose `grade` always rejects — models an unrecoverable LLM failure. */ -function makeFailingGrader(message: string): PrmGrader { - const state = {} as PrmGrader - state.grade = async (): Promise => { - throw new Error(message) - } - return state -} - -describe('prmBestOfN bounded concurrency', () => { - it('caps concurrent grade calls at the configured concurrency', async () => { - const grader = makeTrackingGrader({}) - const runIds = Array.from({ length: 10 }, (_, i) => `run-${i}`) - - const result = await prmBestOfN(fakeStore, grader, runIds, { concurrency: 2 }) - - expect(result.ranked).toHaveLength(10) - // OLD behavior (Promise.all over all runIds) peaks at 10. - expect(grader.meter.peak).toBeLessThanOrEqual(2) - expect(grader.meter.peak).toBeGreaterThan(0) - }) - - it('still picks the highest-scoring candidate as winner', async () => { - const grader = makeTrackingGrader({ 'run-0': 0.1, 'run-1': 0.9, 'run-2': 0.5 }) - const result = await prmBestOfN(fakeStore, grader, ['run-0', 'run-1', 'run-2'], { - concurrency: 2, - }) - expect(result.winner.runId).toBe('run-1') - }) -}) - -describe('prmEnsembleBestOfN bounded concurrency + isolation', () => { - it('caps concurrent grade calls across the flattened graderxrunId product', async () => { - const scores = { 'run-0': 0.2, 'run-1': 0.8, 'run-2': 0.5 } - const meter: ConcurrencyMeter = { peak: 0, active: 0 } - const g1 = makeTrackingGrader(scores, meter) - const g2 = makeTrackingGrader(scores, meter) - const g3 = makeTrackingGrader(scores, meter) - const runIds = ['run-0', 'run-1', 'run-2'] - - const result = await prmEnsembleBestOfN(fakeStore, [g1, g2, g3], runIds, { concurrency: 2 }) - - expect(result.ranked).toHaveLength(3) - // OLD nested Promise.all fired 3 graders x 3 runIds = 9 calls at once. - // The shared meter measures the true simultaneous in-flight count across - // the whole flattened pool; it must never exceed the configured bound. - expect(meter.peak).toBeLessThanOrEqual(2) - expect(meter.peak).toBeGreaterThan(0) - }) - - it('one grader failing on a candidate does not void the ensemble (allSettled)', async () => { - const scores = { 'run-0': 0.2, 'run-1': 0.9, 'run-2': 0.4 } - const good = makeTrackingGrader(scores) - // Second grader rejects on every candidate; the ensemble must survive. - const flaky = makeFailingGrader('rate limited') - const runIds = ['run-0', 'run-1', 'run-2'] - - const result = await prmEnsembleBestOfN(fakeStore, [good, flaky], runIds, { concurrency: 4 }) - - // OLD behavior: nested Promise.all rejects the whole call on the first - // grader failure. New behavior: the surviving grader still produces a vote. - expect(result.ranked).toHaveLength(3) - expect(result.winner.runId).toBe('run-1') - }) - - it('throws only when every grader fails on every candidate', async () => { - const runIds = ['run-0', 'run-1'] - await expect( - prmEnsembleBestOfN( - fakeStore, - [makeFailingGrader('boom-a'), makeFailingGrader('boom-b')], - runIds, - ), - ).rejects.toThrow(/every grader failed/) - }) -}) diff --git a/src/prm/inference.ts b/src/prm/inference.ts deleted file mode 100644 index eeb85601..00000000 --- a/src/prm/inference.ts +++ /dev/null @@ -1,136 +0,0 @@ -/** - * Inference-time PRM scoring — pick the best of N candidate trajectories - * using a trained reward model (or a rule-based PRM as a proxy). - * - * The canonical Best-of-N pattern: generate N completions, score each - * with a PRM, pick the winner. Here the scoring loop is framework-agnostic - * — supply a TraceStore + PrmGrader + N run IDs → get ranking + winner. - */ - -import type { TraceStore } from '../trace/store' -import type { PrmGradedTrace, PrmGrader } from './rubric' - -export interface BestOfNResult { - winner: PrmGradedTrace - ranked: PrmGradedTrace[] - /** Standard deviation of aggregate scores — small = candidates were homogenous. */ - stdDev: number -} - -/** Default max concurrent grader calls. Bounds LLM fan-out so a wide ensemble - * doesn't trigger a provider rate-limit storm. */ -const DEFAULT_PRM_CONCURRENCY = 4 - -export interface PrmBestOfNOptions { - /** Max concurrent `grader.grade` calls. Default 4. */ - concurrency?: number -} - -export async function prmBestOfN( - store: TraceStore, - grader: PrmGrader, - runIds: string[], - options: PrmBestOfNOptions = {}, -): Promise { - if (runIds.length === 0) throw new Error('prmBestOfN: at least 1 candidate required') - const concurrency = Math.max(1, options.concurrency ?? DEFAULT_PRM_CONCURRENCY) - const graded: PrmGradedTrace[] = new Array(runIds.length) - let cursor = 0 - async function worker(): Promise { - while (true) { - const i = cursor++ - if (i >= runIds.length) return - graded[i] = await grader.grade(store, runIds[i]!) - } - } - await Promise.all(Array.from({ length: Math.min(concurrency, runIds.length) }, () => worker())) - const ranked = [...graded].sort((a, b) => b.aggregateScore - a.aggregateScore) - const mean = graded.reduce((a, g) => a + g.aggregateScore, 0) / graded.length - const variance = graded.reduce((a, g) => a + (g.aggregateScore - mean) ** 2, 0) / graded.length - return { winner: ranked[0]!, ranked, stdDev: Math.sqrt(variance) } -} - -/** - * Weighted vote across multiple graders — use when you want a PRM ensemble - * (e.g. rule-based + LLM-based + trained model). Each grader produces its - * own ranking; we aggregate via rank-sum (Borda count) so no single grader - * dominates via a different score scale. - */ -export async function prmEnsembleBestOfN( - store: TraceStore, - graders: PrmGrader[], - runIds: string[], - options: PrmBestOfNOptions = {}, -): Promise { - if (graders.length === 0) throw new Error('prmEnsembleBestOfN: at least 1 grader') - if (runIds.length === 0) throw new Error('prmEnsembleBestOfN: at least 1 candidate required') - const concurrency = Math.max(1, options.concurrency ?? DEFAULT_PRM_CONCURRENCY) - - // Flatten the (grader, runId) product and run it through a single bounded - // pool. Nested unbounded fan-out (graders × runIds) would launch every LLM - // call at once. allSettled isolates failures: one grader (or one runId) - // failing doesn't void the whole ensemble. - type Job = { graderIdx: number; runId: string } - const jobs: Job[] = [] - for (let gi = 0; gi < graders.length; gi++) { - for (const runId of runIds) jobs.push({ graderIdx: gi, runId }) - } - const settled: PromiseSettledResult[] = new Array(jobs.length) - let cursor = 0 - async function worker(): Promise { - while (true) { - const i = cursor++ - if (i >= jobs.length) return - const job = jobs[i]! - settled[i] = await graders[job.graderIdx]!.grade(store, job.runId) - .then((value): PromiseSettledResult => ({ status: 'fulfilled', value })) - .catch((reason): PromiseSettledResult => ({ status: 'rejected', reason })) - } - } - await Promise.all(Array.from({ length: Math.min(concurrency, jobs.length) }, () => worker())) - - // Regroup fulfilled results into per-grader rankings. A grader contributes - // only the candidates it successfully graded; a grader that graded nothing - // is dropped from the vote rather than skewing it with phantom zeros. - const perGrader: PrmGradedTrace[][] = graders.map(() => []) - const failures: { graderIdx: number; runId: string; error: string }[] = [] - for (let i = 0; i < jobs.length; i++) { - const job = jobs[i]! - const result = settled[i]! - if (result.status === 'fulfilled') perGrader[job.graderIdx]!.push(result.value) - else { - const error = result.reason instanceof Error ? result.reason.message : String(result.reason) - failures.push({ graderIdx: job.graderIdx, runId: job.runId, error }) - } - } - const survivingGraders = perGrader.filter((ranking) => ranking.length > 0) - if (survivingGraders.length === 0) { - throw new Error( - `prmEnsembleBestOfN: every grader failed on every candidate (${failures.length} call(s)). First error: ${failures[0]?.error ?? 'unknown'}`, - ) - } - for (const ranking of survivingGraders) - ranking.sort((a, b) => b.aggregateScore - a.aggregateScore) - - // Borda: rank-sum across surviving graders. - const bordaScores = new Map() - for (const ranking of survivingGraders) { - ranking.forEach((g, rank) => { - bordaScores.set(g.runId, (bordaScores.get(g.runId) ?? 0) + (ranking.length - rank)) - }) - } - // Synthesize a ranking from the union of every successfully-graded trace, - // ordered by Borda score. aggregateScore field kept for UX. Using the union - // (not just the first grader) keeps a candidate that one grader dropped but - // another graded. - const byRun = new Map() - for (const ranking of survivingGraders) { - for (const g of ranking) if (!byRun.has(g.runId)) byRun.set(g.runId, g) - } - const ranked = [...byRun.values()].sort( - (a, b) => (bordaScores.get(b.runId) ?? 0) - (bordaScores.get(a.runId) ?? 0), - ) - const mean = ranked.reduce((a, g) => a + g.aggregateScore, 0) / ranked.length - const variance = ranked.reduce((a, g) => a + (g.aggregateScore - mean) ** 2, 0) / ranked.length - return { winner: ranked[0]!, ranked, stdDev: Math.sqrt(variance) } -} diff --git a/src/prm/rubric.ts b/src/prm/rubric.ts deleted file mode 100644 index e5237e0f..00000000 --- a/src/prm/rubric.ts +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Process Reward Modeling — per-step rubric grading. - * - * A StepRubric inspects one span and returns a score + rationale. - * PrmGrader applies an array of rubrics to every LLM span in a - * trajectory (consumers can broaden to tool/retrieval spans via the - * `kind` filter on each rubric). - * - * Why this matters: outcome-only eval (did the final artifact work?) - * gives sparse reward — most agent turns are unattributable. PRMs - * densify the signal so optimizers and RL fine-tuning can assign - * credit per turn. - */ - -import { TraceEmitter } from '../trace/emitter' -import type { JudgeSpan, Span } from '../trace/schema' -import type { TraceStore } from '../trace/store' -import { buildTrajectory, type Trajectory, type TrajectoryStep } from '../trajectory' - -export interface StepContext { - trajectory: Trajectory - step: TrajectoryStep - /** Steps preceding `step` in trajectory order. */ - prior: TrajectoryStep[] - /** Steps following `step`. */ - next: TrajectoryStep[] -} - -export interface StepRubric { - id: string - /** Only grade spans of these kinds (default: all). */ - kinds?: Array - /** Weight in the aggregate score. Default 1. */ - weight?: number - /** Returns score in 0..1 + optional rationale/evidence. Return `null` to - * skip grading (rubric doesn't apply to this step). */ - grade: ( - ctx: StepContext, - ) => Promise<{ score: number; rationale?: string; evidence?: string } | null> -} - -export interface GradedStep { - spanId: string - rubricId: string - score: number - weight: number - rationale?: string - evidence?: string -} - -export interface PrmGradedTrace { - runId: string - steps: GradedStep[] - /** Weighted mean of all graded steps; 0..1. */ - aggregateScore: number - /** Number of spans graded — useful for sanity-checking coverage. */ - gradedCount: number - /** Number of spans in the trajectory that no rubric matched. */ - ungradedCount: number -} - -export class PrmGrader { - constructor(private rubrics: StepRubric[]) { - if (rubrics.length === 0) throw new Error('PrmGrader: at least 1 rubric required') - } - - /** - * Grade every eligible span in a run. Emits a JudgeVerdict span for each - * (rubric × span) verdict so the result is visible to downstream pipelines - * (judgeAgreementView, etc.) — PRM is just "a judge that runs per span." - */ - async grade(store: TraceStore, runId: string): Promise { - const trajectory = await buildTrajectory(store, runId) - const emitter = new TraceEmitter(store, { runId }) - const steps: GradedStep[] = [] - let ungraded = 0 - for (let i = 0; i < trajectory.steps.length; i++) { - const step = trajectory.steps[i]! - const ctx: StepContext = { - trajectory, - step, - prior: trajectory.steps.slice(0, i), - next: trajectory.steps.slice(i + 1), - } - let gradedThis = false - for (const rubric of this.rubrics) { - if (rubric.kinds && !rubric.kinds.includes(step.span.kind)) continue - const verdict = await rubric.grade(ctx) - if (verdict === null) continue - const weight = rubric.weight ?? 1 - steps.push({ - spanId: step.span.spanId, - rubricId: rubric.id, - score: verdict.score, - weight, - rationale: verdict.rationale, - evidence: verdict.evidence, - }) - gradedThis = true - // Persist the verdict as a JudgeSpan so the query pipelines see it - await emitter.recordJudge({ - judgeId: `prm:${rubric.id}`, - targetSpanId: step.span.spanId, - dimension: 'step_quality', - score: verdict.score, - rationale: verdict.rationale, - evidence: verdict.evidence, - name: `prm:${rubric.id}`, - }) - } - if (!gradedThis) ungraded++ - } - - const totalWeight = steps.reduce((a, s) => a + s.weight, 0) - const aggregateScore = - totalWeight === 0 ? 0 : steps.reduce((a, s) => a + s.score * s.weight, 0) / totalWeight - - return { runId, steps, aggregateScore, gradedCount: steps.length, ungradedCount: ungraded } - } -} - -/** Helper: reads JudgeVerdict spans that PRM emitted so downstream pipelines - * can distinguish PRM verdicts from human or top-level LLM judges. */ -export function isPrmVerdict(verdict: JudgeSpan): boolean { - return verdict.judgeId.startsWith('prm:') -} diff --git a/src/prm/training-export.ts b/src/prm/training-export.ts deleted file mode 100644 index d4d2d071..00000000 --- a/src/prm/training-export.ts +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Export PRM-graded traces as training data for downstream reward-model - * fine-tuning. Canonical format is NDJSON of - * `{ trajectory_text, step_index, rubric, score }` so a small model can - * learn to predict step rewards from step context. - * - * The framework doesn't train the model — we emit the data; callers - * plug it into their preferred trainer (TRL, Unsloth, custom). - */ - -import type { LlmSpan, Span } from '../trace/schema' -import { isLlmSpan, isToolSpan } from '../trace/schema' -import type { TraceStore } from '../trace/store' -import { buildTrajectory } from '../trajectory' -import type { PrmGradedTrace } from './rubric' - -export interface PrmTrainingSample { - runId: string - spanId: string - rubricId: string - score: number - /** Serialized step context — step + surrounding conversation. */ - context: { - priorTurns: Array<{ role: string; content: string }> - step: { kind: Span['kind']; text: string } - } - /** Optional evidence + rationale for auditability. */ - rationale?: string - evidence?: string -} - -export async function exportTrainingData( - store: TraceStore, - graded: PrmGradedTrace[], - options: { contextWindow?: number } = {}, -): Promise { - const window = options.contextWindow ?? 5 - const out: PrmTrainingSample[] = [] - for (const g of graded) { - const trajectory = await buildTrajectory(store, g.runId) - const spanById = new Map(trajectory.steps.map((s) => [s.span.spanId, s])) - for (const gs of g.steps) { - const node = spanById.get(gs.spanId) - if (!node) continue - const idx = trajectory.steps.indexOf(node) - const priorSpans = trajectory.steps.slice(Math.max(0, idx - window), idx).map((s) => s.span) - out.push({ - runId: g.runId, - spanId: gs.spanId, - rubricId: gs.rubricId, - score: gs.score, - context: { - priorTurns: priorSpans - .map(spanToTurn) - .filter((t): t is { role: string; content: string } => t !== null), - step: { kind: node.span.kind, text: spanToText(node.span) }, - }, - rationale: gs.rationale, - evidence: gs.evidence, - }) - } - } - return out -} - -/** NDJSON serialization — write to file or stream directly to a trainer. */ -export function toNdjson(samples: PrmTrainingSample[]): string { - return `${samples.map((s) => JSON.stringify(s)).join('\n')}\n` -} - -function spanToTurn(span: Span): { role: string; content: string } | null { - if (isLlmSpan(span)) { - const text = span.output ?? span.messages.map((m) => `${m.role}: ${m.content}`).join('\n') - return { role: 'assistant', content: text } - } - if (isToolSpan(span)) { - return { - role: 'tool', - content: `${span.toolName}(${safeStringify(span.args)}) → ${safeStringify(span.result)}`, - } - } - return null -} - -function spanToText(span: Span): string { - if (isLlmSpan(span)) return (span as LlmSpan).output ?? '' - if (isToolSpan(span)) - return `${span.toolName}(${safeStringify(span.args)}) → ${safeStringify(span.result)}` - return span.name -} - -function safeStringify(v: unknown): string { - if (v === null || v === undefined) return '' - if (typeof v === 'string') return v - try { - return JSON.stringify(v) - } catch { - return String(v) - } -} diff --git a/src/registry.ts b/src/registry.ts deleted file mode 100644 index 0eda17b4..00000000 --- a/src/registry.ts +++ /dev/null @@ -1,72 +0,0 @@ -import type { Scenario, ScenarioFile } from './types' - -/** - * ScenarioRegistry — manages scenario discovery and filtering. - * - * Each agent registers its scenarios. The registry handles conversion - * from ScenarioFile format to the framework's Scenario type. - */ -export class ScenarioRegistry { - private scenarios: Scenario[] = [] - private scenarioFiles: ScenarioFile[] = [] - - /** Register scenarios from ScenarioFile format */ - registerFiles(files: ScenarioFile[]): void { - this.scenarioFiles.push(...files) - this.scenarios.push(...files.map(toScenario)) - } - - /** Register pre-built Scenario objects directly */ - register(scenarios: Scenario[]): void { - this.scenarios.push(...scenarios) - } - - /** Get all scenarios */ - all(): Scenario[] { - return [...this.scenarios] - } - - /** Get scenarios filtered by category */ - byCategory(category: string): Scenario[] { - const fromFiles = this.scenarioFiles.filter((sf) => sf.category === category).map(toScenario) - return fromFiles - } - - /** List all categories with counts */ - listCategories(): { category: string; count: number }[] { - const counts: Record = {} - for (const sf of this.scenarioFiles) { - counts[sf.category] = (counts[sf.category] ?? 0) + 1 - } - return Object.entries(counts).map(([category, count]) => ({ category, count })) - } - - /** Get scenarios filtered by persona */ - byPersona(persona: string): Scenario[] { - return this.scenarios.filter((s) => s.persona === persona) - } - - /** Get a single scenario by ID */ - byId(id: string): Scenario | undefined { - return this.scenarios.find((s) => s.id === id) - } - - /** Count total scenarios */ - get count(): number { - return this.scenarios.length - } -} - -/** Convert ScenarioFile to the framework's Scenario type */ -function toScenario(sf: ScenarioFile): Scenario { - return { - id: sf.id, - persona: sf.persona, - label: sf.label, - thesis: sf.thesis, - dimensions: [], - turns: sf.turns, - artifactChecks: sf.artifactChecks, - systemPromptAppend: sf.isControl ? 'You are a helpful AI assistant.' : undefined, - } -} diff --git a/src/reviewer.test.ts b/src/reviewer.test.ts deleted file mode 100644 index a1a9819f..00000000 --- a/src/reviewer.test.ts +++ /dev/null @@ -1,201 +0,0 @@ -import { describe, expect, it, vi } from 'vitest' -import { CostLedger } from './cost-ledger' -import { buildReviewerPrompt, createDefaultReviewer } from './reviewer' - -const BASE_INPUT = { - shot: 2, - userRequest: 'build an NFT mint page with supply counter, mint button', - traceSummary: 'tool calls: {Write: 3, Edit: 2}, errors: none', - verification: { - blendedScore: 0.5, - allPass: false, - failCount: 2, - failingLayers: ['typecheck', 'semantic'], - }, - memory: [ - { - shot: 1, - confidence: 0.85, - shouldContinue: true, - observations: 'worker wrote App.tsx', - diagnosis: 'wagmi imports wrong', - nextShotInstruction: 'fix imports', - }, - ], -} - -describe('buildReviewerPrompt', () => { - it('emits system + user with all context blocks present', () => { - const { system, user } = buildReviewerPrompt(BASE_INPUT) - expect(system).toMatch(/senior-engineer-grade reviewer/) - expect(user).toMatch(/shot 2 of the review loop/) - expect(user).toMatch(/build an NFT mint page/) - expect(user).toMatch(/tool calls:/) - expect(user).toMatch(/blendedScore: 0.50/) - expect(user).toMatch(/failing layers: typecheck, semantic/) - expect(user).toMatch(/shot 1 — confidence=0.85/) - expect(user).toMatch(/STRICT JSON/) - }) - - it('injects extraContext block when provided', () => { - const { user } = buildReviewerPrompt({ - ...BASE_INPUT, - extraContext: 'workdir: src/App.tsx, src/MintButton.tsx', - }) - expect(user).toMatch(/EXTRA CONTEXT/) - expect(user).toMatch(/src\/MintButton\.tsx/) - }) - - it('omits extraContext block entirely when not provided', () => { - const { user } = buildReviewerPrompt(BASE_INPUT) - expect(user).not.toMatch(/EXTRA CONTEXT/) - }) - - it('shows "(no prior shots)" when memory is empty', () => { - const { user } = buildReviewerPrompt({ ...BASE_INPUT, memory: [] }) - expect(user).toMatch(/\(no prior shots\)/) - }) - - it('signals no-failing-layers when verification.failingLayers is empty', () => { - const { user } = buildReviewerPrompt({ - ...BASE_INPUT, - verification: { blendedScore: 1, allPass: true, failCount: 0, failingLayers: [] }, - }) - expect(user).toMatch(/no layers failing/) - }) - - it('trailingContext renders at the end when provided', () => { - const { user } = buildReviewerPrompt({ - ...BASE_INPUT, - trailingContext: 'leaf_id: nft-mint-page', - }) - expect(user).toMatch(/TRAILING CONTEXT[\s\S]+leaf_id: nft-mint-page/) - }) -}) - -describe('createDefaultReviewer', () => { - function mockFetch(responses: Array) { - let i = 0 - return (async () => { - const r = responses[Math.min(i++, responses.length - 1)]! - if ('status' in r && 'body' in r) { - return new Response((r as { body: string }).body, { - status: (r as { status: number }).status, - }) - } - return new Response( - JSON.stringify({ - model: 'mock', - choices: [{ message: { content: JSON.stringify(r) } }], - usage: { total_tokens: 100 }, - _response_cost: 0.001, - }), - { status: 200, headers: { 'content-type': 'application/json' } }, - ) - }) as unknown as typeof fetch - } - - it('calls LLM, parses structured output, returns ReviewerOutput', async () => { - const costLedger = new CostLedger() - const fetch = mockFetch([ - { - observations: 'worker wrote 3 files via Edit, no errors logged, build failed on typecheck.', - diagnosis: 'wagmi v2 API misuse — useAccount from wrong import path, ts will not compile.', - nextShotInstruction: - 'FIX THESE: 1) change `import { useAccount } from "wagmi/core"` to `from "wagmi"` in src/App.tsx', - shouldContinue: true, - confidence: 0.85, - }, - ]) - const reviewer = createDefaultReviewer({ model: 'mock-model', llm: { fetch }, costLedger }) - const r = await reviewer(BASE_INPUT) - expect(r.available).toBe(true) - expect(r.shot).toBe(2) - expect(r.confidence).toBeCloseTo(0.85) - expect(r.shouldContinue).toBe(true) - expect(r.diagnosis).toMatch(/wagmi v2/) - expect(r.costUsd).toBeCloseTo(0.001) - expect(costLedger.list()).toEqual([ - expect.objectContaining({ channel: 'analyst', actor: 'default-reviewer', costUsd: 0.001 }), - ]) - }) - - it('clamps confidence to [0, 1]', async () => { - const fetch = mockFetch([ - { - observations: 'x'.repeat(30), - diagnosis: 'y'.repeat(30), - nextShotInstruction: 'z'.repeat(50), - shouldContinue: false, - confidence: 1.5, - }, - ]) - const r = await createDefaultReviewer({ model: 'm', llm: { fetch } })(BASE_INPUT) - expect(r.confidence).toBe(1) - }) - - it('soft-fails available=false on LLM error; uses soft-fail defaults', async () => { - const fetch = mockFetch([{ status: 500, body: 'upstream oops' }]) - const r = await createDefaultReviewer({ - model: 'm', - llm: { fetch, maximumAttempts: 1 }, - })(BASE_INPUT) - expect(r.available).toBe(false) - expect(r.error).toMatch(/500/) - expect(r.confidence).toBe(0.3) - expect(r.shouldContinue).toBe(true) - expect(r.nextShotInstruction).toMatch(/Inspect the verification findings/) - }) - - it('honors custom soft-fail defaults', async () => { - const fetch = mockFetch([{ status: 503, body: 'rate-limited' }]) - const r = await createDefaultReviewer({ - model: 'm', - llm: { fetch, maximumAttempts: 1 }, - softFailDefaults: { shouldContinue: false, confidence: 0 }, - })(BASE_INPUT) - expect(r.shouldContinue).toBe(false) - expect(r.confidence).toBe(0) - }) - - it('custom promptBuilder is used instead of default', async () => { - const fetch = vi.fn( - async () => - new Response( - JSON.stringify({ - choices: [ - { - message: { - content: - '{"observations":"' + - 'o'.repeat(25) + - '","diagnosis":"' + - 'd'.repeat(25) + - '","nextShotInstruction":"' + - 'i'.repeat(50) + - '","shouldContinue":false,"confidence":0.5}', - }, - }, - ], - usage: {}, - }), - { status: 200 }, - ), - ) as unknown as typeof globalThis.fetch - const custom = vi.fn((_: unknown) => ({ system: 'CUSTOM-SYS', user: 'CUSTOM-USER' })) - const reviewer = createDefaultReviewer({ - model: 'm', - llm: { fetch }, - promptBuilder: custom as never, - }) - await reviewer(BASE_INPUT) - expect(custom).toHaveBeenCalledOnce() - const call = (fetch as unknown as ReturnType).mock.calls[0]! as unknown as [ - string, - RequestInit, - ] - const body = JSON.parse(call[1].body as string) - expect(body.messages[0].content).toBe('CUSTOM-SYS') - expect(body.messages[1].content).toBe('CUSTOM-USER') - }) -}) diff --git a/src/reviewer.ts b/src/reviewer.ts deleted file mode 100644 index 7a9a758e..00000000 --- a/src/reviewer.ts +++ /dev/null @@ -1,305 +0,0 @@ -/** - * Reviewer primitives — prompt builder + default ReviewFn factory. - * - * `buildReviewerPrompt` is the pure, LLM-agnostic piece: takes - * `ReviewerPromptInput` (user request, trace summary, verification - * summary, memory, optional extra context) and emits the system + - * user message pair. No LLM dependency — callers that want to drive - * their own transport get full control. - * - * `createDefaultReviewer` is the convenience factory: wires the prompt - * builder to `callLlmJson` with a default schema + soft-fail policy. - * Returns a function that maps `ReviewerPromptInput` to `ReviewerOutput`. - * - * Same pattern as `runSemanticConceptJudge` / `createSemanticConceptJudge`: - * low-level pure builder + high-level factory built on top. - */ - -import { CostLedger, type CostLedgerHandle, type CostReceipt } from './cost-ledger' -import { - callLlmJson, - costReceiptFromLlm, - costReceiptFromLlmError, - type LlmCallRequest, - type LlmClientOptions, - maximumChargeForLlmRequest, -} from './llm-client' - -// ─── Types ────────────────────────────────────────────────────────────── - -export interface ReviewerMemoryEntry { - shot: number - ts?: string - observations?: string - diagnosis?: string - nextShotInstruction?: string - shouldContinue?: boolean - confidence?: number -} - -export interface ReviewerVerificationSummary { - blendedScore: number - allPass: boolean - failCount: number - failingLayers?: string[] -} - -export interface ReviewerPromptInput { - shot: number - userRequest: string - /** - * Compact trace summary — tool-call counts, errors, recent activity - * lines. Built by the caller from whatever trace format they have; - * agent-eval does not prescribe. - */ - traceSummary: string - verification: ReviewerVerificationSummary - memory: ReviewerMemoryEntry[] - /** - * Optional extra context injected into the prompt between the trace - * and the verification blocks. Use for workdir file-tree snapshots, - * scaffold descriptions, or any environmental fact the reviewer - * needs to direct the next shot accurately. - */ - extraContext?: string - /** - * Optional extra section appended at the end of the prompt (e.g. - * leaf metadata, scenario id). Free-form — no agent-eval-shaped - * schema. - */ - trailingContext?: string -} - -export interface ReviewerOutput { - shot: number - observations: string - diagnosis: string - nextShotInstruction: string - shouldContinue: boolean - /** 0..1 self-assessed confidence in the directive. */ - confidence: number - /** LLM cost in USD if the transport reports it, else null. */ - costUsd: number | null - durationMs: number - /** False when the LLM errored or returned malformed JSON; caller soft-fails to defaults. */ - available: boolean - error?: string -} - -export interface ReviewerSoftFailDefaults { - observations?: string - diagnosis?: string - nextShotInstruction?: string - shouldContinue?: boolean - confidence?: number -} - -export interface CreateDefaultReviewerOptions { - /** Model id to call. */ - model: string - /** Per-call timeout. Default 300s. */ - timeoutMs?: number - /** Provider-enforced output limit. Default 4000. */ - maxTokens?: number - /** LlmClient transport config (baseUrl, apiKey, authHeader, etc.). */ - llm?: LlmClientOptions - /** Shared run spend account. */ - costLedger?: CostLedgerHandle - costPhase?: string - signal?: AbortSignal - /** - * Override the prompt builder. Default: `buildReviewerPrompt`. - * Consumers with different reviewer voices pass their own. - */ - promptBuilder?: (input: ReviewerPromptInput) => { system: string; user: string } - /** - * Soft-fail values when the LLM throws or returns unparseable JSON. - * Matches VerticalBench's shipped policy: continue with generic - * instruction at confidence 0.3 so the worker keeps trying. - */ - softFailDefaults?: ReviewerSoftFailDefaults -} - -// ─── JSON schema ─────────────────────────────────────────────────────── - -const REVIEWER_SCHEMA = { - type: 'object', - additionalProperties: false, - required: ['observations', 'diagnosis', 'nextShotInstruction', 'shouldContinue', 'confidence'], - properties: { - observations: { type: 'string', minLength: 20, maxLength: 2000 }, - diagnosis: { type: 'string', minLength: 20, maxLength: 1500 }, - nextShotInstruction: { type: 'string', minLength: 40, maxLength: 3000 }, - shouldContinue: { type: 'boolean' }, - confidence: { type: 'number', minimum: 0, maximum: 1 }, - }, -} as const - -// ─── Prompt builder ──────────────────────────────────────────────────── - -function summarizeMemory(memory: ReviewerMemoryEntry[]): string { - if (memory.length === 0) return '(no prior shots)' - return memory - .map((m) => { - const header = `shot ${m.shot} — confidence=${(m.confidence ?? 0).toFixed(2)} shouldContinue=${m.shouldContinue ?? '?'}` - const obs = m.observations ? ` observations: ${m.observations.slice(0, 400)}` : '' - const diag = m.diagnosis ? ` diagnosis: ${m.diagnosis.slice(0, 400)}` : '' - const instr = m.nextShotInstruction - ? ` instruction given: ${m.nextShotInstruction.slice(0, 400)}` - : '' - return [header, obs, diag, instr].filter(Boolean).join('\n') - }) - .join('\n\n') -} - -/** - * Build the reviewer's system + user messages. Pure function, no LLM - * call. Callers that want their own transport or a different structured - * output can use this and skip `createDefaultReviewer` entirely. - */ -export function buildReviewerPrompt(input: ReviewerPromptInput): { system: string; user: string } { - const system = - 'You are a senior-engineer-grade reviewer directing an agent through a multi-shot build. ' + - "Your job is NOT to grade; your job IS to direct the worker's next shot using the trace, " + - 'verification result, prior memory, and user request. Return STRICT JSON. No prose outside the JSON.' - - const failingLayersBlock = - input.verification.failingLayers && input.verification.failingLayers.length > 0 - ? `failing layers: ${input.verification.failingLayers.join(', ')}` - : 'no layers failing' - - const user = `=== SHOT NUMBER === -shot ${input.shot} of the review loop - -=== USER REQUEST === -${input.userRequest} - -=== WORKER TRACE (shot ${input.shot}) === -${input.traceSummary} -${input.extraContext ? `\n=== EXTRA CONTEXT ===\n${input.extraContext}\n` : ''} -=== VERIFICATION (shot ${input.shot}) === -blendedScore: ${input.verification.blendedScore.toFixed(2)} -allPass: ${input.verification.allPass} -failCount: ${input.verification.failCount} -${failingLayersBlock} - -=== REVIEWER MEMORY === -${summarizeMemory(input.memory)} -${input.trailingContext ? `\n=== TRAILING CONTEXT ===\n${input.trailingContext}\n` : ''} -=== YOUR TASK === -Return STRICT JSON: - -1. observations (20-2000 chars): first-person worker behavior from the trace (tool call counts, errors, loops). -2. diagnosis (20-1500 chars): root cause of current failures, not a restatement of verification. -3. nextShotInstruction (40-3000 chars): concrete "FIX THESE:" directive for the worker's next shot. Reference memory when instructions repeat. -4. shouldContinue (boolean): FALSE if verification.allPass=true, if worker is thrashing, if confidence < 0.3, or if the request looks unachievable. TRUE otherwise. -5. confidence (0-1): self-assessment. - -RULES: -- If verification.allPass is true, shouldContinue MUST be false. -- If memory shows the same failing layer for 2 shots, reduce confidence — strategy isn't working. -- If the trace shows zero tool calls, the worker didn't run — surface that. -- Do NOT re-grade. Direct.` - - return { system, user } -} - -// ─── Default reviewer factory ─────────────────────────────────────────── - -const DEFAULT_SOFT_FAIL: Required = { - observations: 'reviewer soft-failed — no observations captured', - diagnosis: 'reviewer soft-failed — inspect verification findings and retry', - nextShotInstruction: - 'Inspect the verification findings above and address the highest-severity failing layer first. ' + - 'If install failed, start there; otherwise work from the first failing gate and address compilation/build errors before layout/semantic issues.', - shouldContinue: true, - confidence: 0.3, -} - -/** - * Factory: returns a function that invokes the default reviewer against - * an LLM and parses the structured output. Soft-fails to the provided - * defaults on LLM throw or JSON-parse error so the shot loop keeps - * moving rather than crashing. - */ -export function createDefaultReviewer( - options: CreateDefaultReviewerOptions, -): (input: ReviewerPromptInput) => Promise { - const softFail: Required = { - ...DEFAULT_SOFT_FAIL, - ...(options.softFailDefaults ?? {}), - } - const promptBuilder = options.promptBuilder ?? buildReviewerPrompt - const timeoutMs = options.timeoutMs ?? 300_000 - const maxTokens = options.maxTokens ?? 4_000 - const costLedger = options.costLedger ?? new CostLedger() - - return async (input) => { - const start = Date.now() - const { system, user } = promptBuilder(input) - let receipt: CostReceipt | undefined - try { - const request = { - model: options.model, - messages: [ - { role: 'system' as const, content: system }, - { role: 'user' as const, content: user }, - ], - jsonSchema: { name: 'reviewer_output', schema: REVIEWER_SCHEMA }, - temperature: 0, - maxTokens, - timeoutMs, - } satisfies LlmCallRequest - const paid = await costLedger.runPaidCall({ - channel: 'analyst', - phase: options.costPhase ?? 'review', - actor: 'default-reviewer', - model: options.model, - maximumCharge: maximumChargeForLlmRequest(request, options.llm), - signal: options.signal, - execute: (signal, callId) => - callLlmJson<{ - observations: string - diagnosis: string - nextShotInstruction: string - shouldContinue: boolean - confidence: number - }>(request, { - ...options.llm, - signal, - idempotencyKey: callId, - }), - receipt: ({ result }) => costReceiptFromLlm(result), - receiptFromError: costReceiptFromLlmError, - }) - receipt = paid.receipt - if (!paid.succeeded) throw paid.error - const { value } = paid.value - - return { - shot: input.shot, - observations: String(value.observations ?? softFail.observations), - diagnosis: String(value.diagnosis ?? softFail.diagnosis), - nextShotInstruction: String(value.nextShotInstruction ?? softFail.nextShotInstruction), - shouldContinue: Boolean(value.shouldContinue), - confidence: Math.max(0, Math.min(1, Number(value.confidence ?? softFail.confidence))), - costUsd: paid.receipt.costUnknown ? null : paid.receipt.costUsd, - durationMs: Date.now() - start, - available: true, - } - } catch (err) { - return { - shot: input.shot, - observations: softFail.observations, - diagnosis: softFail.diagnosis, - nextShotInstruction: softFail.nextShotInstruction, - shouldContinue: softFail.shouldContinue, - confidence: softFail.confidence, - costUsd: receipt && !receipt.costUnknown ? receipt.costUsd : null, - durationMs: Date.now() - start, - available: false, - error: err instanceof Error ? err.message : String(err), - } - } - } -} diff --git a/src/slo.ts b/src/slo.ts deleted file mode 100644 index 888d2d07..00000000 --- a/src/slo.ts +++ /dev/null @@ -1,131 +0,0 @@ -/** - * SLO gates — quantified pass/fail primitives beyond score thresholds. - * - * Lifted from ADC's sandbox eval suite. Each SLO defines a metric, a - * threshold, and a severity (critical | warning). Critical breaches fail - * the eval; warnings are reported but don't gate CI. Margin is the - * ratio of actual to threshold for histogramming "how close are we?" - * - * Consumers assemble their own SLO arrays; DEFAULT_AGENT_SLOS covers - * the generic agent flow (provision, first token, pass rate, cost). - */ - -export type SloSeverity = 'critical' | 'warning' -export type SloComparator = 'lte' | 'gte' - -export interface Slo { - /** Stable identifier — must be unique within an SLO set. */ - id: string - /** Human description, shown in reports. */ - description: string - /** Metric key looked up in the candidate record. */ - metric: string - /** Whether the metric should stay below (lte) or above (gte) threshold. */ - comparator: SloComparator - /** Threshold value. */ - threshold: number - severity: SloSeverity -} - -export interface SloCheckResult { - slo: Slo - actual: number | undefined - passed: boolean - /** actual/threshold for lte, threshold/actual for gte. >1 means safe margin; <1 means breach. 0 when actual is missing. */ - margin: number - detail: string -} - -export interface SloReport { - results: SloCheckResult[] - passedCritical: boolean - criticalBreaches: SloCheckResult[] - warnings: SloCheckResult[] -} - -/** - * Evaluate an SLO set against a candidate metrics object. Missing metrics - * count as breaches — if you declared it, you must measure it. - */ -export function checkSlos(metrics: Record, slos: Slo[]): SloReport { - const results: SloCheckResult[] = slos.map((slo) => check(slo, metrics[slo.metric])) - const criticalBreaches = results.filter((r) => !r.passed && r.slo.severity === 'critical') - const warnings = results.filter((r) => !r.passed && r.slo.severity === 'warning') - return { results, passedCritical: criticalBreaches.length === 0, criticalBreaches, warnings } -} - -function check(slo: Slo, actual: number | undefined): SloCheckResult { - if (actual === undefined || !Number.isFinite(actual)) { - return { - slo, - actual, - passed: false, - margin: 0, - detail: `metric "${slo.metric}" missing — declared SLOs must be measured`, - } - } - if (slo.comparator === 'lte') { - const passed = actual <= slo.threshold - const margin = slo.threshold === 0 ? (actual === 0 ? Infinity : 0) : slo.threshold / actual - return { - slo, - actual, - passed, - margin, - detail: `${actual} ≤ ${slo.threshold}: ${passed ? 'ok' : 'breach'}`, - } - } - const passed = actual >= slo.threshold - const margin = actual === 0 ? 0 : actual / slo.threshold - return { - slo, - actual, - passed, - margin, - detail: `${actual} ≥ ${slo.threshold}: ${passed ? 'ok' : 'breach'}`, - } -} - -/** Reference SLO set for agent-style evals. Tune per-product by cloning + overriding. */ -export const DEFAULT_AGENT_SLOS: Slo[] = [ - { - id: 'provision_ms', - description: 'Sandbox/session provision under 60s', - metric: 'provisionMs', - comparator: 'lte', - threshold: 60_000, - severity: 'critical', - }, - { - id: 'first_token_ms', - description: 'First token under 15s', - metric: 'firstTokenMs', - comparator: 'lte', - threshold: 15_000, - severity: 'critical', - }, - { - id: 'pass_rate', - description: 'Scenario pass rate ≥ 90%', - metric: 'passRate', - comparator: 'gte', - threshold: 0.9, - severity: 'critical', - }, - { - id: 'cost_usd', - description: 'Per-scenario cost under $0.05', - metric: 'costUsd', - comparator: 'lte', - threshold: 0.05, - severity: 'warning', - }, - { - id: 'overall_score', - description: 'Overall score ≥ 0.7', - metric: 'overallScore', - comparator: 'gte', - threshold: 0.7, - severity: 'critical', - }, -] diff --git a/src/ui-finding.ts b/src/ui-finding.ts deleted file mode 100644 index 0a34f4d1..00000000 --- a/src/ui-finding.ts +++ /dev/null @@ -1,139 +0,0 @@ -/** - * UI audit finding — substrate primitive for "what is wrong with the UI?" - * - * Used by: - * - `@tangle-network/agent-runtime` (ui-auditor profile + delegate) — - * produced as the canonical output of an audit iteration, persisted to - * disk as GitHub-issue Markdown, surfaced over MCP. - * - Downstream ship gates / dashboards / analyst consumers — load and - * transform findings without depending on the runtime. - * - * Repo layering: agent-eval is the substrate (no upward deps). Consumers - * read this type from here; the reverse is forbidden. See CLAUDE.md - * "Repo layering" for the rule. A UI finding makes sense WITHOUT a running - * agent loop (you can load a saved finding, ship-gate against a set of - * them, render them in a dashboard), which puts it firmly in substrate. - * - * The shape is intentionally minimal — runtime-shaped state (capture - * timestamps, OTel trace IDs, sandbox placement) lives on auxiliary - * runtime types in `agent-runtime`, not on the finding itself. - */ - -/** - * Canonical audit lenses. Each lens scopes a finding to a single class of - * problem so a single audit pass can iterate them without pile-on findings - * under a generic label. - * - * Naming is fixed for cross-package wire compatibility. Treat additions as - * a substrate-level decision — analysts, gates, and writers all branch on - * the lens. - */ -export type UiLens = - | 'consistency' - | 'hierarchy' - | 'layout' - | 'ux-flow' - | 'duplication' - | 'accessibility' - | 'responsive' - | 'states' - | 'content' - | 'interaction' - | 'performance-perceived' - | 'other' - -/** Frozen tuple of lenses for validation + iteration. */ -export const UI_LENSES: readonly UiLens[] = [ - 'consistency', - 'hierarchy', - 'layout', - 'ux-flow', - 'duplication', - 'accessibility', - 'responsive', - 'states', - 'content', - 'interaction', - 'performance-perceived', - 'other', -] as const - -/** - * Severity scale — intentionally narrow. - * - * - `critical` — blocks a core task or is an accessibility blocker. - * - `high` — confusing, broken-looking, or noticeable friction. - * - `med` — visible polish issue, would be caught in code review. - * - `low` — nitpick worth fixing eventually. - */ -export type UiFindingSeverity = 'low' | 'med' | 'high' | 'critical' - -/** Frozen severity tuple, ordered worst → least bad for sort/report. */ -export const UI_FINDING_SEVERITIES: readonly UiFindingSeverity[] = [ - 'critical', - 'high', - 'med', - 'low', -] as const - -/** - * Pointer to a screenshot referenced by the finding. The path is - * intentionally a relative string (relative to the audit workspace root) - * so findings remain portable across machines and into GitHub issues. - */ -export interface UiFindingScreenshot { - /** Workspace-relative path to the screenshot file (e.g. `screenshots/home--1280x800--...png`). */ - path: string - /** Optional viewport the screenshot was taken at, e.g. `1280x800`. */ - viewport?: string - /** Optional short label that disambiguates multiple captures of the same surface (e.g. `t0`, `step-1`). */ - label?: string -} - -/** - * A single UI audit finding — the unit of work a contributor can act on. - * - * Every field except the documented optionals is required. The shape is - * deliberately constraining: a finding without a screenshot, a lens, a - * concrete title, and a suggested fix is not actionable, and the auditor - * validator hard-fails on those gaps. - */ -export interface UiFinding { - /** - * Stable identifier within a single audit workspace. Monotonically - * increasing integer (1, 2, …) assigned by the writer when persisting. - * Optional in transit (before persistence) — undefined on freshly minted - * findings emitted from a loop iteration. - */ - id?: number - /** Concrete title — names the offending element AND what's wrong. */ - title: string - /** Lens this finding belongs to. */ - lens: UiLens - /** Severity. */ - severity: UiFindingSeverity - /** Logical route the finding was observed on (e.g. `home`, `checkout-step-2`). */ - route: string - /** Fully qualified URL the finding was observed at. */ - url?: string - /** Viewport string the offending capture was taken at (e.g. `1280x800`). */ - viewport?: string - /** CSS selector pinning the offending element, when one can be identified. */ - selector?: string - /** 1–3 sentences describing what the screenshot shows that is wrong. */ - observation: string - /** Who is affected and how. Concrete user impact. */ - impact: string - /** A specific change a contributor could apply without asking back. */ - suggestedFix: string - /** Optional explicit reproduction steps. Writer synthesizes from route/url/selector when omitted. */ - reproSteps?: string - /** Free-form tags. */ - tags?: readonly string[] - /** Screenshot references — required to be non-empty for actionable findings. */ - screenshots: readonly UiFindingScreenshot[] - /** Cross-references to similar findings already on file, by id. */ - similarTo?: readonly number[] - /** ISO-8601 creation timestamp set by the writer when persisted. */ - createdAt?: string -} diff --git a/src/worker-driver-seed.test.ts b/src/worker-driver-seed.test.ts deleted file mode 100644 index 039871f9..00000000 --- a/src/worker-driver-seed.test.ts +++ /dev/null @@ -1,71 +0,0 @@ -/** - * The worker-driver knowledge is DATA now. These tests pin the same - * contract the deleted `buildWorkerDriverSystemPrompt`'s tests pinned — a - * future edit cannot quietly soften the doctrine or strip a harness brief — - * plus the discriminating assertion that the deleted function stays deleted: - * if the role-as-function path returns to the public surface, this suite - * fails. - */ - -import { describe, expect, it } from 'vitest' -import * as api from './worker-driver-seed' -import { HARNESS_BRIEFS, WORKER_DRIVER_DOCTRINE } from './worker-driver-seed' - -describe('WORKER_DRIVER_DOCTRINE — the driving contract as seed data', () => { - it('demands rich, high-signal instructions and forbids thin steers', () => { - expect(WORKER_DRIVER_DOCTRINE).toMatch(/dense, specific/i) - expect(WORKER_DRIVER_DOCTRINE).toMatch(/thin steer/i) - expect(WORKER_DRIVER_DOCTRINE).toMatch(/out-drive a human/i) - }) - - it('drives the worker to exploit its harness — parallelize, sub-agents, run-to-completion', () => { - expect(WORKER_DRIVER_DOCTRINE).toMatch(/parallel/i) - expect(WORKER_DRIVER_DOCTRINE).toMatch(/sub-agent/i) - expect(WORKER_DRIVER_DOCTRINE).toMatch(/run to completion/i) - }) - - it('requires verification and refuses self-declared completion', () => { - expect(WORKER_DRIVER_DOCTRINE).toMatch(/verif/i) - expect(WORKER_DRIVER_DOCTRINE).toMatch(/never accept "done" without the check/i) - expect(WORKER_DRIVER_DOCTRINE).toMatch(/the deliverable's checker does/i) - }) - - it('reads the worker trace, decomposes, and names the traps each turn', () => { - expect(WORKER_DRIVER_DOCTRINE).toMatch(/not what it claims/i) - expect(WORKER_DRIVER_DOCTRINE).toMatch(/in sequence/i) - expect(WORKER_DRIVER_DOCTRINE).toMatch(/failure modes/i) - }) -}) - -describe('HARNESS_BRIEFS — capability briefs as seed data', () => { - it('covers the conventional harnesses', () => { - for (const harness of ['claude-code', 'codex', 'opencode', 'router-tools']) { - expect(HARNESS_BRIEFS[harness], `brief for ${harness}`).toBeTruthy() - } - }) - - it('claude-code brief names sub-agent fan-out, web access, and MCP', () => { - const brief = HARNESS_BRIEFS['claude-code']! - expect(brief).toMatch(/parallel/i) - expect(brief).toMatch(/sub-agent/i) - expect(brief).toMatch(/WebSearch/i) - expect(brief).toMatch(/MCP/i) - }) - - it('every brief carries a caveat, not just capabilities — no brief is a bare feature list', () => { - for (const [harness, brief] of Object.entries(HARNESS_BRIEFS)) { - expect(brief.length, `brief for ${harness}`).toBeGreaterThan(80) - expect(brief, `brief for ${harness} should mention a limit or a "never/no/not"`).toMatch( - /\b(no|not|never|cannot)\b/i, - ) - } - }) -}) - -describe('role-as-function stays deleted (discriminating)', () => { - it('the package no longer exports buildWorkerDriverSystemPrompt — the knowledge is data', () => { - expect(api).not.toHaveProperty('buildWorkerDriverSystemPrompt') - expect(api).toHaveProperty('WORKER_DRIVER_DOCTRINE') - expect(api).toHaveProperty('HARNESS_BRIEFS') - }) -}) diff --git a/src/worker-driver-seed.ts b/src/worker-driver-seed.ts deleted file mode 100644 index bb641729..00000000 --- a/src/worker-driver-seed.ts +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Worker-driver seed DATA — the knowledge that used to be hardcoded inside - * `buildWorkerDriverSystemPrompt` (deleted; zero callers), re-expressed as - * plain data so a driver PROFILE can seed its directive prompt from it and an - * optimizer (GEPA/DSPy) can then improve on it. A role expressed as a code - * function can never improve; a role expressed as versionable prompt data can - * (tangle-network/agent-runtime#694). - * - * Two exports: - * - `WORKER_DRIVER_DOCTRINE` — the "never write a thin steer" driving - * contract: the bar for every instruction a driver sends a worker. - * - `HARNESS_BRIEFS` — per-harness capability + caveat briefs the driver - * weaves into its instructions so it drives the worker to exploit what - * THAT harness can actually do, and never asks for a capability it lacks. - * - * Both are seeds, not law: a consumer copies them into its directive prompt - * registry (e.g. `prompts//v0.md`) and optimizes from there. - */ - -/** - * The driving contract a worker-driver directive seeds from. Every message the - * driver writes is held to this bar; a thin steer is the driver's failure, not - * the worker's. - */ -export const WORKER_DRIVER_DOCTRINE = `You are a DRIVER: a meta-agent whose entire job is to drive a capable coding worker to ACHIEVE A GOAL by writing it precise, high-signal instructions. You do not do the work yourself — you direct the worker to do it, brilliantly, and you hold it to a standard it would not hold itself to. - -THE BAR — non-negotiable. Every instruction you write is dense, specific, and complete. You write the message a world-class engineering lead writes to a strong report: the exact next objective, the concrete sub-steps, what to do in parallel vs in sequence, what to verify and how, what "done" looks like, and the failure modes to avoid. A thin steer — "try again", "fix the issues", a single vague sentence — is a failure on YOUR part, not the worker's. Out-drive a human power-user. - -HOW to drive (each turn): -1. Read what the worker actually did (its trace/state), not what it claims. Find the real gap between here and the goal. -2. Decide the next objective — the largest correct step the worker can take now. -3. Decompose it: name the sub-tasks that can run IN PARALLEL (independent files/checks/searches) and tell the worker to fan them out (sub-agents / parallel tool calls where its harness allows); name what must run in SEQUENCE and why. -4. Make it exploit the harness: drive it to run to completion under its own autonomy, spawn sub-agents for separable work, use its tools/MCP, and not stop at the first plausible stopping point. -5. Specify verification: the exact command / test / check that proves the step landed, and tell it to run that and report the result — never accept "done" without the check. -6. Name the traps: the specific failure modes for THIS step (editing the wrong file, a check that passes vacuously, scope creep) and forbid them. -7. As the task gets harder, decompose MORE, not less — more parallel branches, deeper sub-agent trees, tighter verification. - -COMPLETION: drive toward the goal's real acceptance check. Do not declare done — the deliverable's checker does. If the worker claims it is done, drive it to PROVE it with the check; if the check fails, drive the fix. - -Output ONLY your next instruction to the worker — direct, detailed, actionable, in the first person as the driver. No meta-commentary, no preamble.` - -/** - * Capability + caveat briefs per harness, keyed by harness id. Free text so - * the substrate stays decoupled from any runtime harness type — the keys are - * the conventional harness ids, but the record accepts any string so a - * consumer can extend it with its own. - */ -export const HARNESS_BRIEFS: Record = { - 'claude-code': [ - '- parallel tool calls in a single turn (batch independent reads/greps/commands)', - '- parallel Task sub-agents (~10 concurrent) for separable investigation or bulk work; sub-agents cannot spawn sub-agents', - '- native WebSearch/WebFetch plus MCP servers when configured', - '- skills, hooks, and slash commands when installed in the workspace', - '- runs to completion under its own autonomy; drive it NOT to stop at the first plausible stopping point', - ].join('\n'), - codex: [ - '- long-horizon autonomous runs (strong run-to-completion; suited to hours-long missions)', - '- sandboxed exec with full shell access; MCP servers when configured', - '- no native sub-agent fan-out: express parallelism as explicit backgrounded commands or sequential decomposition', - '- structured output via --output-schema when a machine-parsed result is needed', - ].join('\n'), - opencode: [ - '- router-backed: any model the router serves; model choice is a live lever', - '- tool calls and MCP servers when configured', - '- no native sub-agent tree; decompose into sequential objectives with explicit verification between them', - ].join('\n'), - 'router-tools': [ - '- a plain tool loop over router chat completions: no autonomy, no sub-agents, no workspace', - '- every capability must be driven explicitly, one tool call at a time', - '- keep objectives small and verification immediate; never assume it will continue on its own', - ].join('\n'), -} diff --git a/src/workspace-inspector.ts b/src/workspace-inspector.ts deleted file mode 100644 index 66a0bc0d..00000000 --- a/src/workspace-inspector.ts +++ /dev/null @@ -1,159 +0,0 @@ -/** - * Workspace inspector — score the persisted state of an agent after a run. - * - * Many evals don't ask "did the response say the right thing" but "did the - * agent put the right rows in the DB / files in the vault / entities on the - * canvas". This is the primitive for that. - * - * Implementations read from D1, KV, filesystem, or any store — the interface - * is deliberately small so consumers plug in their own backends. - */ - -export interface WorkspaceSnapshot { - /** Vault files: logical path → content */ - files: Record - /** DB rows: table name → array of rows (post-validation) */ - rows: Record>> - /** KV entries: key → value (scoped to whatever prefix the inspector chose) */ - kv: Record - /** Free-form blob metadata: for large binaries the inspector stores summary, not bytes */ - blobs?: Record -} - -export interface InspectorContext { - /** Workspace / agent / thread id — whatever the backend uses to scope the snapshot */ - scopeId: string - /** Optional scenario id — allows scenario-specific snapshot shaping */ - scenarioId?: string -} - -export interface WorkspaceInspector { - name: string - snapshot(context: InspectorContext): Promise -} - -// --------------------------------------------------------------------------- -// In-memory inspector — useful for tests, not for production -// --------------------------------------------------------------------------- - -export class InMemoryWorkspaceInspector implements WorkspaceInspector { - readonly name = 'in-memory' - private readonly snapshots = new Map() - - set(scopeId: string, snapshot: WorkspaceSnapshot): void { - this.snapshots.set(scopeId, snapshot) - } - - async snapshot(context: InspectorContext): Promise { - return this.snapshots.get(context.scopeId) ?? { files: {}, rows: {}, kv: {} } - } -} - -// --------------------------------------------------------------------------- -// Snapshot-level assertions -// --------------------------------------------------------------------------- - -export interface WorkspaceAssertion { - name: string - description?: string - check(snapshot: WorkspaceSnapshot): WorkspaceAssertionResult -} - -export interface WorkspaceAssertionResult { - pass: boolean - /** 0..1 — partial credit for assertions that admit it */ - score: number - detail?: string -} - -export function fileExists(path: string): WorkspaceAssertion { - return { - name: `file_exists:${path}`, - check(snapshot) { - const pass = path in snapshot.files - return { - pass, - score: pass ? 1 : 0, - detail: pass ? undefined : `No file at ${path}`, - } - }, - } -} - -export function fileContains(path: string, needle: string): WorkspaceAssertion { - return { - name: `file_contains:${path}:${needle}`, - check(snapshot) { - const content = snapshot.files[path] - if (content === undefined) { - return { pass: false, score: 0, detail: `File ${path} missing` } - } - const pass = content.includes(needle) - return { - pass, - score: pass ? 1 : 0, - detail: pass ? undefined : `File ${path} missing substring "${needle}"`, - } - }, - } -} - -export function rowCount(table: string, min: number, max?: number): WorkspaceAssertion { - return { - name: `row_count:${table}:[${min},${max ?? '∞'}]`, - check(snapshot) { - const rows = snapshot.rows[table] ?? [] - const count = rows.length - const upper = max ?? Infinity - const pass = count >= min && count <= upper - const score = pass ? 1 : count < min ? Math.max(0, count / min) : Math.max(0, upper / count) - return { - pass, - score, - detail: pass - ? undefined - : `Table ${table} has ${count} rows, expected [${min}, ${max ?? '∞'}]`, - } - }, - } -} - -export function rowWhere>( - table: string, - predicate: (row: T) => boolean, - options?: { min?: number }, -): WorkspaceAssertion { - const min = options?.min ?? 1 - return { - name: `row_where:${table}`, - check(snapshot) { - const rows = (snapshot.rows[table] ?? []) as T[] - const matching = rows.filter(predicate).length - const pass = matching >= min - return { - pass, - score: pass ? 1 : Math.max(0, matching / min), - detail: pass - ? undefined - : `Table ${table} has ${matching} matching rows, expected ≥ ${min}`, - } - }, - } -} - -/** Run many assertions; return aggregate pass + mean score + per-assertion details. */ -export function runAssertions( - snapshot: WorkspaceSnapshot, - assertions: WorkspaceAssertion[], -): { - pass: boolean - score: number - results: Array<{ assertion: string; result: WorkspaceAssertionResult }> -} { - const results = assertions.map((a) => ({ assertion: a.name, result: a.check(snapshot) })) - const pass = results.every((r) => r.result.pass) - const score = results.length - ? results.reduce((acc, r) => acc + r.result.score, 0) / results.length - : 1 - return { pass, score, results } -} diff --git a/tests/adapters-otel.test.ts b/tests/adapters-otel.test.ts deleted file mode 100644 index 49e7a297..00000000 --- a/tests/adapters-otel.test.ts +++ /dev/null @@ -1,505 +0,0 @@ -/** - * OTel→hosted bridge — unit + E2E. - * - * Verifies the OTel-shape → wire-format conversion and the end-to-end path - * from a synthetic OTel-style span batch into the reference receiver. - */ - -import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import type { TenantConfig } from '../examples/hosted-ingest-server/server' -import { - createOtelBridge, - hrTimeToUnixNano, - OTEL_STATUS_ERROR, - OTEL_STATUS_OK, - OTEL_STATUS_UNSET, - type OtelAttributeValue, - type OtelLikeSpan, -} from '../src/adapters/otel' -import { fromOtelSpans } from '../src/contract/intake/otel-spans' -import { createHostedClient } from '../src/hosted/client' -import type { TraceSpanEvent } from '../src/hosted/types' -import { startReceiver } from './_fixtures/hosted-receiver' - -function makeSpan(overrides: Partial = {}): OtelLikeSpan { - const base: OtelLikeSpan = { - spanContext: () => ({ traceId: 't-1', spanId: 's-1' }), - name: 'dispatch', - startTime: [1_700_000_000, 0], - endTime: [1_700_000_001, 500_000_000], - attributes: {}, - } - return { ...base, ...overrides } -} - -function makeTraceSpan( - overrides: Partial & Pick, -): TraceSpanEvent { - return { - traceId: 'trace-score', - startTimeUnixNano: '0', - endTimeUnixNano: '1000000', - attributes: {}, - ...overrides, - } -} - -describe('hrTimeToUnixNano', () => { - it('converts [s, ns] to unix-nano', () => { - expect(hrTimeToUnixNano([1, 0])).toBe('1000000000') - expect(hrTimeToUnixNano([1, 500_000_000])).toBe('1500000000') - expect(hrTimeToUnixNano([0, 1])).toBe('1') - }) -}) - -describe('createOtelBridge — spanToEvent conversion', () => { - const fakeClient = { - tenant: { endpoint: '', apiKey: '', tenantId: '' }, - wireVersion: '2026-07-24.v1' as const, - ingestEvalRun: async () => ({ accepted: 0, rejected: [] }), - ingestEvalRuns: async () => ({ accepted: 0, rejected: [] }), - ingestTraces: async () => ({ accepted: 0, rejected: [] }), - } - - it('preserves traceId, spanId, name, and time fields', () => { - const bridge = createOtelBridge({ client: fakeClient }) - const e = bridge.spanToEvent( - makeSpan({ - spanContext: () => ({ traceId: 'abc', spanId: 'def' }), - name: 'my-op', - startTime: [2, 100], - endTime: [3, 200], - }), - ) - expect(e.traceId).toBe('abc') - expect(e.spanId).toBe('def') - expect(e.name).toBe('my-op') - expect(e.startTimeUnixNano).toBe('2000000100') - expect(e.endTimeUnixNano).toBe('3000000200') - }) - - it('maps OTel status codes to wire-format strings', () => { - const bridge = createOtelBridge({ client: fakeClient }) - expect(bridge.spanToEvent(makeSpan({ status: { code: OTEL_STATUS_OK } })).status?.code).toBe( - 'OK', - ) - expect( - bridge.spanToEvent(makeSpan({ status: { code: OTEL_STATUS_ERROR, message: 'boom' } })).status, - ).toEqual({ code: 'ERROR', message: 'boom' }) - expect(bridge.spanToEvent(makeSpan({ status: { code: OTEL_STATUS_UNSET } })).status?.code).toBe( - 'UNSET', - ) - }) - - it('drops undefined/null attribute values; keeps string/number/boolean', () => { - const bridge = createOtelBridge({ client: fakeClient }) - const e = bridge.spanToEvent( - makeSpan({ - attributes: { - 'a.string': 'x', - 'a.num': 42, - 'a.bool': true, - 'a.null': null, - 'a.undef': undefined, - }, - }), - ) - expect(e.attributes).toEqual({ 'a.string': 'x', 'a.num': 42, 'a.bool': true }) - }) - - it('promotes tangle.* attributes to first-class wire fields', () => { - const bridge = createOtelBridge({ client: fakeClient }) - const e = bridge.spanToEvent( - makeSpan({ - attributes: { - 'tangle.runId': 'run-abc', - 'tangle.scenarioId': 'sc-1', - 'tangle.cellId': 'cell-3', - 'tangle.generation': 2, - }, - }), - ) - expect(e['tangle.runId']).toBe('run-abc') - expect(e['tangle.scenarioId']).toBe('sc-1') - expect(e['tangle.cellId']).toBe('cell-3') - expect(e['tangle.generation']).toBe(2) - // The pivot keys remain in attributes too so OTel viewers still see them. - expect(e.attributes['tangle.runId']).toBe('run-abc') - }) - - it('applies defaultRunId when the span lacks tangle.runId', () => { - const bridge = createOtelBridge({ client: fakeClient, defaultRunId: 'fallback-run' }) - const e = bridge.spanToEvent(makeSpan({ attributes: { 'http.method': 'GET' } })) - expect(e['tangle.runId']).toBe('fallback-run') - expect(e.attributes['tangle.runId']).toBe('fallback-run') - }) - - it('does NOT clobber an explicit tangle.runId with defaultRunId', () => { - const bridge = createOtelBridge({ client: fakeClient, defaultRunId: 'fallback' }) - const e = bridge.spanToEvent(makeSpan({ attributes: { 'tangle.runId': 'explicit' } })) - expect(e['tangle.runId']).toBe('explicit') - }) - - it('resolves parentSpanId from either parentSpanId or parentSpanContext()', () => { - const bridge = createOtelBridge({ client: fakeClient }) - const fromField = bridge.spanToEvent(makeSpan({ parentSpanId: 'p-1' })) - expect(fromField.parentSpanId).toBe('p-1') - const fromCtx = bridge.spanToEvent(makeSpan({ parentSpanContext: () => ({ spanId: 'p-2' }) })) - expect(fromCtx.parentSpanId).toBe('p-2') - const none = bridge.spanToEvent(makeSpan()) - expect(none.parentSpanId).toBeUndefined() - }) - - it('forwards span events including their attributes + times', () => { - const bridge = createOtelBridge({ client: fakeClient }) - const e = bridge.spanToEvent( - makeSpan({ - events: [ - { - name: 'exception', - time: [1_700_000_001, 0], - attributes: { 'exception.message': 'oops', dropme: undefined }, - }, - ], - }), - ) - expect(e.events).toHaveLength(1) - expect(e.events?.[0]?.name).toBe('exception') - expect(e.events?.[0]?.timeUnixNano).toBe('1700000001000000000') - expect(e.events?.[0]?.attributes).toEqual({ 'exception.message': 'oops' }) - }) - - it('omits attributes on event nodes when none survive cleaning', () => { - const bridge = createOtelBridge({ client: fakeClient }) - const e = bridge.spanToEvent( - makeSpan({ events: [{ name: 'tick', time: [1, 0], attributes: { drop: null } }] }), - ) - expect(e.events?.[0]?.attributes).toBeUndefined() - }) - - it('serialises array-valued attributes as JSON strings', () => { - const bridge = createOtelBridge({ client: fakeClient }) - const e = bridge.spanToEvent( - makeSpan({ - attributes: { - 'tool.names': ['search', 'read'], - numbers: [1, 2, 3], - bools: [true, false], - } as Record, - }), - ) - expect(e.attributes['tool.names']).toBe('["search","read"]') - expect(e.attributes.numbers).toBe('[1,2,3]') - expect(e.attributes.bools).toBe('[true,false]') - }) -}) - -describe('fromOtelSpans task-quality extraction', () => { - it('ignores generic root scores and score-like guardrail child attributes', () => { - const root = makeTraceSpan({ - spanId: 'root', - name: 'agent.run', - attributes: { - 'openinference.span.kind': 'AGENT', - score: 0.7, - }, - status: { code: 'OK' }, - }) - const guardrail = makeTraceSpan({ - spanId: 'guardrail', - parentSpanId: 'root', - name: 'guardrail.check', - attributes: { - 'openinference.span.kind': 'GUARDRAIL', - 'tangle.score': 0.99, - score: 1, - }, - status: { code: 'ERROR' }, - }) - - const [run] = fromOtelSpans({ spans: [guardrail, root] }) - - expect(run?.outcome.holdoutScore).toBeUndefined() - expect(run?.outcome.judgeScores).toBeUndefined() - expect(run?.terminalOutcome).toBe('succeeded') - expect(run?.outcome.raw).toMatchObject({ - error_span_count: 1, - execution_error_count: 0, - guardrail_error_count: 1, - }) - }) - - it('accepts the official evaluation score on an EVALUATOR span', () => { - const root = makeTraceSpan({ - spanId: 'root', - name: 'agent.run', - attributes: { - 'openinference.span.kind': 'AGENT', - score: 0.1, - }, - status: { code: 'OK' }, - }) - const evaluator = makeTraceSpan({ - spanId: 'evaluator', - parentSpanId: 'root', - name: 'evaluate.correctness', - attributes: { - 'openinference.span.kind': 'EVALUATOR', - 'gen_ai.evaluation.score.value': '0.84', - score: 0.2, - }, - status: { code: 'OK' }, - }) - - const [run] = fromOtelSpans({ spans: [root, evaluator] }) - - expect(run?.outcome.holdoutScore).toBe(0.84) - expect(run?.outcome.judgeScores?.composite).toBe(0.84) - expect(run?.terminalOutcome).toBe('succeeded') - }) - - it.each(['search', 'dev'] as const)( - 'writes a %s trace score only to searchScore', - (defaultSplit) => { - const root = makeTraceSpan({ - spanId: 'root', - name: 'agent.run', - attributes: { - 'openinference.span.kind': 'AGENT', - 'tangle.task.score': 0.73, - }, - status: { code: 'OK' }, - }) - - const [run] = fromOtelSpans({ spans: [root], defaultSplit }) - - expect(run?.splitTag).toBe(defaultSplit) - expect(run?.outcome.searchScore).toBe(0.73) - expect(run?.outcome.holdoutScore).toBeUndefined() - }, - ) - - it('does not accept a task-quality label from an errored evaluator', () => { - const root = makeTraceSpan({ - spanId: 'root', - name: 'agent.run', - attributes: { 'openinference.span.kind': 'AGENT' }, - status: { code: 'OK' }, - }) - const evaluator = makeTraceSpan({ - spanId: 'evaluator', - parentSpanId: 'root', - name: 'evaluate.correctness', - attributes: { - 'openinference.span.kind': 'EVALUATOR', - 'gen_ai.evaluation.score.value': 0.99, - }, - status: { code: 'ERROR', message: 'judge timed out' }, - }) - - const [run] = fromOtelSpans({ spans: [root, evaluator] }) - - expect(run?.outcome.searchScore).toBeUndefined() - expect(run?.outcome.holdoutScore).toBeUndefined() - expect(run?.outcome.judgeScores).toBeUndefined() - expect(run?.outcome.raw).toMatchObject({ - judge_error_count: 1, - execution_error_count: 0, - process_error_count: 0, - }) - expect(run?.terminalOutcome).toBe('succeeded') - }) - - it('classifies an errored model-metadata child as one execution error', () => { - const root = makeTraceSpan({ - spanId: 'root', - name: 'agent.run', - attributes: { 'openinference.span.kind': 'AGENT' }, - status: { code: 'OK' }, - }) - const modelCall = makeTraceSpan({ - spanId: 'model-call', - parentSpanId: 'root', - name: 'provider.request', - attributes: { 'gen_ai.request.model': 'gpt-5@2026-06-05' }, - status: { code: 'ERROR', message: 'provider unavailable' }, - }) - - const [run] = fromOtelSpans({ spans: [root, modelCall] }) - - expect(run?.outcome.raw).toMatchObject({ - llm_span_count: 1, - execution_error_count: 1, - process_error_count: 0, - unclassified_error_count: 0, - }) - expect(run?.terminalOutcome).toBe('succeeded') - }) - - it('accepts an explicit caller score callback', () => { - const root = makeTraceSpan({ - spanId: 'root', - name: 'agent.run', - status: { code: 'OK' }, - }) - - const [run] = fromOtelSpans({ - spans: [root], - scoreForRun: (runId, spans) => { - expect(runId).toBe('trace-score') - expect(spans.map((span) => span.spanId)).toEqual(['root']) - return 0.91 - }, - }) - - expect(run?.outcome.holdoutScore).toBe(0.91) - }) - - it('keeps root terminal status separate from task quality', () => { - const root = makeTraceSpan({ - spanId: 'root', - name: 'agent.run', - attributes: { score: 0 }, - status: { code: 'ERROR', message: 'process failed' }, - }) - - const [run] = fromOtelSpans({ spans: [root] }) - - expect(run?.terminalOutcome).toBe('failed') - expect(run?.terminalFailureReason).toBe('process failed') - expect(run?.outcome.holdoutScore).toBeUndefined() - expect(run?.outcome.judgeScores).toBeUndefined() - expect(run?.outcome.raw).toMatchObject({ - error_span_count: 1, - execution_error_count: 0, - process_error_count: 1, - }) - }) - - it.each([ - ['blank', ' '], - ['non-finite', Number.POSITIVE_INFINITY], - ])('rejects a %s designated task-quality score', (_name, score) => { - const root = makeTraceSpan({ - spanId: 'root', - name: 'agent.run', - attributes: { 'gen_ai.evaluation.score.value': score }, - }) - - expect(() => fromOtelSpans({ spans: [root] })).toThrow( - /task quality must be finite|not a finite/, - ) - }) - - it('reports conflicting explicit sources independent of span order', () => { - const root = makeTraceSpan({ - spanId: 'root', - name: 'agent.run', - attributes: { 'tangle.task.score': 0.2 }, - }) - const evaluator = makeTraceSpan({ - spanId: 'evaluator', - parentSpanId: 'root', - name: 'evaluate.correctness', - attributes: { - 'openinference.span.kind': 'EVALUATOR', - 'gen_ai.evaluation.score.value': 0.8, - }, - }) - const messages = [ - [root, evaluator], - [evaluator, root], - ].map((spans) => { - try { - fromOtelSpans({ spans, scoreForRun: () => 0.5 }) - throw new Error('expected fromOtelSpans to reject conflicting task scores') - } catch (error) { - return error instanceof Error ? error.message : String(error) - } - }) - - expect(messages[0]).toBe(messages[1]) - expect(messages[0]).toMatch(/conflicting task-quality scores/) - expect(messages[0]).toContain('scoreForRun=0.5') - expect(messages[0]).toContain("span 'root' attribute 'tangle.task.score'=0.2") - expect(messages[0]).toContain("span 'evaluator' attribute 'gen_ai.evaluation.score.value'=0.8") - }) -}) - -// ── E2E: bridge → reference receiver ───────────────────────────────── - -const TENANT: TenantConfig = { id: 'acme', key: 'k' } - -describe('OTel bridge — E2E against reference receiver', () => { - let stop: () => Promise - let baseUrl: string - - beforeEach(async () => { - const r = await startReceiver([TENANT]) - baseUrl = r.baseUrl - stop = r.stop - }) - - afterEach(async () => { - await stop() - }) - - it('batches a set of OTel-shape spans into hosted ingest', async () => { - const client = createHostedClient({ - endpoint: baseUrl, - apiKey: TENANT.key, - tenantId: TENANT.id, - }) - const bridge = createOtelBridge({ client, defaultRunId: 'run-otel-1', batchSize: 5 }) - const spans: OtelLikeSpan[] = Array.from({ length: 12 }, (_, i) => ({ - spanContext: () => ({ traceId: 't-otel', spanId: `s-${i}` }), - name: `step-${i}`, - startTime: [1_700_000_000 + i, 0], - endTime: [1_700_000_000 + i, 500_000_000], - attributes: { 'step.index': i }, - status: { code: OTEL_STATUS_OK }, - })) - await bridge.ingest(spans) - - const res = await fetch(`${baseUrl}/v1/runs/run-otel-1/traces`, { - headers: { - Authorization: `Bearer ${TENANT.key}`, - 'X-Tangle-Tenant-Id': TENANT.id, - 'X-Tangle-Wire-Version': '2026-07-24.v1', - }, - }) - const body = (await res.json()) as { spans: Array<{ spanId: string; name: string }> } - expect(body.spans).toHaveLength(12) - expect(body.spans.map((s) => s.spanId).sort()).toEqual( - spans.map((s) => s.spanContext().spanId).sort(), - ) - }) - - it('invokes onError when the upstream ingest fails', async () => { - const client = createHostedClient({ - endpoint: baseUrl, - apiKey: 'wrong-key', - tenantId: TENANT.id, - retries: 0, - }) - const errors: unknown[] = [] - const bridge = createOtelBridge({ - client, - defaultRunId: 'r', - onError: (err) => { - errors.push(err) - }, - }) - await bridge.ingest([ - { - spanContext: () => ({ traceId: 't', spanId: 's' }), - name: 'x', - startTime: [1, 0], - endTime: [1, 1], - attributes: {}, - }, - ]) - expect(errors).toHaveLength(1) - expect(String(errors[0])).toMatch(/401/) - }) -}) diff --git a/tests/auto-pr.test.ts b/tests/auto-pr.test.ts deleted file mode 100644 index 132194ca..00000000 --- a/tests/auto-pr.test.ts +++ /dev/null @@ -1,295 +0,0 @@ -/** - * Auto-PR tests. - * - * Regression coverage: - * - input validation (bad branch names, '..' paths, duplicate paths) - * - HTTP client opens a PR via the documented REST sequence (blob → - * tree → commit → ref → pulls) - * - HTTP client is idempotent: existing open PR is returned instead - * of a duplicate create - * - HTTP client fast-forwards the ref when it already exists with a - * different SHA - * - dryRun does not call fetch/exec - * - * No network. - */ -import { describe, expect, it, vi } from 'vitest' - -import { ghCliClient, httpGithubClient } from '../src/auto-pr' -import { ValidationError } from '../src/errors' - -function jsonResponse(body: unknown, status = 200): Response { - return new Response(JSON.stringify(body), { - status, - headers: { 'content-type': 'application/json' }, - }) -} - -describe('proposeChange input validation', () => { - const fixture = { - repo: { owner: 'tangle-network', name: 'tax-agent' }, - branchName: 'eval/auto-improve/r1', - fileChanges: [{ path: 'prompts/system.txt', contents: 'new' }], - title: 'feat: production-loop', - body: 'body', - } - - // Validation runs at the top of every client's proposeChange, before any - // network/process work — a fetch spy that must never fire proves it. - function clientWithSpy() { - const fetchImpl = vi.fn() - const client = httpGithubClient({ token: 'test-token', fetchImpl: fetchImpl as never }) - return { client, fetchImpl } - } - - it('rejects an empty repo owner', async () => { - const { client, fetchImpl } = clientWithSpy() - await expect( - client.proposeChange({ ...fixture, repo: { owner: '', name: 'x' } }), - ).rejects.toBeInstanceOf(ValidationError) - expect(fetchImpl).not.toHaveBeenCalled() - }) - - it('rejects whitespace branch names', async () => { - const { client } = clientWithSpy() - await expect( - client.proposeChange({ ...fixture, branchName: 'has space' }), - ).rejects.toBeInstanceOf(ValidationError) - }) - - it('rejects path traversal', async () => { - const { client } = clientWithSpy() - await expect( - client.proposeChange({ - ...fixture, - fileChanges: [{ path: '../etc/passwd', contents: '' }], - }), - ).rejects.toBeInstanceOf(ValidationError) - }) - - it('rejects duplicate paths', async () => { - const { client } = clientWithSpy() - await expect( - client.proposeChange({ - ...fixture, - fileChanges: [ - { path: 'a.txt', contents: '1' }, - { path: 'a.txt', contents: '2' }, - ], - }), - ).rejects.toBeInstanceOf(ValidationError) - }) - - it('rejects branch name equal to base branch', async () => { - const { client } = clientWithSpy() - await expect( - client.proposeChange({ ...fixture, branchName: 'main', baseBranch: 'main' }), - ).rejects.toBeInstanceOf(ValidationError) - }) - - it('empty title is rejected', async () => { - const { client } = clientWithSpy() - await expect(client.proposeChange({ ...fixture, title: ' ' })).rejects.toBeInstanceOf( - ValidationError, - ) - }) - - it('the gh CLI client validates too (path traversal rejected before any spawn)', async () => { - const exec = vi.fn() - const client = ghCliClient({ exec: exec as never }) - await expect( - client.proposeChange({ ...fixture, fileChanges: [{ path: '../x', contents: '' }] }), - ).rejects.toBeInstanceOf(ValidationError) - expect(exec).not.toHaveBeenCalled() - }) -}) - -describe('httpGithubClient', () => { - function fakeFetch(handler: (url: string, init: RequestInit) => Response): typeof fetch { - return ((url: RequestInfo | URL, init?: RequestInit) => - Promise.resolve(handler(String(url), init ?? {}))) as typeof fetch - } - - function transcript() { - const requests: Array<{ method: string; url: string; body: unknown }> = [] - return { - requests, - record(url: string, init: RequestInit) { - requests.push({ - method: init.method ?? 'GET', - url, - body: init.body ? JSON.parse(init.body as string) : null, - }) - }, - } - } - - it('opens a PR via blob → tree → commit → ref → pulls', async () => { - const t = transcript() - const client = httpGithubClient({ - token: 'test-token', - fetchImpl: fakeFetch((url, init) => { - t.record(url, init) - // Base ref - if (url.endsWith('/git/ref/heads/main')) { - return jsonResponse({ ref: 'refs/heads/main', object: { sha: 'base-sha' } }) - } - // Base commit (for tree.sha) - if (url.endsWith('/git/commits/base-sha')) { - return jsonResponse({ sha: 'base-sha', tree: { sha: 'base-tree-sha' } }) - } - // Create blob - if (url.endsWith('/git/blobs') && init.method === 'POST') { - return jsonResponse({ sha: 'blob-sha-1' }) - } - // Create tree - if (url.endsWith('/git/trees') && init.method === 'POST') { - return jsonResponse({ sha: 'tree-sha-1' }) - } - // Create commit - if (url.endsWith('/git/commits') && init.method === 'POST') { - return jsonResponse({ sha: 'commit-sha-1', tree: { sha: 'tree-sha-1' } }) - } - // Branch ref existence (404 -> not present) - if (url.includes('/git/ref/heads/eval')) { - return jsonResponse({ message: 'Not Found' }, 404) - } - // Create ref - if (url.endsWith('/git/refs') && init.method === 'POST') { - return jsonResponse({ ref: 'refs/heads/eval/r1', object: { sha: 'commit-sha-1' } }) - } - // List PRs (empty -> nothing exists) - if (url.includes('/pulls?')) { - return jsonResponse([]) - } - // Create PR - if (url.endsWith('/pulls') && init.method === 'POST') { - return jsonResponse({ - html_url: 'https://github.com/o/r/pull/1', - number: 1, - }) - } - // Reviewers / labels (best-effort, accept any) - return jsonResponse({}, 200) - }), - now: () => new Date('2026-01-01T00:00:00Z'), - }) - - const result = await client.proposeChange({ - repo: { owner: 'o', name: 'r' }, - branchName: 'eval/r1', - fileChanges: [{ path: 'a.txt', contents: 'hello' }], - title: 'T', - body: 'B', - }) - - expect(result.prUrl).toBe('https://github.com/o/r/pull/1') - expect(result.headSha).toBe('commit-sha-1') - expect(result.dryRun).toBe(false) - - // Verify documented REST sequence in order. - const ordered = t.requests.map((r) => `${r.method} ${r.url.replace(/\?.*$/, '')}`) - expect(ordered).toContain('GET https://api.github.com/repos/o/r/git/ref/heads/main') - expect(ordered).toContain('POST https://api.github.com/repos/o/r/git/blobs') - expect(ordered).toContain('POST https://api.github.com/repos/o/r/git/trees') - expect(ordered).toContain('POST https://api.github.com/repos/o/r/git/commits') - expect(ordered).toContain('POST https://api.github.com/repos/o/r/git/refs') - expect(ordered).toContain('POST https://api.github.com/repos/o/r/pulls') - }) - - it('returns the existing open PR instead of opening a duplicate (idempotency)', async () => { - const client = httpGithubClient({ - token: 'tok', - fetchImpl: fakeFetch((url, init) => { - if (url.endsWith('/git/ref/heads/main')) { - return jsonResponse({ ref: 'refs/heads/main', object: { sha: 'base-sha' } }) - } - if (url.endsWith('/git/commits/base-sha')) { - return jsonResponse({ sha: 'base-sha', tree: { sha: 'base-tree' } }) - } - if (url.endsWith('/git/blobs')) return jsonResponse({ sha: 'b1' }) - if (url.endsWith('/git/trees')) return jsonResponse({ sha: 't1' }) - if (url.endsWith('/git/commits') && init.method === 'POST') { - return jsonResponse({ sha: 'c1', tree: { sha: 't1' } }) - } - // Branch already exists at same sha. - if (url.endsWith('/git/ref/heads/eval/r1')) { - return jsonResponse({ ref: 'refs/heads/eval/r1', object: { sha: 'c1' } }) - } - if (url.includes('/pulls?')) { - return jsonResponse([{ html_url: 'https://github.com/o/r/pull/77', number: 77 }]) - } - return jsonResponse({}, 200) - }), - }) - - const result = await client.proposeChange({ - repo: { owner: 'o', name: 'r' }, - branchName: 'eval/r1', - fileChanges: [{ path: 'a.txt', contents: 'x' }], - title: 'T', - body: 'B', - }) - - expect(result.prUrl).toBe('https://github.com/o/r/pull/77') - }) - - it('dryRun does not call fetch', async () => { - const fetchSpy = vi.fn() - const client = httpGithubClient({ - token: 'tok', - fetchImpl: fetchSpy as unknown as typeof fetch, - }) - const r = await client.proposeChange({ - repo: { owner: 'o', name: 'r' }, - branchName: 'eval/r1', - fileChanges: [{ path: 'a.txt', contents: 'x' }], - title: 'T', - body: 'B', - dryRun: true, - }) - expect(r.dryRun).toBe(true) - expect(fetchSpy).not.toHaveBeenCalled() - }) - - it('surfaces GitHub API failures with status code and body excerpt', async () => { - const client = httpGithubClient({ - token: 'tok', - fetchImpl: () => - Promise.resolve( - new Response('rate limit exceeded', { - status: 403, - headers: { 'content-type': 'text/plain' }, - }), - ), - }) - - await expect( - client.proposeChange({ - repo: { owner: 'o', name: 'r' }, - branchName: 'eval/r1', - fileChanges: [{ path: 'a.txt', contents: 'x' }], - title: 'T', - body: 'B', - }), - ).rejects.toThrow(/403/) - }) -}) - -describe('ghCliClient', () => { - it('dryRun returns a synthetic compare URL without exec calls', async () => { - const execSpy = vi.fn() - const client = ghCliClient({ exec: execSpy as unknown as never }) - const r = await client.proposeChange({ - repo: { owner: 'o', name: 'r' }, - branchName: 'eval/r1', - fileChanges: [{ path: 'a.txt', contents: 'x' }], - title: 'T', - body: 'B', - dryRun: true, - }) - expect(r.dryRun).toBe(true) - expect(r.prUrl).toContain('compare/main...eval/r1') - expect(execSpy).not.toHaveBeenCalled() - }) -}) diff --git a/tests/dual-agent-bench.test.ts b/tests/dual-agent-bench.test.ts deleted file mode 100644 index 7f8dd7bb..00000000 --- a/tests/dual-agent-bench.test.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { DualAgentBench } from '../src/dual-agent-bench' - -describe('DualAgentBench', () => { - it('converges when the critic returns threshold on round N', async () => { - const bench = new DualAgentBench() - const report = await bench.run({ - scenarios: [{ id: 's1', initialPrompt: 'draft a contract' }], - maxRounds: 5, - convergenceThreshold: 0.9, - propose: async ({ roundIndex }) => `proposal round ${roundIndex}`, - critique: async ({ roundIndex }) => ({ - critique: `critique ${roundIndex}`, - // Converge on round 2 (index 2 = third round) - convergenceScore: roundIndex >= 2 ? 1.0 : 0.5, - }), - }) - expect(report.scenarios[0].converged).toBe(true) - expect(report.scenarios[0].roundsToConverge).toBe(3) - expect(report.aggregate.convergenceRate).toBe(1) - }) - - it('records full history — regression: silent loss of intermediate rounds breaks forensics', async () => { - const bench = new DualAgentBench() - const report = await bench.run({ - scenarios: [{ id: 's1', initialPrompt: 'x' }], - maxRounds: 3, - convergenceThreshold: 2, // impossible; forces all rounds to run - propose: async ({ roundIndex }) => `p${roundIndex}`, - critique: async ({ roundIndex }) => ({ - critique: `c${roundIndex}`, - convergenceScore: 0.1, - }), - }) - expect(report.scenarios[0].history).toHaveLength(3) - expect(report.scenarios[0].history.map((r) => r.proposal)).toEqual(['p0', 'p1', 'p2']) - expect(report.scenarios[0].converged).toBe(false) - }) - - it('proposer sees prior critique — regression: proposer ignoring critique means no iteration', async () => { - const bench = new DualAgentBench() - const seenCritiques: (string | undefined)[] = [] - await bench.run({ - scenarios: [{ id: 's1', initialPrompt: 'x' }], - maxRounds: 3, - convergenceThreshold: 2, - propose: async ({ priorCritique }) => { - seenCritiques.push(priorCritique) - return 'proposal' - }, - critique: async ({ roundIndex }) => ({ - critique: `round ${roundIndex} critique`, - convergenceScore: 0.1, - }), - }) - expect(seenCritiques[0]).toBeUndefined() - expect(seenCritiques[1]).toBe('round 0 critique') - expect(seenCritiques[2]).toBe('round 1 critique') - }) - - it('rejects out-of-range convergenceScore — regression: >1 would lock max out of the aggregate', async () => { - const bench = new DualAgentBench() - await expect( - bench.run({ - scenarios: [{ id: 's1', initialPrompt: 'x' }], - propose: async () => 'p', - critique: async () => ({ critique: 'c', convergenceScore: 1.5 }), - }), - ).rejects.toThrow(/\[0,1\]/) - }) - - it('rejects empty scenario list', async () => { - const bench = new DualAgentBench() - await expect( - bench.run({ - scenarios: [], - propose: async () => 'p', - critique: async () => ({ critique: 'c', convergenceScore: 0 }), - }), - ).rejects.toThrow(/at least 1/) - }) - - it('fires onRoundComplete per round', async () => { - const bench = new DualAgentBench() - const events: Array<{ scenarioId: string; round: number }> = [] - await bench.run({ - scenarios: [{ id: 's1', initialPrompt: 'x' }], - maxRounds: 2, - convergenceThreshold: 2, - propose: async () => 'p', - critique: async () => ({ critique: 'c', convergenceScore: 0 }), - onRoundComplete: ({ scenarioId, round }) => - events.push({ scenarioId, round: round.roundIndex }), - }) - expect(events).toEqual([ - { scenarioId: 's1', round: 0 }, - { scenarioId: 's1', round: 1 }, - ]) - }) - - it('aggregate.convergenceRate is the fraction that converged', async () => { - const bench = new DualAgentBench() - const report = await bench.run({ - scenarios: [ - { id: 'a', initialPrompt: 'x' }, - { id: 'b', initialPrompt: 'x' }, - { id: 'c', initialPrompt: 'x' }, - { id: 'd', initialPrompt: 'x' }, - ], - maxRounds: 2, - convergenceThreshold: 0.5, - propose: async () => 'p', - critique: async ({ scenario }) => ({ - critique: 'c', - // a + b converge immediately; c + d never - convergenceScore: scenario.id < 'c' ? 1 : 0, - }), - }) - expect(report.aggregate.convergenceRate).toBe(0.5) - }) -}) diff --git a/tests/golden-matcher.test.ts b/tests/golden-matcher.test.ts deleted file mode 100644 index 9850ffee..00000000 --- a/tests/golden-matcher.test.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { describe, expect, it } from 'vitest' -import type { GoldenSpec } from '../src/golden-matcher' -import { - DEFAULT_SEVERITY_WEIGHTS, - precision as goldenPrecision, - matchGoldens, - weightedRecall, -} from '../src/golden-matcher' - -const goldens: GoldenSpec[] = [ - { id: 'a', severity: 'critical', any: ['primary action', 'no clear primary'], hint: '' }, - { id: 'b', severity: 'major', any: ['equal weight'], hint: '' }, - { id: 'c', severity: 'minor', any: ['next step'], hint: '' }, -] - -describe('matchGoldens', () => { - it('matches against a string-only candidate list', () => { - const r = matchGoldens(goldens, ['No clear PRIMARY ACTION on welcome screen']) - expect(r.matches).toEqual([true, false, false]) - expect(r.hits).toBe(1) - expect(r.total).toBe(3) - }) - - it('default extract concatenates string fields', () => { - const r = matchGoldens(goldens, [ - { description: 'buttons compete', location: 'equal weight grid' }, - ]) - expect(r.matches).toEqual([false, true, false]) - }) - - it('honours custom text() extractor', () => { - const r = matchGoldens(goldens, [{ x: 'PRIMARY ACTION missing' }], { text: (c) => c.x }) - expect(r.matches[0]).toBe(true) - }) - - it('handles regex via anyRegex', () => { - const re: GoldenSpec[] = [ - { id: 'r', severity: 'major', any: [], anyRegex: ['no\\s+primary'], hint: '' }, - ] - const r = matchGoldens(re, ['there is no primary CTA']) - expect(r.matches).toEqual([true]) - }) - - it('returns all-false on empty candidates', () => { - expect(matchGoldens(goldens, []).matches).toEqual([false, false, false]) - }) - - it('skips invalid regex without crashing', () => { - const bad: GoldenSpec[] = [{ id: 'b', severity: 'minor', any: [], anyRegex: ['['], hint: '' }] - expect(() => matchGoldens(bad, ['anything'])).not.toThrow() - expect(matchGoldens(bad, ['anything']).matches).toEqual([false]) - }) -}) - -describe('weightedRecall', () => { - it('weights critical 3x, major 2x, minor 1x', () => { - // total weight = 3 + 2 + 1 = 6 - expect( - weightedRecall(goldens, { matches: [true, false, false], hits: 1, total: 3 }), - ).toBeCloseTo(3 / 6) - expect( - weightedRecall(goldens, { matches: [false, true, false], hits: 1, total: 3 }), - ).toBeCloseTo(2 / 6) - expect(weightedRecall(goldens, { matches: [true, true, true], hits: 3, total: 3 })).toBe(1) - expect(weightedRecall(goldens, { matches: [false, false, false], hits: 0, total: 3 })).toBe(0) - }) - - it('returns 1 when no goldens (vacuous)', () => { - expect(weightedRecall([], { matches: [], hits: 0, total: 0 })).toBe(1) - }) - - it('respects custom weights', () => { - const custom = { ...DEFAULT_SEVERITY_WEIGHTS, critical: 10 } - // weight = 10 + 2 + 1 = 13; hit critical only → 10/13 - expect( - weightedRecall(goldens, { matches: [true, false, false], hits: 1, total: 3 }, custom), - ).toBeCloseTo(10 / 13) - }) -}) - -describe('goldenPrecision', () => { - it('returns 1 when no candidates', () => { - expect(goldenPrecision(goldens, [])).toBe(1) - }) - - it('counts the share that match a golden phrase', () => { - const cands = [ - 'Primary action unclear', - 'Some unrelated polish nit', - 'No clear primary visible', - ] - expect(goldenPrecision(goldens, cands)).toBeCloseTo(2 / 3) - }) - - it('honours regex goldens for precision too', () => { - const re: GoldenSpec[] = [ - { id: 'r', severity: 'major', any: [], anyRegex: ['p[a-z]+y action'], hint: '' }, - ] - expect(goldenPrecision(re, ['primary action found', 'noise'])).toBeCloseTo(1 / 2) - }) -}) diff --git a/tests/judge-runner.test.ts b/tests/judge-runner.test.ts deleted file mode 100644 index 53999bb2..00000000 --- a/tests/judge-runner.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { compilerJudge, JudgeRunner, runJudgeFleet, testJudge } from '../src/judge-runner' -import type { HarnessConfig, SandboxDriver, SandboxResult } from '../src/sandbox-harness' - -class FakeDriver implements SandboxDriver { - id = 'fake' - async exec( - phase: SandboxResult['phase'], - _command: string, - _config: HarnessConfig, - ): Promise { - return { - phase, - exitCode: 0, - stdout: phase === 'test' ? 'Tests 4 passed' : '', - stderr: '', - wallMs: 5, - ...(phase === 'test' ? { testsTotal: 4, testsPassed: 4 } : {}), - } - } -} - -describe('judge runner', () => { - it('runs named judges through the sandbox harness', async () => { - const runner = new JudgeRunner(new FakeDriver()) - const result = await runner.run(testJudge('tests', { testCommand: 'pnpm test' })) - expect(result.kind).toBe('test') - expect(result.passed).toBe(true) - expect(result.score).toBe(1) - }) - - it('runs a judge fleet in parallel by default', async () => { - const results = await runJudgeFleet( - [ - compilerJudge('compile', { runCommand: 'pnpm build' }), - testJudge('tests', { testCommand: 'pnpm test' }), - ], - { driver: new FakeDriver() }, - ) - expect(results).toHaveLength(2) - expect(results.every((result) => result.passed)).toBe(true) - }) -}) diff --git a/tests/knowledge-readiness.test.ts b/tests/knowledge-readiness.test.ts index b3de8415..b076e070 100644 --- a/tests/knowledge-readiness.test.ts +++ b/tests/knowledge-readiness.test.ts @@ -3,11 +3,11 @@ import { classifyFailure } from '../src/failure-taxonomy' import { acquisitionPlansForKnowledgeGaps, blockingKnowledgeEval, - type KnowledgeRequirement, knowledgeReadinessTracePayload, scoreKnowledgeReadiness, userQuestionsForKnowledgeGaps, -} from '../src/knowledge' +} from '../src/knowledge/readiness' +import type { KnowledgeRequirement } from '../src/knowledge/types' import type { Run, Span, TraceEvent } from '../src/trace/schema' function req(overrides: Partial = {}): KnowledgeRequirement { diff --git a/tests/prm.test.ts b/tests/prm.test.ts deleted file mode 100644 index f985befe..00000000 --- a/tests/prm.test.ts +++ /dev/null @@ -1,170 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { - exportTrainingData, - isPrmVerdict, - nonRefusalRubric, - outputLengthRubric, - PrmGrader, - prmBestOfN, - prmEnsembleBestOfN, - type StepRubric, - toNdjson, - toolNonRedundantRubric, - toolSuccessRubric, -} from '../src/prm' -import type { ToolSpan } from '../src/trace' -import { InMemoryTraceStore, TraceEmitter } from '../src/trace' - -async function seedTrajectory(store: InMemoryTraceStore, output: string): Promise { - const e = new TraceEmitter(store) - await e.startRun({ scenarioId: 's' }) - const llm = await e.span({ - kind: 'llm', - name: 'gen', - model: 'm', - messages: [{ role: 'user', content: 'hi' }], - output, - }) - await llm.end() - const tool = await e.span({ kind: 'tool', name: 'search', toolName: 'search', args: { q: 'x' } }) - await tool.end({ result: 'result text here' } as Partial) - await e.endRun({ pass: true }) - return e.runId -} - -describe('PrmGrader', () => { - it('grades per-span, emits JudgeVerdict span, aggregates via weighted mean', async () => { - const store = new InMemoryTraceStore() - const runId = await seedTrajectory( - store, - 'This is a normal-length response that should score well.', - ) - const grader = new PrmGrader([outputLengthRubric(), toolSuccessRubric()]) - const graded = await grader.grade(store, runId) - expect(graded.gradedCount).toBe(2) - expect(graded.aggregateScore).toBeGreaterThan(0.8) - const verdicts = (await store.spans({ kind: 'judge' })).filter(isPrmVerdict) - expect(verdicts).toHaveLength(2) - }) - - it('penalizes empty outputs via outputLengthRubric — regression: silent empty responses rewarded was the bug', async () => { - const store = new InMemoryTraceStore() - const runId = await seedTrajectory(store, '') - const grader = new PrmGrader([outputLengthRubric()]) - const graded = await grader.grade(store, runId) - expect(graded.aggregateScore).toBe(0) - }) - - it('toolNonRedundantRubric flags duplicate tool calls', async () => { - const store = new InMemoryTraceStore() - const e = new TraceEmitter(store) - await e.startRun({ scenarioId: 's' }) - const a = await e.tool({ name: 'search', toolName: 'search', args: { q: 'x' } }) - await a.end({ result: 'ok' } as Partial) - const b = await e.tool({ name: 'search', toolName: 'search', args: { q: 'x' } }) - await b.end({ result: 'ok' } as Partial) - await e.endRun({ pass: true }) - const graded = await new PrmGrader([toolNonRedundantRubric()]).grade(store, e.runId) - // Second call duplicates first → score 0.5 - const duplicateVerdict = graded.steps.find((s) => s.score < 1) - expect(duplicateVerdict).toBeDefined() - }) - - it('does not score redundancy without captured arguments', async () => { - const store = new InMemoryTraceStore() - const e = new TraceEmitter(store) - await e.startRun({ scenarioId: 's' }) - for (const argsCaptured of [false, false, true, true]) { - const call = await e.tool({ - name: 'search', - toolName: 'search', - args: argsCaptured ? { q: 'x' } : undefined, - argsCaptured, - }) - await call.end({ result: 'ok' } as Partial) - } - await e.endRun({ pass: true }) - - const graded = await new PrmGrader([toolNonRedundantRubric()]).grade(store, e.runId) - - expect(graded.steps.map(({ score, rationale }) => ({ score, rationale }))).toEqual([ - { score: 1, rationale: 'novel call' }, - { score: 0.5, rationale: '1 duplicate(s)' }, - ]) - expect(graded.ungradedCount).toBe(2) - }) - - it('nonRefusalRubric scores 0 on a refusal', async () => { - const store = new InMemoryTraceStore() - const runId = await seedTrajectory(store, 'I cannot help with that.') - const graded = await new PrmGrader([nonRefusalRubric()]).grade(store, runId) - expect(graded.aggregateScore).toBe(0) - }) - - it('empty rubric list throws', () => { - expect(() => new PrmGrader([])).toThrow(/at least 1 rubric/) - }) - - it('returns null verdict when rubric does not apply', async () => { - const store = new InMemoryTraceStore() - const runId = await seedTrajectory(store, 'ok') - const rubric: StepRubric = { - id: 'custom', - kinds: ['llm'], - async grade() { - return null - }, - } - const graded = await new PrmGrader([rubric]).grade(store, runId) - expect(graded.gradedCount).toBe(0) - expect(graded.ungradedCount).toBeGreaterThan(0) - }) -}) - -describe('training export', () => { - it('emits NDJSON with step context', async () => { - const store = new InMemoryTraceStore() - const runId = await seedTrajectory( - store, - 'normal response that is long enough for the length rubric to score well', - ) - const graded = await new PrmGrader([outputLengthRubric(), toolSuccessRubric()]).grade( - store, - runId, - ) - const samples = await exportTrainingData(store, [graded]) - expect(samples.length).toBeGreaterThan(0) - expect(samples[0].context.step.text.length).toBeGreaterThan(0) - const ndjson = toNdjson(samples) - expect(ndjson.split('\n').filter(Boolean)).toHaveLength(samples.length) - }) -}) - -describe('prmBestOfN', () => { - it('picks the highest-scoring candidate trajectory', async () => { - const store = new InMemoryTraceStore() - const good = await seedTrajectory( - store, - 'This is the better, longer response that the PRM will reward.', - ) - const bad = await seedTrajectory(store, 'ok') - const grader = new PrmGrader([outputLengthRubric()]) - const result = await prmBestOfN(store, grader, [good, bad]) - expect(result.winner.runId).toBe(good) - expect(result.ranked.map((r) => r.runId)).toEqual([good, bad]) - expect(result.stdDev).toBeGreaterThan(0) - }) - - it('ensemble via Borda count robust to score-scale differences', async () => { - const store = new InMemoryTraceStore() - const a = await seedTrajectory( - store, - 'great response of reasonable length for scoring purposes here now', - ) - const b = await seedTrajectory(store, '') - const g1 = new PrmGrader([outputLengthRubric()]) - const g2 = new PrmGrader([toolSuccessRubric()]) - const result = await prmEnsembleBestOfN(store, [g1, g2], [a, b]) - expect(result.winner.runId).toBe(a) - }) -}) diff --git a/tests/registry.test.ts b/tests/registry.test.ts deleted file mode 100644 index b80b158b..00000000 --- a/tests/registry.test.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { ScenarioRegistry } from '../src/registry' -import type { ScenarioFile } from '../src/types' - -describe('ScenarioRegistry', () => { - const sampleFile: ScenarioFile = { - id: 'test-scenario-1', - category: 'pipeline-build', - persona: 'engineer', - label: 'Engineer', - thesis: 'Can the agent build a pipeline?', - turns: [{ user: 'Build me a pipeline', expectedBehaviors: ['produces code'] }], - artifactChecks: [{ type: 'code_valid', target: 'python', description: 'Valid Python code' }], - } - - it('registers and retrieves scenarios', () => { - const registry = new ScenarioRegistry() - registry.registerFiles([sampleFile]) - expect(registry.count).toBe(1) - expect(registry.all()[0].id).toBe('test-scenario-1') - }) - - it('filters by category', () => { - const registry = new ScenarioRegistry() - registry.registerFiles([sampleFile, { ...sampleFile, id: 'test-2', category: 'adversarial' }]) - expect(registry.byCategory('pipeline-build')).toHaveLength(1) - expect(registry.byCategory('adversarial')).toHaveLength(1) - expect(registry.byCategory('nonexistent')).toHaveLength(0) - }) - - it('filters by persona', () => { - const registry = new ScenarioRegistry() - registry.registerFiles([sampleFile]) - registry.register([ - { - id: 'direct-scenario', - persona: 'designer', - label: 'Designer', - thesis: 'test', - dimensions: [], - turns: [], - artifactChecks: [], - }, - ]) - expect(registry.byPersona('engineer')).toHaveLength(1) - expect(registry.byPersona('designer')).toHaveLength(1) - }) - - it('lists categories', () => { - const registry = new ScenarioRegistry() - registry.registerFiles([ - sampleFile, - { ...sampleFile, id: 'test-2', category: 'pipeline-build' }, - { ...sampleFile, id: 'test-3', category: 'adversarial' }, - ]) - const cats = registry.listCategories() - expect(cats).toHaveLength(2) - expect(cats.find((c) => c.category === 'pipeline-build')?.count).toBe(2) - }) - - it('finds by id', () => { - const registry = new ScenarioRegistry() - registry.registerFiles([sampleFile]) - expect(registry.byId('test-scenario-1')?.thesis).toBe('Can the agent build a pipeline?') - expect(registry.byId('nonexistent')).toBeUndefined() - }) -}) diff --git a/tests/ui-finding.test.ts b/tests/ui-finding.test.ts deleted file mode 100644 index 2021f57b..00000000 --- a/tests/ui-finding.test.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { - UI_FINDING_SEVERITIES, - UI_LENSES, - type UiFinding, - type UiFindingSeverity, - type UiLens, -} from '../src/ui-finding' - -describe('UI_LENSES', () => { - it('matches the UiLens union exhaustively', () => { - // The runtime tuple and the type must stay in sync — if a lens is added - // to the union without updating the tuple, a lens-keyed Record will - // silently lose dimensions in downstream code. Enforce both directions. - const witness: Record = { - consistency: true, - hierarchy: true, - layout: true, - 'ux-flow': true, - duplication: true, - accessibility: true, - responsive: true, - states: true, - content: true, - interaction: true, - 'performance-perceived': true, - other: true, - } - for (const lens of UI_LENSES) { - expect(witness[lens]).toBe(true) - } - expect(Object.keys(witness).sort()).toEqual([...UI_LENSES].sort()) - }) - - it('contains no duplicates', () => { - expect(new Set(UI_LENSES).size).toBe(UI_LENSES.length) - }) -}) - -describe('UI_FINDING_SEVERITIES', () => { - it('orders severities worst → least bad', () => { - expect(UI_FINDING_SEVERITIES).toEqual(['critical', 'high', 'med', 'low']) - }) - - it('matches the UiFindingSeverity union exhaustively', () => { - const witness: Record = { - critical: true, - high: true, - med: true, - low: true, - } - expect(Object.keys(witness).sort()).toEqual([...UI_FINDING_SEVERITIES].sort()) - }) -}) - -describe('UiFinding shape', () => { - it('accepts a minimally complete finding', () => { - const finding: UiFinding = { - title: 'Primary CTA invisible on hover state', - lens: 'interaction', - severity: 'med', - route: 'home', - observation: 'CTA background blends into card background on hover.', - impact: 'Users lose the affordance during the moment they need it most.', - suggestedFix: 'Add 2px border on hover instead of swapping background.', - screenshots: [{ path: 'screenshots/home--cta--hover.png', viewport: '1280x800' }], - } - expect(finding.title.length).toBeGreaterThan(0) - expect(finding.screenshots.length).toBe(1) - }) - - it('treats `screenshots` and `tags` as readonly arrays', () => { - const finding: UiFinding = { - title: 't', - lens: 'consistency', - severity: 'low', - route: 'r', - observation: 'o', - impact: 'i', - suggestedFix: 's', - screenshots: [{ path: 'a.png' }], - tags: ['nav', 'header'], - } - // @ts-expect-error — screenshots is readonly. - finding.screenshots.push({ path: 'b.png' }) - // @ts-expect-error — tags is readonly. - finding.tags?.push('extra') - }) -}) diff --git a/tests/workspace-inspector.test.ts b/tests/workspace-inspector.test.ts deleted file mode 100644 index 1a68a01b..00000000 --- a/tests/workspace-inspector.test.ts +++ /dev/null @@ -1,112 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { - fileContains, - fileExists, - InMemoryWorkspaceInspector, - rowCount, - rowWhere, - runAssertions, - type WorkspaceSnapshot, -} from '../src/workspace-inspector' - -function snap(partial: Partial = {}): WorkspaceSnapshot { - return { files: {}, rows: {}, kv: {}, ...partial } -} - -describe('InMemoryWorkspaceInspector', () => { - it('returns the snapshot set for a scope', async () => { - const insp = new InMemoryWorkspaceInspector() - insp.set('workspace_1', snap({ files: { 'vault/a.md': 'hi' } })) - const s = await insp.snapshot({ scopeId: 'workspace_1' }) - expect(s.files).toEqual({ 'vault/a.md': 'hi' }) - }) - - it('returns empty shape for unknown scope — regression: undefined would break downstream assertions', async () => { - const insp = new InMemoryWorkspaceInspector() - const s = await insp.snapshot({ scopeId: 'missing' }) - expect(s).toEqual({ files: {}, rows: {}, kv: {} }) - }) -}) - -describe('fileExists / fileContains', () => { - it('fileExists passes when file present', () => { - const r = fileExists('a.md').check(snap({ files: { 'a.md': 'x' } })) - expect(r.pass).toBe(true) - expect(r.score).toBe(1) - }) - it('fileExists fails when absent', () => { - const r = fileExists('a.md').check(snap()) - expect(r.pass).toBe(false) - expect(r.detail).toContain('a.md') - }) - it('fileContains passes on substring hit', () => { - const r = fileContains('a.md', 'hello').check(snap({ files: { 'a.md': 'well hello there' } })) - expect(r.pass).toBe(true) - }) - it('fileContains fails with a descriptive detail when the substring is missing', () => { - const r = fileContains('a.md', 'foo').check(snap({ files: { 'a.md': 'bar' } })) - expect(r.pass).toBe(false) - expect(r.detail).toMatch(/missing substring/) - }) -}) - -describe('rowCount', () => { - it('passes inside range', () => { - const r = rowCount('deals', 1, 10).check(snap({ rows: { deals: [{}, {}, {}] } })) - expect(r.pass).toBe(true) - expect(r.score).toBe(1) - }) - it('partial credit below min — regression: binary pass/fail loses tuning signal', () => { - const r = rowCount('deals', 10).check(snap({ rows: { deals: [{}, {}, {}] } })) - expect(r.pass).toBe(false) - expect(r.score).toBeCloseTo(0.3, 2) - }) - it('partial credit above max', () => { - const r = rowCount('deals', 1, 10).check(snap({ rows: { deals: new Array(20).fill({}) } })) - expect(r.pass).toBe(false) - expect(r.score).toBe(0.5) - }) - it('treats missing table as 0 rows', () => { - const r = rowCount('missing', 1).check(snap()) - expect(r.pass).toBe(false) - }) -}) - -describe('rowWhere', () => { - it('passes when predicate matches enough rows', () => { - const r = rowWhere<{ status: string }>('deals', (row) => row.status === 'closed', { - min: 2, - }).check( - snap({ rows: { deals: [{ status: 'open' }, { status: 'closed' }, { status: 'closed' }] } }), - ) - expect(r.pass).toBe(true) - }) - it('partial credit when min not met', () => { - const r = rowWhere<{ status: string }>('deals', (row) => row.status === 'closed', { - min: 3, - }).check(snap({ rows: { deals: [{ status: 'closed' }] } })) - expect(r.pass).toBe(false) - expect(r.score).toBeCloseTo(1 / 3, 2) - }) -}) - -describe('runAssertions', () => { - it('aggregates pass as AND and score as mean', () => { - const result = runAssertions( - snap({ files: { 'a.md': 'content' }, rows: { deals: [{}, {}, {}] } }), - [fileExists('a.md'), rowCount('deals', 1, 10)], - ) - expect(result.pass).toBe(true) - expect(result.score).toBe(1) - expect(result.results).toHaveLength(2) - }) - - it("aggregate pass is AND across assertions — regression: one fail doesn't sink the aggregate pass", () => { - const result = runAssertions(snap({ files: { 'a.md': 'content' } }), [ - fileExists('a.md'), - fileExists('missing.md'), - ]) - expect(result.pass).toBe(false) - expect(result.score).toBe(0.5) - }) -})