diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b87575b..42b0b6ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,28 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- --- +## [0.149.0] — 2026-08-18 + +### Removed + +- Twelve modules and their tests: `reviewer`, `analyst/knowledge-capture`, `multi-toolchain-layer`, `dual-agent-bench`, `workspace-inspector`, `golden-matcher`, `ui-finding`, `slo`, `adapters/langchain`, `judge-runner`, `cost-report` and `worker-driver-seed`. + + 0.145.3 tiered the root barrel to consumer-imported symbols and documented front doors. These twelve lost their last export path in that pass and kept their source files. Each is unreachable from all 28 build entry points, so none has shipped in `dist` since 0.145.3, and the only importer each retained was its own test. + + **The public export surface does not move.** A build of 0.148.0 and a build of this release each declare 3,447 export entries, and the two lists are identical. No consumer of any published version can be importing a deleted name, because none of them was reachable to import. + +### Changed + +- Four doc comments named a deleted symbol and now describe the surviving behaviour: `multi-layer-verifier` (twice), `fuzz/types` and `contract/self-improve`, plus the `docs/feature-guide` feature map. `dist` loses 753 bytes, all of it that doc-comment text. + +### Migration + +Nothing to do. No symbol removed here has been importable since 0.145.3. + +One repository still names three of them. `starter-foundry` imports `runAssertions`, `WorkspaceAssertion` and `WorkspaceSnapshot` from this package and pins it to exactly `0.135.1`, where those symbols still exist, so it builds today and this release does not change that. The constraint it already carries is that it cannot move past 0.145.3 without porting them, and that predates this release by four versions. The workspace-assertion helpers were a thin projection over a snapshot it already builds itself. + +--- + ## [0.148.0] — 2026-08-18 ### Added diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 5fc262d1..22e0812f 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-eval-rpc" -version = "0.148.0" +version = "0.149.0" description = "Python RPC client, official optimizer bridge, and DSPy metric adapter for @tangle-network/agent-eval." readme = "README.md" requires-python = ">=3.10" diff --git a/clients/python/src/agent_eval_rpc/__init__.py b/clients/python/src/agent_eval_rpc/__init__.py index a65f377c..0c6dbbe5 100644 --- a/clients/python/src/agent_eval_rpc/__init__.py +++ b/clients/python/src/agent_eval_rpc/__init__.py @@ -53,7 +53,7 @@ try: __version__ = version("agent-eval-rpc") except PackageNotFoundError: - __version__ = "0.148.0" + __version__ = "0.149.0" __all__ = [ "Client", diff --git a/clients/python/uv.lock b/clients/python/uv.lock index d47028fd..4a1ff011 100644 --- a/clients/python/uv.lock +++ b/clients/python/uv.lock @@ -34,7 +34,7 @@ conflicts = [[ [[package]] name = "agent-eval-rpc" -version = "0.148.0" +version = "0.149.0" source = { editable = "." } dependencies = [ { name = "filelock" }, diff --git a/docs/feature-guide.md b/docs/feature-guide.md index bfd5c5b8..245ea0d7 100644 --- a/docs/feature-guide.md +++ b/docs/feature-guide.md @@ -152,7 +152,7 @@ Store as `FeedbackTrajectory`, then derive: | Area | Key exports | Best for | Notes | | --- | --- | --- | --- | | Judging | `llmJudge`, semantic judges, anti-slop, wire rubrics | Content, voice, semantic quality | Pair with objective checks when possible. | -| Verification | `MultiLayerVerifier`, `JudgeRunner`, sandbox harness | Code and multi-step gates | Do not let semantic judges override failed builds. | +| Verification | `MultiLayerVerifier`, sandbox harness | Code and multi-step gates | Do not let semantic judges override failed builds. | | Control | `runAgentControlLoop`, `objectiveEval`, `subjectiveEval` | Long-running agent tasks | Supports budgets, cost, stop policies, trace spans. | | Propose/review | `runProposeReview` | Iterative artifact repair | Good for code, docs, plans, briefs. | | Feedback data | `FeedbackTrajectory`, stores, converters | Human/environment labels | Domain adapters live in downstream repos. | diff --git a/package.json b/package.json index 5406d405..caf98014 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-eval", - "version": "0.148.0", + "version": "0.149.0", "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.", "homepage": "https://github.com/tangle-network/agent-eval#readme", "repository": { diff --git a/src/adapters/langchain.ts b/src/adapters/langchain.ts deleted file mode 100644 index ede22388..00000000 --- a/src/adapters/langchain.ts +++ /dev/null @@ -1,125 +0,0 @@ -/** - * # `@tangle-network/agent-eval/adapters/langchain` — wrap any LangChain - * Runnable as a `Dispatch` (or `JudgeConfig`). - * - * **Why structural, not pinned**: we don't depend on `@langchain/core` at - * install time. The adapter accepts anything with the canonical LangChain - * Runnable shape (`invoke(input, config?)`), so it works with their - * `Runnable`, `RunnableSequence`, `RunnableMap`, `RunnablePassthrough`, - * and any custom Runnable-shaped object. No version pin, no peer dep, - * no bundle-bloat risk. - * - * **Why this exists**: the most-asked question from foreign agent - * builders is "I'm already on LangChain — how do I plug in?". The answer - * is one function. Wrap your existing Runnable, pass the Dispatch into - * `runEval` / `runImprovementLoop`, ship. - */ - -import type { Dispatch, JudgeConfig, JudgeScore, Scenario } from '../contract' - -// ── Minimal structural type ────────────────────────────────────────── -// -// Whatever has `invoke(input, config?)` qualifies. We accept any -// config shape (LangChain's RunnableConfig has many optional fields) -// — the only thing we need is the AbortSignal seam, which LangChain's -// RunnableConfig already supports as `signal?: AbortSignal`. - -export interface RunnableLike { - invoke(input: TInput, config?: { signal?: AbortSignal; [key: string]: unknown }): Promise -} - -// ── Dispatch wrapper ──────────────────────────────────────────────── - -export interface LangchainDispatchOptions { - /** The Runnable (or RunnableSequence, or anything `.invoke`able). */ - runnable: RunnableLike - /** - * Optional config merged into every `invoke` call — tags, metadata, - * callbacks, runName. The substrate's per-cell `AbortSignal` is - * always merged in last (and so wins). - */ - config?: Record -} - -/** - * Wrap a LangChain Runnable as a `Dispatch`. The Runnable's input must - * accept the scenario (typically you'll shape it via - * `RunnableMap`/`RunnableLambda` upstream); its output is the artifact - * the engine + judges see. - * - * @example - * const chain = prompt.pipe(model).pipe(parser) - * const dispatch = langchainDispatch({ runnable: chain }) - * await runEval({ scenarios, dispatch, judges: [...], storage, runDir }) - */ -export function langchainDispatch( - opts: LangchainDispatchOptions, -): Dispatch { - return async (scenario, ctx) => { - return opts.runnable.invoke(scenario, { - ...opts.config, - signal: ctx.signal, - }) - } -} - -// ── Judge wrapper ─────────────────────────────────────────────────── - -export interface LangchainJudgeOptions { - /** Judge name; appears in `CampaignResult.aggregates.byJudge`. */ - name: string - /** - * Dimensions the judge scores. Used both for the judge's own prompt - * (if it reads them) and for the aggregator's `byJudge` rollup. - */ - dimensions: { key: string; description: string }[] - /** - * A Runnable that takes `{ artifact, scenario }` and returns a - * partial `JudgeScore` — the dimensions map at minimum. `composite` - * is computed by averaging `dimensions` when the Runnable doesn't - * provide it; `notes` defaults to an empty string. - */ - runnable: RunnableLike<{ artifact: TArtifact; scenario: TScenario }, Partial> - appliesTo?: (scenario: TScenario) => boolean -} - -/** - * Wrap a LangChain Runnable as a `JudgeConfig`. The Runnable can be any - * structured-output chain (e.g. `prompt.pipe(model).pipe(StructuredOutputParser)`) - * that returns a `Partial`. - * - * The substrate's invariant — throw on judge failure, never silently - * fold errors into a zero — is preserved: any error from the Runnable - * propagates and the substrate records a failed cell. - * - * @example - * const scorePrompt = ChatPromptTemplate.fromTemplate(`...`) - * const judgeChain = scorePrompt.pipe(judgeModel).pipe(jsonParser) - * const judge = langchainJudge({ - * name: 'marketing-quality', - * dimensions: [{ key: 'hook_strength', description: '...' }, ...], - * runnable: judgeChain, - * }) - */ -export function langchainJudge( - opts: LangchainJudgeOptions, -): JudgeConfig { - return { - name: opts.name, - dimensions: opts.dimensions, - appliesTo: opts.appliesTo, - async score({ artifact, scenario, signal }) { - const result = await opts.runnable.invoke({ artifact, scenario }, { signal }) - const dims = (result.dimensions ?? {}) as Record - const dimValues = Object.values(dims) - const composite = - result.composite ?? - (dimValues.length > 0 ? dimValues.reduce((a, b) => a + b, 0) / dimValues.length : 0) - return { - dimensions: dims, - composite, - notes: result.notes ?? '', - } - }, - } -} diff --git a/src/analyst/benchmark-implementation.ts b/src/analyst/benchmark-implementation.ts index 83282e6b..c8e5dac3 100644 --- a/src/analyst/benchmark-implementation.ts +++ b/src/analyst/benchmark-implementation.ts @@ -10,7 +10,7 @@ export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([ ]) export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = - '5c7e5561a2ecaa48aaffc52d8a44f582642ed434a92839ae4741ad5556fd9789' + '3e17b679d34215f636931e6d39cb5215d878cdaa501c38fd6cef513ce844677f' /** The published benchmark evidence was produced at this package version, by * the retired one-shot direct runner, before trace analysts moved to the diff --git a/src/analyst/knowledge-capture.test.ts b/src/analyst/knowledge-capture.test.ts deleted file mode 100644 index 65bfa633..00000000 --- a/src/analyst/knowledge-capture.test.ts +++ /dev/null @@ -1,281 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { - captureKnowledgeCandidates, - GROUNDING_EVIDENCE_KINDS, - KNOWLEDGE_CANDIDATE_TAG, -} from './knowledge-capture' -import { type AnalystFinding, type EvidenceRef, makeFinding } from './types' - -/** - * The observed session this module was built for: the agent filed a public - * issue naming the wrong root cause, then discovered mid-run that the - * platform grants a signup credit. The fact was learned and then lost. - */ -function signupCreditFinding( - overrides: Partial> = {}, -): AnalystFinding { - // Overrides are folded in BEFORE makeFinding so finding_id derives from the - // claim and subject the test actually uses; computing it first would give - // two different findings the same identity. - return makeFinding({ - analyst_id: 'knowledge-gap', - severity: 'high', - area: 'knowledge-gap', - claim: - 'The platform grants a $2 signup credit to new accounts; the agent did not know this and attributed the balance to a billing bug.', - rationale: - 'The agent filed issue #412 blaming a double-charge before reading the credit line in the account response.', - recommended_action: - 'Record the signup credit amount and its eligibility window on the billing wiki page.', - evidence_refs: [ - { - kind: 'span', - uri: 'otlp:span/2f9c1a4b', - excerpt: 'account.credits[0] = { source: "signup", amount_usd: 2 }', - }, - { kind: 'span', uri: 'otlp:span/7d31e880', excerpt: 'Actually, that $2 is a signup credit.' }, - ], - confidence: 0.9, - subject: 'agent-knowledge:wiki:platform-billing-signup-credit', - ...overrides, - }) -} - -describe('captureKnowledgeCandidates — routing', () => { - it('turns an agent-knowledge:wiki finding into a candidate page', () => { - const { candidates, dropped } = captureKnowledgeCandidates([signupCreditFinding()]) - - expect(dropped).toEqual([]) - expect(candidates).toHaveLength(1) - const candidate = candidates[0]! - expect(candidate.slug).toBe('platform-billing-signup-credit') - expect(candidate.heading).toBeUndefined() - expect(candidate.page.id).toBe('platform-billing-signup-credit') - expect(candidate.page.path).toBe('knowledge/platform-billing-signup-credit.md') - expect(candidate.page.title).toBe('Platform Billing Signup Credit') - expect(candidate.page.tags).toEqual([KNOWLEDGE_CANDIDATE_TAG]) - expect(candidate.analystId).toBe('knowledge-gap') - expect(candidate.severity).toBe('high') - expect(candidate.confidence).toBe(0.9) - expect(candidate.sourceFindingId).toBe(signupCreditFinding().finding_id) - }) - - it('ignores loci that belong to other runtime layers', () => { - const otherLayers = [ - 'system-prompt:account-access', - 'tool-doc:billing-api:credits', - 'memory:last-invoice', - 'skill:billing-triage', - 'websearch:outdated:stripe-fees', - 'agent-knowledge:claim:signup-credit', - 'agent-knowledge:raw:billing-docs-2026', - 'agent-knowledge:stale:platform-billing-signup-credit', - ] - const findings = otherLayers.map((subject) => signupCreditFinding({ subject })) - - const { candidates, dropped } = captureKnowledgeCandidates(findings) - - expect(candidates).toEqual([]) - expect(dropped.map((d) => d.reason)).toEqual(otherLayers.map(() => 'non-wiki-locus')) - expect(dropped.map((d) => d.subject)).toEqual(otherLayers) - }) - - it('drops a finding with no subject and one whose subject is not in the grammar', () => { - const noSubject = signupCreditFinding({ subject: undefined }) - const prose = signupCreditFinding({ subject: 'please add this to the wiki' }) - - const { candidates, dropped } = captureKnowledgeCandidates([noSubject, prose]) - - expect(candidates).toEqual([]) - expect(dropped).toEqual([ - { findingId: noSubject.finding_id, reason: 'no-subject' }, - { - findingId: prose.finding_id, - subject: 'please add this to the wiki', - reason: 'unparsed-subject', - }, - ]) - }) - - it('accounts for every input finding exactly once', () => { - const findings = [ - signupCreditFinding({ claim: 'kept: the signup credit is $2.' }), - signupCreditFinding({ claim: 'other layer', subject: 'system-prompt:account-access' }), - signupCreditFinding({ claim: 'ungrounded', evidence_refs: [] }), - signupCreditFinding({ claim: 'no locus', subject: undefined }), - ] - - const { candidates, dropped } = captureKnowledgeCandidates(findings) - - expect(candidates.length + dropped.length).toBe(findings.length) - const seen = [...candidates.map((c) => c.sourceFindingId), ...dropped.map((d) => d.findingId)] - expect(new Set(seen)).toEqual(new Set(findings.map((f) => f.finding_id))) - }) -}) - -describe('captureKnowledgeCandidates — grounding', () => { - it('drops a finding with no evidence refs rather than emitting empty anchors', () => { - const ungrounded = signupCreditFinding({ evidence_refs: [] }) - - const { candidates, dropped } = captureKnowledgeCandidates([ungrounded]) - - expect(candidates).toEqual([]) - expect(dropped).toEqual([ - { - findingId: ungrounded.finding_id, - subject: 'agent-knowledge:wiki:platform-billing-signup-credit', - reason: 'no-grounding-evidence', - }, - ]) - }) - - it('drops a finding anchored only to another finding or a metric', () => { - const refs: EvidenceRef[] = [ - { kind: 'finding', uri: 'f_1234567890abcdef1234' }, - { kind: 'metric', uri: 'metric:clarifying_questions' }, - ] - for (const kind of refs.map((r) => r.kind)) { - expect(GROUNDING_EVIDENCE_KINDS).not.toContain(kind) - } - - const { candidates, dropped } = captureKnowledgeCandidates([ - signupCreditFinding({ evidence_refs: refs }), - ]) - - expect(candidates).toEqual([]) - expect(dropped[0]!.reason).toBe('no-grounding-evidence') - }) - - it('drops a wiki-routed finding whose claim is blank', () => { - const { candidates, dropped } = captureKnowledgeCandidates([ - signupCreditFinding({ claim: ' ' }), - ]) - - expect(candidates).toEqual([]) - expect(dropped[0]!.reason).toBe('empty-claim') - }) - - it('anchors match the finding evidence URIs exactly, in order', () => { - const finding = signupCreditFinding() - const { candidates } = captureKnowledgeCandidates([finding]) - const candidate = candidates[0]! - - const expectedUris = finding.evidence_refs.map((r) => r.uri) - expect(candidate.anchors.map((a) => a.uri)).toEqual(expectedUris) - expect(candidate.page.sourceIds).toEqual(expectedUris) - expect(candidate.page.frontmatter.sources).toEqual(expectedUris) - expect(candidate.anchors).toEqual( - finding.evidence_refs.map((r) => ({ kind: r.kind, uri: r.uri, excerpt: r.excerpt })), - ) - }) - - it('keeps grounded refs while discarding blank, duplicate, and non-observed ones', () => { - const finding = signupCreditFinding({ - evidence_refs: [ - { kind: 'span', uri: 'otlp:span/2f9c1a4b' }, - { kind: 'span', uri: ' ' }, - { kind: 'span', uri: 'otlp:span/2f9c1a4b' }, - { kind: 'finding', uri: 'f_1234567890abcdef1234' }, - { kind: 'artifact', uri: 'artifact:issue-412.json' }, - ], - }) - - const { candidates } = captureKnowledgeCandidates([finding]) - - expect(candidates[0]!.anchors).toEqual([ - { kind: 'span', uri: 'otlp:span/2f9c1a4b' }, - { kind: 'artifact', uri: 'artifact:issue-412.json' }, - ]) - }) -}) - -describe('captureKnowledgeCandidates — page content', () => { - it('carries the claim, recommended action, and evidence excerpts, and nothing else', () => { - const finding = signupCreditFinding() - const { text } = captureKnowledgeCandidates([finding]).candidates[0]!.page - - expect(text).toContain(finding.claim) - expect(text).toContain(finding.rationale) - expect(text).toContain(finding.recommended_action) - for (const ref of finding.evidence_refs) { - expect(text).toContain(ref.uri) - expect(text).toContain(ref.excerpt) - } - // The frontmatter block belongs to the `frontmatter` field, matching how - // the knowledge base splits a page it loads from disk. - expect(text.startsWith('---')).toBe(false) - expect(text).toContain('# Platform Billing Signup Credit') - }) - - it('omits sections the finding did not supply', () => { - const { text } = captureKnowledgeCandidates([ - signupCreditFinding({ rationale: undefined, recommended_action: undefined }), - ]).candidates[0]!.page - - expect(text).not.toContain('Rationale') - expect(text).not.toContain('Recommended action') - expect(text).toContain('Sources') - }) - - it('never fabricates outgoing links', () => { - const { page } = captureKnowledgeCandidates([signupCreditFinding()]).candidates[0]! - - expect(page.outLinks).toEqual([]) - expect(page.text).not.toMatch(/\[\[/) - }) - - it('mirrors sourceIds and tags into frontmatter so a disk round-trip keeps them', () => { - const finding = signupCreditFinding() - const { page } = captureKnowledgeCandidates([finding]).candidates[0]! - - expect(page.frontmatter.sources).toEqual(page.sourceIds) - expect(page.frontmatter.tags).toEqual(page.tags) - expect(page.frontmatter.id).toBe(page.id) - expect(page.frontmatter.title).toBe(page.title) - expect(page.frontmatter.status).toBe('candidate') - expect(page.frontmatter.drafted_from_finding).toBe(finding.finding_id) - expect(page.frontmatter.analyst_id).toBe('knowledge-gap') - expect(page.frontmatter.severity).toBe('high') - expect(page.frontmatter.confidence).toBe(0.9) - expect(page.frontmatter.derived_from_judge).toBe(false) - }) - - it('flags a finding lifted from a judge verdict rather than read off a trace', () => { - const { page } = captureKnowledgeCandidates([signupCreditFinding({ derived_from_judge: true })]) - .candidates[0]! - - expect(page.frontmatter.derived_from_judge).toBe(true) - }) - - it('records a heading locus as a section body under the same page', () => { - const { candidates } = captureKnowledgeCandidates([ - signupCreditFinding({ - subject: 'agent-knowledge:wiki:platform-billing-signup-credit#eligibility', - }), - ]) - const candidate = candidates[0]! - - expect(candidate.heading).toBe('eligibility') - expect(candidate.slug).toBe('platform-billing-signup-credit') - expect(candidate.page.path).toBe('knowledge/platform-billing-signup-credit.md') - expect(candidate.page.text.startsWith('## Eligibility\n')).toBe(true) - expect(candidate.page.text).not.toContain('# Platform Billing Signup Credit') - }) - - it('preserves two findings that route to the same page instead of merging them', () => { - const first = signupCreditFinding() - const second = signupCreditFinding({ - claim: 'The signup credit expires 30 days after account creation.', - evidence_refs: [{ kind: 'span', uri: 'otlp:span/aa11bb22' }], - }) - - const { candidates } = captureKnowledgeCandidates([first, second]) - - expect(candidates).toHaveLength(2) - expect(candidates.map((c) => c.page.path)).toEqual([ - 'knowledge/platform-billing-signup-credit.md', - 'knowledge/platform-billing-signup-credit.md', - ]) - expect(candidates[0]!.sourceFindingId).not.toBe(candidates[1]!.sourceFindingId) - }) -}) diff --git a/src/analyst/knowledge-capture.ts b/src/analyst/knowledge-capture.ts deleted file mode 100644 index 774f6c5f..00000000 --- a/src/analyst/knowledge-capture.ts +++ /dev/null @@ -1,300 +0,0 @@ -/** - * Knowledge capture — knowledge-gap findings become CANDIDATE wiki pages. - * - * The gap this closes: the knowledge-gap analyst already detects facts the - * agent had to discover at run time. Observed case — an agent learned - * mid-session that the platform grants a signup credit, but only after it - * had filed a public issue naming the wrong root cause. It corrected - * itself, the session ended, and the fact died with it; the next session - * starts ignorant and can repeat the mistake. A detector existed - * (knowledge-gap) and a store existed (`@tangle-network/agent-knowledge`), - * and nothing joined them. This module is that join. - * - * These are CANDIDATES, never writes. The transform is pure: no file I/O, - * no knowledge-base handle, no network. That boundary is the point, not an - * omission. An analyst finding is a MODEL CLAIM about a trace, and writing - * model claims straight into a curated knowledge base is precisely the - * corruption the knowledge-poisoning analyst exists to detect — a KB that - * ingests its own analyst's guesses will later cite them as sources and - * launder a guess into a fact. So the output is a reviewable value: a human - * or a gated activation path decides what gets written. - * - * Routing is by subject prefix. Knowledge-gap findings carry a typed locus - * (`agent-knowledge:wiki:platform-billing-signup-credit`, - * `system-prompt:account-access`, `tool-doc:gh:auth`, ...), and the prefix - * says which layer owns the fix. Only `agent-knowledge:wiki:*` is a page; - * every other locus belongs to the improvement adapter or another layer and - * is dropped here with a stated reason rather than silently skipped. - * - * Caller responsibility this transform deliberately does NOT take on: - * `knowledge-poisoning` findings also emit `agent-knowledge:wiki:*` loci, - * but their claim describes a page that is WRONG — not knowledge to add. - * Every candidate carries `analystId` so a caller feeding mixed findings - * can filter; encoding that policy here would hide it from the operator. - * - * Structural contract: `KnowledgeCandidatePage` mirrors `KnowledgePage` - * from `@tangle-network/agent-knowledge` field for field, so a reviewed - * candidate can be handed to that package's page store unchanged. The shape - * is restated rather than imported because agent-eval does not depend on - * agent-knowledge — the dependency runs the other way (agent-knowledge - * imports `AnalystFinding` from agent-eval), and importing back would close - * a package cycle. - */ - -import { parseFindingSubject } from './finding-subject' -import type { AnalystFinding, AnalystSeverity, EvidenceRef } from './types' - -/** - * Evidence kinds admissible as page anchors: references that point at a - * recorded observation a reviewer can go re-read. - * - * `finding` and `metric` refs are excluded on purpose. A `finding` ref - * anchors one model claim to another model claim, so the resulting page - * would be grounded in nothing observable no matter how deep the chain is - * followed. A `metric` ref names a scalar reading with no retrievable text, - * so it cannot support the claim it would be cited for. - */ -export const GROUNDING_EVIDENCE_KINDS: ReadonlyArray = [ - 'span', - 'event', - 'artifact', -] - -/** - * Tag stamped on every candidate page. Survives into the knowledge base if - * a candidate is ever activated, so an unreviewed model claim that reached - * the store is still greppable after the fact. - */ -export const KNOWLEDGE_CANDIDATE_TAG = 'analyst-candidate' - -/** One admitted evidence reference, carried verbatim from the finding. */ -export interface KnowledgeCandidateAnchor { - kind: EvidenceRef['kind'] - uri: string - excerpt?: string -} - -/** - * A page value shaped exactly like agent-knowledge's `KnowledgePage`. - * - * Arrays are mutable, not `readonly`, because a `readonly string[]` is not - * assignable to that package's `string[]` fields — the whole point of the - * shape is that a reviewed candidate passes through without a translation - * step that could drop provenance. - * - * Invariant that the knowledge base's own loader depends on: `text` is the - * body WITHOUT the frontmatter block, and `sourceIds` / `tags` mirror - * `frontmatter.sources` / `frontmatter.tags`. That loader reconstructs the - * page by parsing the frontmatter off the markdown file, so a candidate - * whose fields disagree with its frontmatter would lose its source anchors - * the first time it round-tripped through disk. - */ -export interface KnowledgeCandidatePage { - id: string - path: string - title: string - text: string - frontmatter: Record - sourceIds: string[] - tags: string[] - outLinks: string[] -} - -/** A candidate page plus the provenance a reviewer needs to judge it. */ -export interface KnowledgeCandidate { - page: KnowledgeCandidatePage - /** Page slug from the finding's locus — the page identity in the KB. */ - slug: string - /** - * Section under `slug` when the locus named one. Present means `page.text` - * is a SECTION body to append under an existing page, not a whole file. - */ - heading?: string - sourceFindingId: string - analystId: string - severity: AnalystSeverity - confidence: number - anchors: KnowledgeCandidateAnchor[] -} - -export type KnowledgeCaptureDropReason = - /** Descriptive finding with no locus — nothing to route. */ - | 'no-subject' - /** Subject present but outside the finding-subject grammar. */ - | 'unparsed-subject' - /** A valid locus owned by another layer (system prompt, tool doc, ...). */ - | 'non-wiki-locus' - /** No evidence reference points at a recorded observation. */ - | 'no-grounding-evidence' - /** Claim text is empty, so the page would have no content. */ - | 'empty-claim' - -/** Why one finding produced no candidate. Drops are reported, never silent. */ -export interface KnowledgeCaptureDrop { - findingId: string - subject?: string - reason: KnowledgeCaptureDropReason -} - -export interface KnowledgeCaptureResult { - candidates: KnowledgeCandidate[] - /** - * Every input finding that produced no candidate, with its reason. An - * invisible skip is how routing tables rot: the caller can assert that - * `candidates.length + dropped.length === findings.length` and see which - * loci its analysts actually emit. - */ - dropped: KnowledgeCaptureDrop[] -} - -/** - * Turn findings into candidate wiki pages. - * - * Pure and total: every input finding appears in exactly one of - * `candidates` or `dropped`. Ordering follows the input. - * - * Several findings may route to the same slug, producing several candidates - * that share `page.id` and `page.path`. That collision is preserved rather - * than resolved — merging two claims about one page is an editorial - * decision, and picking a winner here would discard evidence the reviewer - * never saw. - * - * Confidence is recorded, never thresholded. Discarding a low-confidence - * finding is a review policy; applying it inside the transform would make - * the policy invisible to the operator who has to defend the KB's contents. - */ -export function captureKnowledgeCandidates( - findings: ReadonlyArray, -): KnowledgeCaptureResult { - const candidates: KnowledgeCandidate[] = [] - const dropped: KnowledgeCaptureDrop[] = [] - - for (const finding of findings) { - const outcome = captureOne(finding) - if ('reason' in outcome) dropped.push(outcome) - else candidates.push(outcome) - } - - return { candidates, dropped } -} - -function captureOne(finding: AnalystFinding): KnowledgeCandidate | KnowledgeCaptureDrop { - const drop = (reason: KnowledgeCaptureDropReason): KnowledgeCaptureDrop => ({ - findingId: finding.finding_id, - ...(finding.subject === undefined ? {} : { subject: finding.subject }), - reason, - }) - - if (finding.subject === undefined || finding.subject.trim().length === 0) { - return drop('no-subject') - } - - const subject = parseFindingSubject(finding.subject) - if (subject === null) return drop('unparsed-subject') - if (subject.kind !== 'knowledge.wiki') return drop('non-wiki-locus') - - const anchors = groundingAnchors(finding.evidence_refs) - if (anchors.length === 0) return drop('no-grounding-evidence') - - const claim = finding.claim.trim() - if (claim.length === 0) return drop('empty-claim') - - const title = humanize(subject.slug) - const frontmatter: Record = { - id: subject.slug, - title, - status: 'candidate', - drafted_from_finding: finding.finding_id, - analyst_id: finding.analyst_id, - severity: finding.severity, - confidence: finding.confidence, - // Distinguishes a claim read off a trace from one lifted out of another - // model's verdict; the second is a weaker basis for a curated page. - derived_from_judge: finding.derived_from_judge === true, - sources: anchors.map((a) => a.uri), - tags: [KNOWLEDGE_CANDIDATE_TAG], - } - - return { - page: { - id: subject.slug, - path: `knowledge/${subject.slug}.md`, - title, - text: renderBody({ finding, claim, title, heading: subject.heading, anchors }), - frontmatter, - sourceIds: anchors.map((a) => a.uri), - tags: [KNOWLEDGE_CANDIDATE_TAG], - // Never inferred. A link to a page that may not exist is fabricated - // structure, and the KB's linter reports it as a broken link. - outLinks: [], - }, - slug: subject.slug, - ...(subject.heading === undefined ? {} : { heading: subject.heading }), - sourceFindingId: finding.finding_id, - analystId: finding.analyst_id, - severity: finding.severity, - confidence: finding.confidence, - anchors, - } -} - -/** - * Admit references that anchor to observation, dropping blank URIs. Dedupes - * on URI so a finding that cited one span twice does not inflate the page's - * apparent source count. - */ -function groundingAnchors(refs: ReadonlyArray): KnowledgeCandidateAnchor[] { - const seen = new Set() - const anchors: KnowledgeCandidateAnchor[] = [] - for (const ref of refs) { - if (!GROUNDING_EVIDENCE_KINDS.includes(ref.kind)) continue - const uri = ref.uri.trim() - if (uri.length === 0 || seen.has(uri)) continue - seen.add(uri) - const excerpt = ref.excerpt?.trim() - anchors.push({ kind: ref.kind, uri, ...(excerpt ? { excerpt } : {}) }) - } - return anchors -} - -/** - * Render the page body. Every line originates in the finding: the claim, its - * rationale, its recommended action, and its evidence URIs with excerpts - * quoted verbatim. The only derived text is the title, a mechanical - * de-kebabing of the slug the analyst itself chose. - * - * The frontmatter block is NOT rendered here — it lives in the `frontmatter` - * field, matching how the knowledge base splits a page it loads from disk. - */ -function renderBody(input: { - finding: AnalystFinding - claim: string - title: string - heading?: string - anchors: ReadonlyArray -}): string { - const { finding, claim, title, heading, anchors } = input - const rationale = finding.rationale?.trim() - const action = finding.recommended_action?.trim() - - const lines = [heading ? `## ${humanize(heading)}` : `# ${title}`, '', claim, ''] - if (rationale) lines.push(heading ? '### Rationale' : '## Rationale', '', rationale, '') - if (action) - lines.push(heading ? '### Recommended action' : '## Recommended action', '', action, '') - - lines.push(heading ? '### Sources' : '## Sources', '') - for (const anchor of anchors) { - lines.push(`- \`${anchor.uri}\``) - // Blockquote per line rather than an inline quoted string: excerpts can - // contain quotes and newlines, and the excerpt must survive unedited. - if (anchor.excerpt) lines.push(...anchor.excerpt.split('\n').map((l) => ` > ${l}`)) - } - - return `${lines.join('\n')}\n` -} - -function humanize(slug: string): string { - const words = slug.split('-').filter((w) => w.length > 0) - if (words.length === 0) return slug - return words.map((w) => w[0]!.toUpperCase() + w.slice(1)).join(' ') -} diff --git a/src/contract/self-improve.ts b/src/contract/self-improve.ts index d3a9497f..e0033d8c 100644 --- a/src/contract/self-improve.ts +++ b/src/contract/self-improve.ts @@ -141,8 +141,7 @@ export interface SelfImproveOptions { * these unless `budget.holdoutScenarios` is set explicitly. */ scenarios: TScenario[] - /** Judge that scores artifacts. Bring your own; use `langchainJudge` - * from `/adapters/langchain` for a Runnable-shaped one. */ + /** Judge that scores artifacts. Bring your own, or wrap `llmJudge`. */ judge: JudgeConfig /** Starting surface — system prompt, JSON config, anything `MutableSurface` diff --git a/src/cost-report.test.ts b/src/cost-report.test.ts deleted file mode 100644 index 9f4ccfac..00000000 --- a/src/cost-report.test.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { describe, expect, it } from 'vitest' -import type { CostReceipt, CostReceiptInput } from './cost-ledger' -import { CostLedger, costForUsage } from './cost-ledger' -import { attachCostToReport, costReport } from './cost-report' -import { ValidationError } from './errors' - -let receiptSequence = 0 - -function receipt(channel: 'agent' | 'judge', input: CostReceiptInput): CostReceipt { - const estimated = costForUsage(input.model, input) - return { - status: 'settled', - callId: `fixture-${receiptSequence++}`, - ...input, - channel, - phase: 'test', - actor: 'fixture', - costUsd: input.actualCostUsd ?? estimated.costUsd, - costUnknown: input.actualCostUsd === undefined && estimated.costUnknown, - timestamp: 1, - } -} - -function buildLedger(): CostLedger { - return new CostLedger({ - receipts: [ - receipt('agent', { model: 'gpt-4o', inputTokens: 1000, outputTokens: 1000 }), - receipt('judge', { model: 'gpt-4o', inputTokens: 2000, outputTokens: 0 }), - receipt('judge', { model: 'made-up-zzz', inputTokens: 1000, outputTokens: 1000 }), - ], - }) -} - -describe('costReport', () => { - it('projects per-channel, total, and per-model rollups from the ledger', () => { - const report = costReport(buildLedger()) - - expect(report.perChannel.map((c) => c.channel)).toEqual(['agent', 'judge']) - const judge = report.perChannel.find((c) => c.channel === 'judge') - expect(judge?.calls).toBe(2) - expect(judge?.unpricedCalls).toBe(1) - - expect(report.total.usd).toBeCloseTo(0.0125 + 0.005, 6) - expect(report.total.unknownEntries).toBe(1) - - expect(report.perModel.map((m) => m.model)).toEqual(['gpt-4o', 'made-up-zzz']) - expect(report.perModel[0]).toEqual({ - model: 'gpt-4o', - usd: 0.0175, - entries: 2, - unpriced: false, - }) - }) - - it('flags an unpriced model unpriced:true — its $0 is never a measured zero', () => { - const report = costReport(buildLedger()) - const unpriced = report.perModel.find((m) => m.model === 'made-up-zzz') - expect(unpriced).toEqual({ model: 'made-up-zzz', usd: 0, entries: 1, unpriced: true }) - }) - - it('an actualCostUsd override clears unpriced — observed dollars are real', () => { - const ledger = new CostLedger({ - receipts: [ - receipt('agent', { - model: 'made-up-zzz', - inputTokens: 100, - outputTokens: 100, - actualCostUsd: 0.42, - }), - ], - }) - const report = costReport(ledger) - expect(report.perModel[0]).toEqual({ - model: 'made-up-zzz', - usd: 0.42, - entries: 1, - unpriced: false, - }) - expect(report.total.unknownEntries).toBe(0) - }) - - it('an empty ledger projects to zeros, never throws', () => { - const report = costReport(new CostLedger()) - expect(report).toEqual({ - perChannel: [], - total: { usd: 0, unknownEntries: 0 }, - perModel: [], - }) - }) -}) - -describe('attachCostToReport', () => { - it('stamps the projection under cost and preserves the report fields', () => { - const stamped = attachCostToReport({ verdict: 'ship', lift: 0.04 }, buildLedger()) - expect(stamped.verdict).toBe('ship') - expect(stamped.lift).toBe(0.04) - expect(stamped.cost.total.unknownEntries).toBe(1) - expect(stamped.cost.perModel).toHaveLength(2) - }) - - it('refuses to overwrite an existing cost stamp', () => { - expect(() => attachCostToReport({ cost: 'already-stamped' }, new CostLedger())).toThrow( - ValidationError, - ) - }) -}) diff --git a/src/cost-report.ts b/src/cost-report.ts deleted file mode 100644 index cb372646..00000000 --- a/src/cost-report.ts +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Program cost report — a thin projection over `CostLedger.summary()` that - * adds the per-model rollup the summary lacks, plus `attachCostToReport`, the - * one way every artifact (capsules, campaign results, diagnose reports) gets - * its cost stamp. - * - * Honesty contract carried through from the ledger: `total.unknownEntries` - * and `perModel[].unpriced` surface the costUnknown axis — a $0 from an - * unpriced model is a lower bound, never a measured zero. - */ - -import type { ChannelRollup, CostLedgerHandle } from './cost-ledger' -import { ValidationError } from './errors' - -export interface ModelCostRollup { - model: string - usd: number - entries: number - /** ≥1 entry for this model was costUnknown — `usd` is a lower bound. An - * `actualCostUsd` override clears the flag for that entry (the dollars are - * observed, even when the model has no pricing). */ - unpriced: boolean -} - -export interface CostReport { - /** Per-channel breakdown — `CostLedgerSummary.byChannel` verbatim. */ - perChannel: ChannelRollup[] - total: { - usd: number - /** Entries whose cost was unknown — non-zero means `usd` is a lower bound. */ - unknownEntries: number - } - /** Per-model spend, sorted by model id. */ - perModel: ModelCostRollup[] -} - -/** Project a ledger into the program cost report. Pure — no I/O, no clock. */ -export function costReport(ledger: CostLedgerHandle): CostReport { - const summary = ledger.summary() - const perModel = new Map() - for (const entry of ledger.list()) { - const roll = perModel.get(entry.model) ?? { - model: entry.model, - usd: 0, - entries: 0, - unpriced: false, - } - roll.usd += entry.costUsd - roll.entries += 1 - if (entry.costUnknown) roll.unpriced = true - perModel.set(entry.model, roll) - } - return { - perChannel: summary.byChannel, - total: { - usd: summary.totalCostUsd, - unknownEntries: summary.byChannel.reduce((sum, c) => sum + c.unpricedCalls, 0), - }, - perModel: [...perModel.values()].sort((a, b) => a.model.localeCompare(b.model)), - } -} - -/** - * Stamp a report-shaped object with its cost projection under the `cost` key. - * Generic so capsules, campaign results, and diagnose reports all stamp the - * same way. Throws when the report already carries a `cost` key — silently - * overwriting an existing stamp would corrupt the artifact's provenance. - */ -export function attachCostToReport( - report: R, - ledger: CostLedgerHandle, -): R & { cost: CostReport } { - if ('cost' in report) { - throw new ValidationError( - "attachCostToReport: report already has a 'cost' key — refusing to overwrite an existing stamp", - ) - } - return { ...report, cost: costReport(ledger) } -} diff --git a/src/dual-agent-bench.ts b/src/dual-agent-bench.ts deleted file mode 100644 index 09025176..00000000 --- a/src/dual-agent-bench.ts +++ /dev/null @@ -1,163 +0,0 @@ -/** - * Dual-agent convergence bench. - * - * Pattern lifted from dual-worker review loops: two agents take turns until - * they converge on a consensus artifact. One proposes, the other critiques; - * the proposer revises; repeat until a score threshold is hit or max rounds. - * - * Generalized so any two "agents" (gateways, local functions, anything with - * `propose` + `critique`) compose in. Returns convergence rounds per - * scenario + whether convergence happened. - */ - -export interface DualAgentScenario { - id: string - initialPrompt: string - /** Optional context the agents can read (e.g. source documents). */ - context?: Record -} - -export interface DualAgentRound { - roundIndex: number - proposal: string - critique: string - convergenceScore: number // 0..1 — how close to convergence -} - -export interface DualAgentScenarioResult { - scenarioId: string - converged: boolean - roundsToConverge: number | null - finalProposal: string - history: DualAgentRound[] - finalScore: number -} - -export interface DualAgentBenchConfig { - scenarios: DualAgentScenario[] - maxRounds?: number - /** Convergence threshold in 0..1 (default 0.85). */ - convergenceThreshold?: number - /** - * Propose an answer given the scenario + the critic's prior critique (if any). - * Returns the proposal string. - */ - propose: (args: { - scenario: DualAgentScenario - roundIndex: number - priorProposal?: string - priorCritique?: string - }) => Promise - /** - * Critique the proposer's current output. Returns a structured critique - * (free text) plus a convergence score: how close the proposal is to - * acceptable. 1.0 = accept, 0.0 = totally off. - */ - critique: (args: { - scenario: DualAgentScenario - roundIndex: number - proposal: string - }) => Promise<{ critique: string; convergenceScore: number }> - /** Optional per-round hook for progress + tracing. */ - onRoundComplete?: (info: { scenarioId: string; round: DualAgentRound }) => void -} - -export interface DualAgentReport { - scenarios: DualAgentScenarioResult[] - aggregate: { - convergenceRate: number // fraction of scenarios that converged within maxRounds - avgRoundsToConverge: number | null // over scenarios that DID converge - avgFinalScore: number - } - config: { - maxRounds: number - convergenceThreshold: number - } -} - -export class DualAgentBench { - async run(config: DualAgentBenchConfig): Promise { - const maxRounds = config.maxRounds ?? 5 - const threshold = config.convergenceThreshold ?? 0.85 - - if (config.scenarios.length === 0) { - throw new Error('DualAgentBench requires at least 1 scenario') - } - - const results: DualAgentScenarioResult[] = [] - - for (const scenario of config.scenarios) { - const history: DualAgentRound[] = [] - let converged = false - let roundsToConverge: number | null = null - let finalProposal = '' - let lastScore = 0 - let priorCritique: string | undefined - - for (let r = 0; r < maxRounds; r++) { - const priorProposal = history[history.length - 1]?.proposal - const proposal = await config.propose({ - scenario, - roundIndex: r, - priorProposal, - priorCritique, - }) - const { critique, convergenceScore } = await config.critique({ - scenario, - roundIndex: r, - proposal, - }) - - if (!Number.isFinite(convergenceScore) || convergenceScore < 0 || convergenceScore > 1) { - throw new Error( - `critique must return convergenceScore in [0,1]; got ${convergenceScore} for scenario ${scenario.id} round ${r}`, - ) - } - - const round: DualAgentRound = { - roundIndex: r, - proposal, - critique, - convergenceScore, - } - history.push(round) - config.onRoundComplete?.({ scenarioId: scenario.id, round }) - - finalProposal = proposal - lastScore = convergenceScore - priorCritique = critique - - if (convergenceScore >= threshold) { - converged = true - roundsToConverge = r + 1 - break - } - } - - results.push({ - scenarioId: scenario.id, - converged, - roundsToConverge, - finalProposal, - history, - finalScore: lastScore, - }) - } - - const convergedResults = results.filter((r) => r.converged) - const convergenceRate = results.length ? convergedResults.length / results.length : 0 - const avgRoundsToConverge = convergedResults.length - ? convergedResults.reduce((acc, r) => acc + (r.roundsToConverge ?? 0), 0) / - convergedResults.length - : null - const avgFinalScore = results.length - ? results.reduce((acc, r) => acc + r.finalScore, 0) / results.length - : 0 - - return { - scenarios: results, - aggregate: { convergenceRate, avgRoundsToConverge, avgFinalScore }, - config: { maxRounds, convergenceThreshold: threshold }, - } - } -} diff --git a/src/fuzz/explorer-cost.test.ts b/src/fuzz/explorer-cost.test.ts index cf0b41f7..922eb504 100644 --- a/src/fuzz/explorer-cost.test.ts +++ b/src/fuzz/explorer-cost.test.ts @@ -1,6 +1,5 @@ import { describe, expect, it } from 'vitest' import { CostLedger } from '../cost-ledger' -import { costReport } from '../cost-report' import { ValidationError } from '../errors' import { renderCapsuleHtml } from './capsule' import { BehaviorExplorer } from './explorer' @@ -95,8 +94,7 @@ describe('BehaviorExplorer cost budget', () => { expect(entry.model).toBe('gpt-4o') expect(entry.tags?.target).toBe('cost-target') } - const report = costReport(ledger) - expect(report.perModel).toEqual([{ model: 'gpt-4o', usd: 1, entries: 2, unpriced: false }]) + expect(entries.reduce((sum, entry) => sum + entry.costUsd, 0)).toBe(1) }) it('labels ledger entries unattributed when costOf names no model', async () => { diff --git a/src/fuzz/types.ts b/src/fuzz/types.ts index f2f439d0..2f420273 100644 --- a/src/fuzz/types.ts +++ b/src/fuzz/types.ts @@ -306,7 +306,7 @@ export interface ExploreOptions { * Sink for per-run cost entries — each known `costOf` result is recorded * with channel 'agent' and `actualCostUsd` (token axes are zero: the * explorer only sees dollars). Pass the program's shared `CostLedger` so - * `costReport` stamps fuzz spend alongside judge/analyst spend. + * `CostLedger.summary()` counts fuzz spend alongside judge/analyst spend. */ ledger?: CostLedgerHandle /** Observer fired for every known-cost run recorded. */ diff --git a/src/golden-matcher.ts b/src/golden-matcher.ts deleted file mode 100644 index d80a847b..00000000 --- a/src/golden-matcher.ts +++ /dev/null @@ -1,159 +0,0 @@ -/** - * GoldenMatcher — fuzzy matcher for "did the agent produce the expected things?". - * - * Universal primitive across agent-eval consumers. Use it for: - * - Test suites: did the run hit the expected assertions? - * - Tool agents: did the agent emit the expected tool call sequence? - * - Judges: did the verdict include the expected concepts? - * - Design audits: did the auditor surface the planted defects? - * - * Match rule (per golden): - * - Any phrase in `golden.any` (case-insensitive substring) appears in the - * candidate's text fields, OR - * - Any pattern in `golden.anyRegex` (case-insensitive) matches. - * - * Recall is severity-weighted by default: critical=3, major=2, minor=1. - * Missing one critical hurts more than missing three minors. - */ - -export type GoldenSeverity = 'critical' | 'major' | 'minor' - -export interface GoldenSpec { - /** Stable identifier — survives across runs so consumers can grep by id. */ - id: string - /** Severity drives recall weighting. */ - severity: GoldenSeverity - /** - * Substring phrases (case-insensitive). A hit on ANY phrase counts as a - * match. Keep these SHORT (3-6 words) and SPECIFIC. - */ - any: string[] - /** Optional regex patterns. ORed with `any`. */ - anyRegex?: string[] - /** Free-form note — surfaces in reports for humans. */ - hint?: string - /** Optional category for grouping/filtering. */ - category?: string -} - -export interface MatchResult { - /** Same length as goldens; `true` when matched. */ - matches: boolean[] - /** Convenience: count of hits. */ - hits: number - /** Convenience: total goldens. */ - total: number -} - -/** - * Match each golden against `candidates`, where each candidate exposes one or - * more text fields the matcher should search. Defaults to searching all - * string-typed fields concatenated. - */ -export function matchGoldens( - goldens: GoldenSpec[], - candidates: T[], - options: { - /** - * Extract the searchable text for a candidate. Default: concatenate every - * top-level string field with a space. - */ - text?: (candidate: T) => string - } = {}, -): MatchResult { - const extract = options.text ?? defaultExtract - const haystacks = candidates.map((c) => extract(c).toLowerCase()) - const matches = goldens.map((golden) => goldenMatched(golden, haystacks)) - return { - matches, - hits: matches.filter(Boolean).length, - total: goldens.length, - } -} - -function defaultExtract(candidate: unknown): string { - if (typeof candidate === 'string') return candidate - if (candidate && typeof candidate === 'object') { - const parts: string[] = [] - for (const v of Object.values(candidate as Record)) { - if (typeof v === 'string') parts.push(v) - } - return parts.join(' ') - } - return String(candidate ?? '') -} - -function goldenMatched(golden: GoldenSpec, haystacks: string[]): boolean { - for (const phrase of golden.any) { - const needle = phrase.toLowerCase().trim() - if (!needle) continue - if (haystacks.some((h) => h.includes(needle))) return true - } - for (const pattern of golden.anyRegex ?? []) { - let re: RegExp - try { - re = new RegExp(pattern, 'i') - } catch { - continue - } - if (haystacks.some((h) => re.test(h))) return true - } - return false -} - -/** Severity weights — exposed so consumers can override (rare). */ -export const DEFAULT_SEVERITY_WEIGHTS: Record = { - critical: 3, - major: 2, - minor: 1, -} - -/** Severity-weighted recall over a MatchResult + the goldens that produced it. */ -export function weightedRecall( - goldens: GoldenSpec[], - result: MatchResult, - weights: Record = DEFAULT_SEVERITY_WEIGHTS, -): number { - if (goldens.length === 0) return 1 - const total = goldens.reduce((s, g) => s + (weights[g.severity] ?? 1), 0) - if (total === 0) return 1 - const hit = goldens.reduce( - (s, g, i) => s + (result.matches[i] ? (weights[g.severity] ?? 1) : 0), - 0, - ) - return hit / total -} - -/** - * Precision proxy: fraction of emitted candidates that match SOME golden. - * - * No human-labelled negatives means unmatched candidates are SOFT false - * positives — punishes verbose agents that pad with filler. Doesn't punish - * unknown-but-real findings; the way to tighten this is to grow the golden - * set, not to invent a stricter score. - */ -export function precision( - goldens: GoldenSpec[], - candidates: T[], - options: { text?: (candidate: T) => string } = {}, -): number { - if (candidates.length === 0) return 1 - const extract = options.text ?? defaultExtract - let matched = 0 - for (const cand of candidates) { - const haystack = extract(cand).toLowerCase() - const matchedAny = goldens.some( - (g) => - g.any.some((phrase) => phrase.length > 0 && haystack.includes(phrase.toLowerCase())) || - (g.anyRegex ?? []).some((pat) => { - try { - return new RegExp(pat, 'i').test(haystack) - } catch { - return false - } - }), - ) - if (matchedAny) matched++ - } - return matched / candidates.length -} diff --git a/src/judge-runner.test.ts b/src/judge-runner.test.ts deleted file mode 100644 index 539914f5..00000000 --- a/src/judge-runner.test.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { compilerJudge, runJudgeFleet } from './judge-runner' -import type { HarnessConfig, SandboxDriver, SandboxResult } from './sandbox-harness' - -/** - * Driver that records peak concurrent `exec` calls. Each exec yields to the - * event loop a few times so the pool's bound is observable: with unbounded - * fan-out every spec's subprocess is in-flight at once. - */ -function makeConcurrencyTrackingDriver(): SandboxDriver & { peak: number; active: number } { - const state = { id: 'tracking', peak: 0, active: 0 } as SandboxDriver & { - peak: number - active: number - } - state.exec = async ( - phase: SandboxResult['phase'], - _command: string, - _config: HarnessConfig, - ): Promise => { - state.active++ - if (state.active > state.peak) state.peak = state.active - // Yield repeatedly so other workers can start while we're "running". - for (let i = 0; i < 5; i++) await Promise.resolve() - state.active-- - return { phase, exitCode: 0, stdout: '', stderr: '', wallMs: 1 } - } - return state -} - -describe('runJudgeFleet bounded concurrency', () => { - it('caps concurrent subprocesses at the configured concurrency', async () => { - const driver = makeConcurrencyTrackingDriver() - const specs = Array.from({ length: 12 }, (_, i) => - compilerJudge(`judge-${i}`, { runCommand: 'true' } as HarnessConfig), - ) - - const results = await runJudgeFleet(specs, { driver, concurrency: 3 }) - - expect(results).toHaveLength(12) - // OLD behavior (Promise.all over all specs) would peak at 12. The bounded - // pool must never exceed the configured concurrency. - expect(driver.peak).toBeLessThanOrEqual(3) - expect(driver.peak).toBeGreaterThan(0) - }) - - it('preserves result order matching input spec order', async () => { - const driver = makeConcurrencyTrackingDriver() - const specs = Array.from({ length: 8 }, (_, i) => - compilerJudge(`spec-${i}`, { runCommand: 'true' } as HarnessConfig), - ) - const results = await runJudgeFleet(specs, { driver, concurrency: 2 }) - expect(results.map((r) => r.id)).toEqual(specs.map((s) => s.id)) - }) - - it('returns [] for an empty spec list without spawning workers', async () => { - const driver = makeConcurrencyTrackingDriver() - const results = await runJudgeFleet([], { driver }) - expect(results).toEqual([]) - expect(driver.peak).toBe(0) - }) -}) diff --git a/src/judge-runner.ts b/src/judge-runner.ts deleted file mode 100644 index da5f1479..00000000 --- a/src/judge-runner.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { cpus } from 'node:os' -import { - type HarnessConfig, - type SandboxDriver, - SandboxHarness, - type SandboxHarnessResult, - SubprocessSandboxDriver, -} from './sandbox-harness' -import { TraceEmitter } from './trace/emitter' -import { InMemoryTraceStore } from './trace/store' - -export type SandboxJudgeKind = 'compiler' | 'test' | 'linter' | 'security' - -export interface SandboxJudgeSpec { - id: string - kind: SandboxJudgeKind - config: HarnessConfig -} - -export interface SandboxJudgeResult { - id: string - kind: SandboxJudgeKind - passed: boolean - score: number - summary: string - detail: SandboxHarnessResult -} - -export interface JudgeFleetOptions { - driver?: SandboxDriver - parallel?: boolean - /** - * Max concurrent judge subprocesses. Each spec spawns its own subprocess - * via the driver, so unbounded fan-out exhausts file descriptors / PIDs. - * Defaults to the host CPU count. Ignored when `parallel === false`. - */ - concurrency?: number -} - -export class JudgeRunner { - private readonly driver: SandboxDriver - - constructor(driver: SandboxDriver = new SubprocessSandboxDriver()) { - this.driver = driver - } - - async run(spec: SandboxJudgeSpec): Promise { - const store = new InMemoryTraceStore() - const emitter = new TraceEmitter(store, { runId: `judge-${spec.id}` }) - await emitter.startRun({ - scenarioId: spec.id, - layer: 'meta', - projectId: 'judge-runner', - }) - const harness = new SandboxHarness(this.driver) - const detail = await harness.run(spec.config, emitter) - await emitter.endRun({ pass: detail.passed, score: detail.score, notes: `${spec.kind} judge` }) - return { - id: spec.id, - kind: spec.kind, - passed: detail.passed, - score: detail.score, - summary: renderJudgeSummary(spec.kind, detail), - detail, - } - } -} - -export async function runJudgeFleet( - specs: SandboxJudgeSpec[], - options: JudgeFleetOptions = {}, -): Promise { - const runner = new JudgeRunner(options.driver) - if (options.parallel === false) { - const results: SandboxJudgeResult[] = [] - for (const spec of specs) results.push(await runner.run(spec)) - return results - } - const concurrency = Math.max(1, options.concurrency ?? cpus().length) - const results: SandboxJudgeResult[] = new Array(specs.length) - let cursor = 0 - async function worker(): Promise { - while (true) { - const i = cursor++ - if (i >= specs.length) return - results[i] = await runner.run(specs[i]!) - } - } - await Promise.all(Array.from({ length: Math.min(concurrency, specs.length) }, () => worker())) - return results -} - -/** Build a `SandboxJudgeSpec` that scores whether the harness compiles without errors. */ -export function compilerJudge(id: string, config: HarnessConfig): SandboxJudgeSpec { - return { id, kind: 'compiler', config } -} - -/** Build a `SandboxJudgeSpec` that scores the harness by its test-suite pass rate. */ -export function testJudge(id: string, config: HarnessConfig): SandboxJudgeSpec { - return { id, kind: 'test', config } -} - -/** Build a `SandboxJudgeSpec` that scores the harness by linter rule violations. */ -export function linterJudge(id: string, config: HarnessConfig): SandboxJudgeSpec { - return { id, kind: 'linter', config } -} - -/** Build a `SandboxJudgeSpec` that scores the harness output for security issues via a security scanner. */ -export function securityJudge(id: string, config: HarnessConfig): SandboxJudgeSpec { - return { id, kind: 'security', config } -} - -function renderJudgeSummary(kind: SandboxJudgeKind, detail: SandboxHarnessResult): string { - if (!detail.passed) return `${kind} judge failed` - if (detail.test?.testsTotal) - return `${kind} judge passed ${detail.test.testsPassed}/${detail.test.testsTotal} tests` - return `${kind} judge passed` -} diff --git a/src/multi-layer-verifier.ts b/src/multi-layer-verifier.ts index d1447c37..907bbf21 100644 --- a/src/multi-layer-verifier.ts +++ b/src/multi-layer-verifier.ts @@ -1,11 +1,9 @@ /** * Multi-layer verifier — ordered pipeline of verification layers. * - * Different contract from {@link JudgeRunner} (which runs parallel - * specs against a sandbox). MultiLayerVerifier is a DAG of layers - * (install → typecheck → build → lint → serve → semantic → …) with - * dependency-based skip, per-layer findings, soft-fail semantics, and - * an aggregated `blendedScore` across all passed layers. + * A DAG of layers (install → typecheck → build → lint → serve → semantic → …) + * with dependency-based skip, per-layer findings, soft-fail semantics, and an + * aggregated `blendedScore` across all passed layers. * * Use when you want: * - ordered stages where a failing upstream stage skips downstream ones @@ -13,13 +11,8 @@ * - a single composite score across stages with per-stage weights * - soft-fail stages whose failure doesn't abort the pipeline * - * Use {@link JudgeRunner} when you want: - * - N independent judges running in parallel against the same artifact - * - no inter-judge dependencies - * - boolean `passed` per judge + overall - * - * Both primitives compose — JudgeRunner can be invoked as a single - * layer inside a MultiLayerVerifier if that suits the caller. + * A layer body is an arbitrary async function, so independent checks that need + * no ordering run as one layer that resolves them in parallel. */ import { packageVersion } from './package-version' @@ -38,8 +31,8 @@ export interface Finding { /** Optional layer name the finding belongs to (set by the verifier if omitted). */ layer?: string /** - * Free-form structured payload — used by `multiToolchainLayer` to attach - * `{ adapter: 'pnpm' }`, by judges to attach evidence pointers, etc. + * Free-form structured payload — a layer attaches its toolchain + * (`{ adapter: 'pnpm' }`), a judge attaches evidence pointers, etc. * Renderers MAY interrogate; agent-eval primitives never assume shape. */ detail?: Record diff --git a/src/multi-toolchain-layer.test.ts b/src/multi-toolchain-layer.test.ts deleted file mode 100644 index bfc735f2..00000000 --- a/src/multi-toolchain-layer.test.ts +++ /dev/null @@ -1,196 +0,0 @@ -import { describe, expect, it } from 'vitest' -import type { LayerResult } from './multi-layer-verifier' -import { mergeLayerResults, multiToolchainLayer } from './multi-toolchain-layer' - -function mkResult( - status: LayerResult['status'], - score?: number, - findings: LayerResult['findings'] = [], -): LayerResult { - return { - layer: 'install', - status, - score, - durationMs: 100, - findings, - } -} - -describe('mergeLayerResults', () => { - it('skipped when no adapters', () => { - const r = mergeLayerResults('install', []) - expect(r.status).toBe('skipped') - expect(r.findings).toHaveLength(0) - expect(r.reason).toBe('no adapters') - }) - - it('passes through when single adapter (preserves findings + reason)', () => { - const r = mergeLayerResults('install', [ - { - adapter: 'pnpm', - result: { - ...mkResult('pass', 1), - findings: [{ severity: 'info', layer: 'install', message: 'pnpm install ok' }], - }, - }, - ]) - expect(r.status).toBe('pass') - expect(r.score).toBe(1) - expect(r.findings[0]!.detail).toEqual({ adapter: 'pnpm' }) - }) - - it('worst-of-parts status reduction', () => { - const r = mergeLayerResults('install', [ - { adapter: 'pnpm', result: mkResult('pass', 1) }, - { adapter: 'npm', result: mkResult('fail', 0) }, - ]) - expect(r.status).toBe('fail') - }) - - it('error > timeout > fail > skipped > pass', () => { - const cases: Array<[Array, LayerResult['status']]> = [ - [['pass', 'skipped'], 'skipped'], - [['pass', 'fail'], 'fail'], - [['fail', 'timeout'], 'timeout'], - [['timeout', 'error'], 'error'], - [['pass', 'pass'], 'pass'], - ] - for (const [statuses, expected] of cases) { - const r = mergeLayerResults( - 'x', - statuses.map((s, i) => ({ adapter: `a${i}`, result: mkResult(s) })), - ) - expect(r.status).toBe(expected) - } - }) - - it('weighted-mean score across numeric adapters; skips contribute null', () => { - const r = mergeLayerResults('build', [ - { adapter: 'pnpm', result: mkResult('pass', 0.8) }, - { adapter: 'npm', result: mkResult('pass', 1.0) }, - { adapter: 'forge', result: mkResult('skipped') }, - ]) - expect(r.score).toBeCloseTo(0.9, 2) - }) - - it('attributes findings with detail.adapter', () => { - const r = mergeLayerResults('typecheck', [ - { - adapter: 'pnpm', - result: mkResult('fail', 0, [ - { severity: 'major', layer: 'typecheck', message: 'tsc 4 errors' }, - ]), - }, - { - adapter: 'forge', - result: mkResult('pass', 1, [ - { severity: 'info', layer: 'typecheck', message: 'forge ok' }, - ]), - }, - ]) - expect(r.findings).toHaveLength(2) - expect(r.findings.find((f) => f.message === 'tsc 4 errors')?.detail).toMatchObject({ - adapter: 'pnpm', - }) - expect(r.findings.find((f) => f.message === 'forge ok')?.detail).toMatchObject({ - adapter: 'forge', - }) - }) - - it('reason concatenates adapter:status; durationMs is max-of-parts', () => { - const r = mergeLayerResults('install', [ - { adapter: 'pnpm', result: { ...mkResult('pass', 1), durationMs: 5000 } }, - { adapter: 'npm', result: { ...mkResult('skipped'), durationMs: 100 } }, - ]) - expect(r.reason).toBe('pnpm: pass · npm: skipped') - expect(r.durationMs).toBe(5000) - }) -}) - -describe('multiToolchainLayer', () => { - it('runs adapters in parallel + merges results', async () => { - const calls: string[] = [] - const layer = multiToolchainLayer({ - name: 'install', - adapters: [{ id: 'pnpm' }, { id: 'npm' }, { id: 'forge' }], - adapterName: (a) => a.id, - run: async (a) => { - calls.push(a.id) - return mkResult(a.id === 'forge' ? 'fail' : 'pass', a.id === 'forge' ? 0 : 1) - }, - }) - const r = await layer.run({ env: null, prior: {}, signal: new AbortController().signal }) - expect(r.status).toBe('fail') - expect(calls.sort()).toEqual(['forge', 'npm', 'pnpm']) - expect(r.detail).toMatchObject({ - adapters: expect.arrayContaining([ - expect.objectContaining({ adapter: 'pnpm', status: 'pass' }), - ]), - }) - }) - - it('catches per-adapter throws as status=error (rest of layer still completes)', async () => { - const layer = multiToolchainLayer({ - name: 'install', - adapters: ['pnpm', 'cursed'], - adapterName: (a) => a, - run: async (a) => { - if (a === 'cursed') throw new Error('boom') - return mkResult('pass', 1) - }, - }) - const r = await layer.run({ env: null, prior: {}, signal: new AbortController().signal }) - expect(r.status).toBe('error') // worst-of (pass + error) - const cursed = r.findings.find( - (f) => f.detail && (f.detail as Record).adapter === 'cursed', - ) - expect(cursed?.message).toBe('boom') - }) - - it('skipped result on zero adapters (no toolchain detected)', async () => { - const layer = multiToolchainLayer({ - name: 'install', - adapters: [], - adapterName: (a) => a, - run: async () => mkResult('pass', 1), - }) - const r = await layer.run({ env: null, prior: {}, signal: new AbortController().signal }) - expect(r.status).toBe('skipped') - expect(r.reason).toMatch(/no adapters detected/) - }) - - it('respects maxParallel — chunks calls', async () => { - let inFlight = 0 - let peak = 0 - const layer = multiToolchainLayer({ - name: 'x', - adapters: [1, 2, 3, 4, 5, 6, 7, 8], - adapterName: (a) => `a${a}`, - maxParallel: 2, - run: async () => { - inFlight++ - peak = Math.max(peak, inFlight) - await new Promise((r) => setTimeout(r, 10)) - inFlight-- - return mkResult('pass', 1) - }, - }) - await layer.run({ env: null, prior: {}, signal: new AbortController().signal }) - expect(peak).toBeLessThanOrEqual(2) - }) - - it('passes verify ctx through to adapter run fn', async () => { - const seen: unknown[] = [] - const layer = multiToolchainLayer<{ marker: number }, string>({ - name: 'x', - adapters: ['pnpm'], - adapterName: (a) => a, - run: async (_a, ctx) => { - seen.push(ctx.env.marker) - return mkResult('pass', 1) - }, - }) - await layer.run({ env: { marker: 42 }, prior: {}, signal: new AbortController().signal }) - expect(seen).toEqual([42]) - }) -}) diff --git a/src/multi-toolchain-layer.ts b/src/multi-toolchain-layer.ts deleted file mode 100644 index ccf3eb04..00000000 --- a/src/multi-toolchain-layer.ts +++ /dev/null @@ -1,266 +0,0 @@ -/** - * Multi-toolchain layer factory + merge helper. - * - * Some verification stages (install, typecheck, build, lint) run the - * SAME logical layer across multiple parallel adapters — pnpm AND npm - * AND cargo AND forge for a polyglot scaffold. The verifier presents - * one row per stage; the toolchain breakdown lives in `findings.detail`. - * - * This module provides the merge: take N independent `LayerResult`s - * (one per adapter) and reduce them to a single `LayerResult` whose - * status is the worst of the parts and whose findings cite the adapter - * that produced each one. Plus a {@link multiToolchainLayer} factory - * that runs the adapter calls in parallel + applies the reducer. - * - * Pure utility — composes with {@link MultiLayerVerifier}.{run}. - */ - -import type { - Layer, - LayerResult, - LayerStatus, - Severity, - VerifyContext, -} from './multi-layer-verifier' - -// ─── Status reduction ────────────────────────────────────────────────── - -const STATUS_RANK: Record = { - pass: 0, - skipped: 1, - fail: 2, - timeout: 3, - error: 4, -} - -function worst(a: LayerStatus, b: LayerStatus): LayerStatus { - return (STATUS_RANK[a] ?? 0) >= (STATUS_RANK[b] ?? 0) ? a : b -} - -const SEVERITY_RANK: Record = { - info: 0, - minor: 1, - major: 2, - critical: 3, -} - -function maxSeverity(findings: ReadonlyArray<{ severity: Severity }>): Severity { - let best: Severity = 'info' - for (const f of findings) { - if (SEVERITY_RANK[f.severity] > SEVERITY_RANK[best]) best = f.severity - } - return best -} - -// ─── Merge ────────────────────────────────────────────────────────────── - -export interface AdapterRun { - /** Identifier for the adapter (e.g. 'pnpm', 'npm', 'cargo', 'forge'). */ - adapter: string - result: LayerResult -} - -export interface MergeOptions { - /** - * How to combine per-adapter `durationMs`. Default `'max'` (parallel - * wall-clock). Set `'sum'` when reporting total work done across - * adapters rather than wall time. - */ - mergeDuration?: 'max' | 'sum' - /** - * Prefix finding messages with a per-adapter tag (e.g. `[pnpm] typecheck failed`). - * Default: no prefix (renderers read `detail.adapter` instead). - */ - messagePrefixer?: (adapter: string) => string - /** - * How to reduce per-adapter `LayerResult.diagnostics` into the merged - * result's diagnostics. `'max'` (default) — for each key, merged = - * max across adapters where value is non-null (matches "if ANY adapter - * saw N errors, merged saw N"). `'sum'` — sum non-null values. - */ - mergeDiagnostics?: 'max' | 'sum' -} - -/** - * Reduce N adapter runs to a single `LayerResult` for a logical layer. - * - * - status: worst of the parts (pass < skipped < fail < timeout < error) - * - score: weighted mean of numeric scores (skip = no contribution) - * - findings: union, each tagged with `detail.adapter` - * - durationMs: `mergeDuration` option (default 'max' for parallel wall-clock) - * - diagnostics: `mergeDiagnostics` option (default 'max' per key) - * - reason: " · "-joined `name: status` per adapter - */ -export function mergeLayerResults( - name: string, - perAdapter: AdapterRun[], - options: MergeOptions = {}, -): LayerResult { - const mergeDuration = options.mergeDuration ?? 'max' - const mergeDiagnostics = options.mergeDiagnostics ?? 'max' - const prefix = options.messagePrefixer - - if (perAdapter.length === 0) { - return { - layer: name, - status: 'skipped', - durationMs: 0, - findings: [], - reason: 'no adapters', - } - } - if (perAdapter.length === 1) { - const only = perAdapter[0]! - return { - ...only.result, - layer: name, - findings: only.result.findings.map((f) => ({ - ...f, - layer: name, - message: prefix ? `${prefix(only.adapter)} ${f.message}` : f.message, - detail: { ...(f.detail ?? {}), adapter: only.adapter }, - })), - reason: only.result.reason ?? `${only.adapter}: ${only.result.status}`, - } - } - - let status: LayerStatus = 'pass' - let weightedScoreSum = 0 - let weightCount = 0 - const findings: LayerResult['findings'] = [] - let durationMs = 0 - const reasonParts: string[] = [] - const diagnostics: Record = {} - - for (const { adapter, result } of perAdapter) { - status = worst(status, result.status) - if (typeof result.score === 'number') { - weightedScoreSum += result.score - weightCount += 1 - } - durationMs = - mergeDuration === 'sum' - ? durationMs + result.durationMs - : Math.max(durationMs, result.durationMs) - reasonParts.push(`${adapter}: ${result.status}`) - for (const f of result.findings) { - findings.push({ - ...f, - layer: name, - message: prefix ? `${prefix(adapter)} ${f.message}` : f.message, - detail: { ...(f.detail ?? {}), adapter }, - }) - } - for (const [k, v] of Object.entries(result.diagnostics ?? {})) { - if (typeof v !== 'number' || !Number.isFinite(v)) continue - const prev = diagnostics[k] - if (prev == null) diagnostics[k] = v - else diagnostics[k] = mergeDiagnostics === 'sum' ? prev + v : Math.max(prev, v) - } - } - - return { - layer: name, - status, - score: weightCount > 0 ? weightedScoreSum / weightCount : undefined, - durationMs, - findings, - reason: reasonParts.join(' · '), - diagnostics: Object.keys(diagnostics).length > 0 ? diagnostics : undefined, - detail: { - adapters: perAdapter.map(({ adapter, result }) => ({ - adapter, - status: result.status, - score: result.score ?? null, - })), - worstSeverity: maxSeverity(findings), - }, - } -} - -// ─── Layer factory ────────────────────────────────────────────────────── - -export interface MultiToolchainLayerConfig { - name: string - adapters: ReadonlyArray - /** Adapter identifier — used in findings + reason. */ - adapterName: (a: Adapter) => string - /** Run a single adapter against the verify context. */ - run: (a: Adapter, ctx: VerifyContext) => Promise | LayerResult - dependsOn?: string[] - weight?: number - failContributesToScore?: boolean - capMs?: number - /** - * Per-adapter parallel cap. Defaults to 8 — defense in depth against a - * caller passing 50 adapters and fanning out 50 simultaneous subprocesses. - * Adapters that need higher concurrency raise this explicitly. - */ - maxParallel?: number -} - -/** - * Build a {@link Layer} that fans the same logical stage across N adapters - * in parallel and merges via {@link mergeLayerResults}. - * - * Per-adapter throws are caught + converted to `status: 'error'` results - * so one bad adapter doesn't poison the whole layer. - */ -export function multiToolchainLayer( - config: MultiToolchainLayerConfig, -): Layer { - const maxParallel = Math.max(1, config.maxParallel ?? 8) - return { - name: config.name, - dependsOn: config.dependsOn, - weight: config.weight, - failContributesToScore: config.failContributesToScore, - capMs: config.capMs, - async run(ctx) { - if (config.adapters.length === 0) { - return { - layer: config.name, - status: 'skipped', - durationMs: 0, - findings: [], - reason: 'no adapters detected', - } - } - - const runOne = async (adapter: Adapter): Promise => { - const adapterName = config.adapterName(adapter) - try { - const r = await config.run(adapter, ctx) - return { adapter: adapterName, result: r } - } catch (err) { - return { - adapter: adapterName, - result: { - layer: config.name, - status: 'error', - durationMs: 0, - findings: [ - { - severity: 'major', - layer: config.name, - message: err instanceof Error ? err.message : String(err), - detail: { adapter: adapterName }, - }, - ], - reason: err instanceof Error ? err.message : String(err), - }, - } - } - } - - // Bounded parallelism — chunked into groups of size maxParallel. - const results: AdapterRun[] = [] - for (let i = 0; i < config.adapters.length; i += maxParallel) { - const chunk = config.adapters.slice(i, i + maxParallel) - const chunkResults = await Promise.all(chunk.map(runOne)) - results.push(...chunkResults) - } - return mergeLayerResults(config.name, results) - }, - } -} diff --git a/src/reviewer.test.ts b/src/reviewer.test.ts deleted file mode 100644 index a1a9819f..00000000 --- a/src/reviewer.test.ts +++ /dev/null @@ -1,201 +0,0 @@ -import { describe, expect, it, vi } from 'vitest' -import { CostLedger } from './cost-ledger' -import { buildReviewerPrompt, createDefaultReviewer } from './reviewer' - -const BASE_INPUT = { - shot: 2, - userRequest: 'build an NFT mint page with supply counter, mint button', - traceSummary: 'tool calls: {Write: 3, Edit: 2}, errors: none', - verification: { - blendedScore: 0.5, - allPass: false, - failCount: 2, - failingLayers: ['typecheck', 'semantic'], - }, - memory: [ - { - shot: 1, - confidence: 0.85, - shouldContinue: true, - observations: 'worker wrote App.tsx', - diagnosis: 'wagmi imports wrong', - nextShotInstruction: 'fix imports', - }, - ], -} - -describe('buildReviewerPrompt', () => { - it('emits system + user with all context blocks present', () => { - const { system, user } = buildReviewerPrompt(BASE_INPUT) - expect(system).toMatch(/senior-engineer-grade reviewer/) - expect(user).toMatch(/shot 2 of the review loop/) - expect(user).toMatch(/build an NFT mint page/) - expect(user).toMatch(/tool calls:/) - expect(user).toMatch(/blendedScore: 0.50/) - expect(user).toMatch(/failing layers: typecheck, semantic/) - expect(user).toMatch(/shot 1 — confidence=0.85/) - expect(user).toMatch(/STRICT JSON/) - }) - - it('injects extraContext block when provided', () => { - const { user } = buildReviewerPrompt({ - ...BASE_INPUT, - extraContext: 'workdir: src/App.tsx, src/MintButton.tsx', - }) - expect(user).toMatch(/EXTRA CONTEXT/) - expect(user).toMatch(/src\/MintButton\.tsx/) - }) - - it('omits extraContext block entirely when not provided', () => { - const { user } = buildReviewerPrompt(BASE_INPUT) - expect(user).not.toMatch(/EXTRA CONTEXT/) - }) - - it('shows "(no prior shots)" when memory is empty', () => { - const { user } = buildReviewerPrompt({ ...BASE_INPUT, memory: [] }) - expect(user).toMatch(/\(no prior shots\)/) - }) - - it('signals no-failing-layers when verification.failingLayers is empty', () => { - const { user } = buildReviewerPrompt({ - ...BASE_INPUT, - verification: { blendedScore: 1, allPass: true, failCount: 0, failingLayers: [] }, - }) - expect(user).toMatch(/no layers failing/) - }) - - it('trailingContext renders at the end when provided', () => { - const { user } = buildReviewerPrompt({ - ...BASE_INPUT, - trailingContext: 'leaf_id: nft-mint-page', - }) - expect(user).toMatch(/TRAILING CONTEXT[\s\S]+leaf_id: nft-mint-page/) - }) -}) - -describe('createDefaultReviewer', () => { - function mockFetch(responses: Array) { - let i = 0 - return (async () => { - const r = responses[Math.min(i++, responses.length - 1)]! - if ('status' in r && 'body' in r) { - return new Response((r as { body: string }).body, { - status: (r as { status: number }).status, - }) - } - return new Response( - JSON.stringify({ - model: 'mock', - choices: [{ message: { content: JSON.stringify(r) } }], - usage: { total_tokens: 100 }, - _response_cost: 0.001, - }), - { status: 200, headers: { 'content-type': 'application/json' } }, - ) - }) as unknown as typeof fetch - } - - it('calls LLM, parses structured output, returns ReviewerOutput', async () => { - const costLedger = new CostLedger() - const fetch = mockFetch([ - { - observations: 'worker wrote 3 files via Edit, no errors logged, build failed on typecheck.', - diagnosis: 'wagmi v2 API misuse — useAccount from wrong import path, ts will not compile.', - nextShotInstruction: - 'FIX THESE: 1) change `import { useAccount } from "wagmi/core"` to `from "wagmi"` in src/App.tsx', - shouldContinue: true, - confidence: 0.85, - }, - ]) - const reviewer = createDefaultReviewer({ model: 'mock-model', llm: { fetch }, costLedger }) - const r = await reviewer(BASE_INPUT) - expect(r.available).toBe(true) - expect(r.shot).toBe(2) - expect(r.confidence).toBeCloseTo(0.85) - expect(r.shouldContinue).toBe(true) - expect(r.diagnosis).toMatch(/wagmi v2/) - expect(r.costUsd).toBeCloseTo(0.001) - expect(costLedger.list()).toEqual([ - expect.objectContaining({ channel: 'analyst', actor: 'default-reviewer', costUsd: 0.001 }), - ]) - }) - - it('clamps confidence to [0, 1]', async () => { - const fetch = mockFetch([ - { - observations: 'x'.repeat(30), - diagnosis: 'y'.repeat(30), - nextShotInstruction: 'z'.repeat(50), - shouldContinue: false, - confidence: 1.5, - }, - ]) - const r = await createDefaultReviewer({ model: 'm', llm: { fetch } })(BASE_INPUT) - expect(r.confidence).toBe(1) - }) - - it('soft-fails available=false on LLM error; uses soft-fail defaults', async () => { - const fetch = mockFetch([{ status: 500, body: 'upstream oops' }]) - const r = await createDefaultReviewer({ - model: 'm', - llm: { fetch, maximumAttempts: 1 }, - })(BASE_INPUT) - expect(r.available).toBe(false) - expect(r.error).toMatch(/500/) - expect(r.confidence).toBe(0.3) - expect(r.shouldContinue).toBe(true) - expect(r.nextShotInstruction).toMatch(/Inspect the verification findings/) - }) - - it('honors custom soft-fail defaults', async () => { - const fetch = mockFetch([{ status: 503, body: 'rate-limited' }]) - const r = await createDefaultReviewer({ - model: 'm', - llm: { fetch, maximumAttempts: 1 }, - softFailDefaults: { shouldContinue: false, confidence: 0 }, - })(BASE_INPUT) - expect(r.shouldContinue).toBe(false) - expect(r.confidence).toBe(0) - }) - - it('custom promptBuilder is used instead of default', async () => { - const fetch = vi.fn( - async () => - new Response( - JSON.stringify({ - choices: [ - { - message: { - content: - '{"observations":"' + - 'o'.repeat(25) + - '","diagnosis":"' + - 'd'.repeat(25) + - '","nextShotInstruction":"' + - 'i'.repeat(50) + - '","shouldContinue":false,"confidence":0.5}', - }, - }, - ], - usage: {}, - }), - { status: 200 }, - ), - ) as unknown as typeof globalThis.fetch - const custom = vi.fn((_: unknown) => ({ system: 'CUSTOM-SYS', user: 'CUSTOM-USER' })) - const reviewer = createDefaultReviewer({ - model: 'm', - llm: { fetch }, - promptBuilder: custom as never, - }) - await reviewer(BASE_INPUT) - expect(custom).toHaveBeenCalledOnce() - const call = (fetch as unknown as ReturnType).mock.calls[0]! as unknown as [ - string, - RequestInit, - ] - const body = JSON.parse(call[1].body as string) - expect(body.messages[0].content).toBe('CUSTOM-SYS') - expect(body.messages[1].content).toBe('CUSTOM-USER') - }) -}) diff --git a/src/reviewer.ts b/src/reviewer.ts deleted file mode 100644 index 7a9a758e..00000000 --- a/src/reviewer.ts +++ /dev/null @@ -1,305 +0,0 @@ -/** - * Reviewer primitives — prompt builder + default ReviewFn factory. - * - * `buildReviewerPrompt` is the pure, LLM-agnostic piece: takes - * `ReviewerPromptInput` (user request, trace summary, verification - * summary, memory, optional extra context) and emits the system + - * user message pair. No LLM dependency — callers that want to drive - * their own transport get full control. - * - * `createDefaultReviewer` is the convenience factory: wires the prompt - * builder to `callLlmJson` with a default schema + soft-fail policy. - * Returns a function that maps `ReviewerPromptInput` to `ReviewerOutput`. - * - * Same pattern as `runSemanticConceptJudge` / `createSemanticConceptJudge`: - * low-level pure builder + high-level factory built on top. - */ - -import { CostLedger, type CostLedgerHandle, type CostReceipt } from './cost-ledger' -import { - callLlmJson, - costReceiptFromLlm, - costReceiptFromLlmError, - type LlmCallRequest, - type LlmClientOptions, - maximumChargeForLlmRequest, -} from './llm-client' - -// ─── Types ────────────────────────────────────────────────────────────── - -export interface ReviewerMemoryEntry { - shot: number - ts?: string - observations?: string - diagnosis?: string - nextShotInstruction?: string - shouldContinue?: boolean - confidence?: number -} - -export interface ReviewerVerificationSummary { - blendedScore: number - allPass: boolean - failCount: number - failingLayers?: string[] -} - -export interface ReviewerPromptInput { - shot: number - userRequest: string - /** - * Compact trace summary — tool-call counts, errors, recent activity - * lines. Built by the caller from whatever trace format they have; - * agent-eval does not prescribe. - */ - traceSummary: string - verification: ReviewerVerificationSummary - memory: ReviewerMemoryEntry[] - /** - * Optional extra context injected into the prompt between the trace - * and the verification blocks. Use for workdir file-tree snapshots, - * scaffold descriptions, or any environmental fact the reviewer - * needs to direct the next shot accurately. - */ - extraContext?: string - /** - * Optional extra section appended at the end of the prompt (e.g. - * leaf metadata, scenario id). Free-form — no agent-eval-shaped - * schema. - */ - trailingContext?: string -} - -export interface ReviewerOutput { - shot: number - observations: string - diagnosis: string - nextShotInstruction: string - shouldContinue: boolean - /** 0..1 self-assessed confidence in the directive. */ - confidence: number - /** LLM cost in USD if the transport reports it, else null. */ - costUsd: number | null - durationMs: number - /** False when the LLM errored or returned malformed JSON; caller soft-fails to defaults. */ - available: boolean - error?: string -} - -export interface ReviewerSoftFailDefaults { - observations?: string - diagnosis?: string - nextShotInstruction?: string - shouldContinue?: boolean - confidence?: number -} - -export interface CreateDefaultReviewerOptions { - /** Model id to call. */ - model: string - /** Per-call timeout. Default 300s. */ - timeoutMs?: number - /** Provider-enforced output limit. Default 4000. */ - maxTokens?: number - /** LlmClient transport config (baseUrl, apiKey, authHeader, etc.). */ - llm?: LlmClientOptions - /** Shared run spend account. */ - costLedger?: CostLedgerHandle - costPhase?: string - signal?: AbortSignal - /** - * Override the prompt builder. Default: `buildReviewerPrompt`. - * Consumers with different reviewer voices pass their own. - */ - promptBuilder?: (input: ReviewerPromptInput) => { system: string; user: string } - /** - * Soft-fail values when the LLM throws or returns unparseable JSON. - * Matches VerticalBench's shipped policy: continue with generic - * instruction at confidence 0.3 so the worker keeps trying. - */ - softFailDefaults?: ReviewerSoftFailDefaults -} - -// ─── JSON schema ─────────────────────────────────────────────────────── - -const REVIEWER_SCHEMA = { - type: 'object', - additionalProperties: false, - required: ['observations', 'diagnosis', 'nextShotInstruction', 'shouldContinue', 'confidence'], - properties: { - observations: { type: 'string', minLength: 20, maxLength: 2000 }, - diagnosis: { type: 'string', minLength: 20, maxLength: 1500 }, - nextShotInstruction: { type: 'string', minLength: 40, maxLength: 3000 }, - shouldContinue: { type: 'boolean' }, - confidence: { type: 'number', minimum: 0, maximum: 1 }, - }, -} as const - -// ─── Prompt builder ──────────────────────────────────────────────────── - -function summarizeMemory(memory: ReviewerMemoryEntry[]): string { - if (memory.length === 0) return '(no prior shots)' - return memory - .map((m) => { - const header = `shot ${m.shot} — confidence=${(m.confidence ?? 0).toFixed(2)} shouldContinue=${m.shouldContinue ?? '?'}` - const obs = m.observations ? ` observations: ${m.observations.slice(0, 400)}` : '' - const diag = m.diagnosis ? ` diagnosis: ${m.diagnosis.slice(0, 400)}` : '' - const instr = m.nextShotInstruction - ? ` instruction given: ${m.nextShotInstruction.slice(0, 400)}` - : '' - return [header, obs, diag, instr].filter(Boolean).join('\n') - }) - .join('\n\n') -} - -/** - * Build the reviewer's system + user messages. Pure function, no LLM - * call. Callers that want their own transport or a different structured - * output can use this and skip `createDefaultReviewer` entirely. - */ -export function buildReviewerPrompt(input: ReviewerPromptInput): { system: string; user: string } { - const system = - 'You are a senior-engineer-grade reviewer directing an agent through a multi-shot build. ' + - "Your job is NOT to grade; your job IS to direct the worker's next shot using the trace, " + - 'verification result, prior memory, and user request. Return STRICT JSON. No prose outside the JSON.' - - const failingLayersBlock = - input.verification.failingLayers && input.verification.failingLayers.length > 0 - ? `failing layers: ${input.verification.failingLayers.join(', ')}` - : 'no layers failing' - - const user = `=== SHOT NUMBER === -shot ${input.shot} of the review loop - -=== USER REQUEST === -${input.userRequest} - -=== WORKER TRACE (shot ${input.shot}) === -${input.traceSummary} -${input.extraContext ? `\n=== EXTRA CONTEXT ===\n${input.extraContext}\n` : ''} -=== VERIFICATION (shot ${input.shot}) === -blendedScore: ${input.verification.blendedScore.toFixed(2)} -allPass: ${input.verification.allPass} -failCount: ${input.verification.failCount} -${failingLayersBlock} - -=== REVIEWER MEMORY === -${summarizeMemory(input.memory)} -${input.trailingContext ? `\n=== TRAILING CONTEXT ===\n${input.trailingContext}\n` : ''} -=== YOUR TASK === -Return STRICT JSON: - -1. observations (20-2000 chars): first-person worker behavior from the trace (tool call counts, errors, loops). -2. diagnosis (20-1500 chars): root cause of current failures, not a restatement of verification. -3. nextShotInstruction (40-3000 chars): concrete "FIX THESE:" directive for the worker's next shot. Reference memory when instructions repeat. -4. shouldContinue (boolean): FALSE if verification.allPass=true, if worker is thrashing, if confidence < 0.3, or if the request looks unachievable. TRUE otherwise. -5. confidence (0-1): self-assessment. - -RULES: -- If verification.allPass is true, shouldContinue MUST be false. -- If memory shows the same failing layer for 2 shots, reduce confidence — strategy isn't working. -- If the trace shows zero tool calls, the worker didn't run — surface that. -- Do NOT re-grade. Direct.` - - return { system, user } -} - -// ─── Default reviewer factory ─────────────────────────────────────────── - -const DEFAULT_SOFT_FAIL: Required = { - observations: 'reviewer soft-failed — no observations captured', - diagnosis: 'reviewer soft-failed — inspect verification findings and retry', - nextShotInstruction: - 'Inspect the verification findings above and address the highest-severity failing layer first. ' + - 'If install failed, start there; otherwise work from the first failing gate and address compilation/build errors before layout/semantic issues.', - shouldContinue: true, - confidence: 0.3, -} - -/** - * Factory: returns a function that invokes the default reviewer against - * an LLM and parses the structured output. Soft-fails to the provided - * defaults on LLM throw or JSON-parse error so the shot loop keeps - * moving rather than crashing. - */ -export function createDefaultReviewer( - options: CreateDefaultReviewerOptions, -): (input: ReviewerPromptInput) => Promise { - const softFail: Required = { - ...DEFAULT_SOFT_FAIL, - ...(options.softFailDefaults ?? {}), - } - const promptBuilder = options.promptBuilder ?? buildReviewerPrompt - const timeoutMs = options.timeoutMs ?? 300_000 - const maxTokens = options.maxTokens ?? 4_000 - const costLedger = options.costLedger ?? new CostLedger() - - return async (input) => { - const start = Date.now() - const { system, user } = promptBuilder(input) - let receipt: CostReceipt | undefined - try { - const request = { - model: options.model, - messages: [ - { role: 'system' as const, content: system }, - { role: 'user' as const, content: user }, - ], - jsonSchema: { name: 'reviewer_output', schema: REVIEWER_SCHEMA }, - temperature: 0, - maxTokens, - timeoutMs, - } satisfies LlmCallRequest - const paid = await costLedger.runPaidCall({ - channel: 'analyst', - phase: options.costPhase ?? 'review', - actor: 'default-reviewer', - model: options.model, - maximumCharge: maximumChargeForLlmRequest(request, options.llm), - signal: options.signal, - execute: (signal, callId) => - callLlmJson<{ - observations: string - diagnosis: string - nextShotInstruction: string - shouldContinue: boolean - confidence: number - }>(request, { - ...options.llm, - signal, - idempotencyKey: callId, - }), - receipt: ({ result }) => costReceiptFromLlm(result), - receiptFromError: costReceiptFromLlmError, - }) - receipt = paid.receipt - if (!paid.succeeded) throw paid.error - const { value } = paid.value - - return { - shot: input.shot, - observations: String(value.observations ?? softFail.observations), - diagnosis: String(value.diagnosis ?? softFail.diagnosis), - nextShotInstruction: String(value.nextShotInstruction ?? softFail.nextShotInstruction), - shouldContinue: Boolean(value.shouldContinue), - confidence: Math.max(0, Math.min(1, Number(value.confidence ?? softFail.confidence))), - costUsd: paid.receipt.costUnknown ? null : paid.receipt.costUsd, - durationMs: Date.now() - start, - available: true, - } - } catch (err) { - return { - shot: input.shot, - observations: softFail.observations, - diagnosis: softFail.diagnosis, - nextShotInstruction: softFail.nextShotInstruction, - shouldContinue: softFail.shouldContinue, - confidence: softFail.confidence, - costUsd: receipt && !receipt.costUnknown ? receipt.costUsd : null, - durationMs: Date.now() - start, - available: false, - error: err instanceof Error ? err.message : String(err), - } - } - } -} diff --git a/src/slo.ts b/src/slo.ts deleted file mode 100644 index 888d2d07..00000000 --- a/src/slo.ts +++ /dev/null @@ -1,131 +0,0 @@ -/** - * SLO gates — quantified pass/fail primitives beyond score thresholds. - * - * Lifted from ADC's sandbox eval suite. Each SLO defines a metric, a - * threshold, and a severity (critical | warning). Critical breaches fail - * the eval; warnings are reported but don't gate CI. Margin is the - * ratio of actual to threshold for histogramming "how close are we?" - * - * Consumers assemble their own SLO arrays; DEFAULT_AGENT_SLOS covers - * the generic agent flow (provision, first token, pass rate, cost). - */ - -export type SloSeverity = 'critical' | 'warning' -export type SloComparator = 'lte' | 'gte' - -export interface Slo { - /** Stable identifier — must be unique within an SLO set. */ - id: string - /** Human description, shown in reports. */ - description: string - /** Metric key looked up in the candidate record. */ - metric: string - /** Whether the metric should stay below (lte) or above (gte) threshold. */ - comparator: SloComparator - /** Threshold value. */ - threshold: number - severity: SloSeverity -} - -export interface SloCheckResult { - slo: Slo - actual: number | undefined - passed: boolean - /** actual/threshold for lte, threshold/actual for gte. >1 means safe margin; <1 means breach. 0 when actual is missing. */ - margin: number - detail: string -} - -export interface SloReport { - results: SloCheckResult[] - passedCritical: boolean - criticalBreaches: SloCheckResult[] - warnings: SloCheckResult[] -} - -/** - * Evaluate an SLO set against a candidate metrics object. Missing metrics - * count as breaches — if you declared it, you must measure it. - */ -export function checkSlos(metrics: Record, slos: Slo[]): SloReport { - const results: SloCheckResult[] = slos.map((slo) => check(slo, metrics[slo.metric])) - const criticalBreaches = results.filter((r) => !r.passed && r.slo.severity === 'critical') - const warnings = results.filter((r) => !r.passed && r.slo.severity === 'warning') - return { results, passedCritical: criticalBreaches.length === 0, criticalBreaches, warnings } -} - -function check(slo: Slo, actual: number | undefined): SloCheckResult { - if (actual === undefined || !Number.isFinite(actual)) { - return { - slo, - actual, - passed: false, - margin: 0, - detail: `metric "${slo.metric}" missing — declared SLOs must be measured`, - } - } - if (slo.comparator === 'lte') { - const passed = actual <= slo.threshold - const margin = slo.threshold === 0 ? (actual === 0 ? Infinity : 0) : slo.threshold / actual - return { - slo, - actual, - passed, - margin, - detail: `${actual} ≤ ${slo.threshold}: ${passed ? 'ok' : 'breach'}`, - } - } - const passed = actual >= slo.threshold - const margin = actual === 0 ? 0 : actual / slo.threshold - return { - slo, - actual, - passed, - margin, - detail: `${actual} ≥ ${slo.threshold}: ${passed ? 'ok' : 'breach'}`, - } -} - -/** Reference SLO set for agent-style evals. Tune per-product by cloning + overriding. */ -export const DEFAULT_AGENT_SLOS: Slo[] = [ - { - id: 'provision_ms', - description: 'Sandbox/session provision under 60s', - metric: 'provisionMs', - comparator: 'lte', - threshold: 60_000, - severity: 'critical', - }, - { - id: 'first_token_ms', - description: 'First token under 15s', - metric: 'firstTokenMs', - comparator: 'lte', - threshold: 15_000, - severity: 'critical', - }, - { - id: 'pass_rate', - description: 'Scenario pass rate ≥ 90%', - metric: 'passRate', - comparator: 'gte', - threshold: 0.9, - severity: 'critical', - }, - { - id: 'cost_usd', - description: 'Per-scenario cost under $0.05', - metric: 'costUsd', - comparator: 'lte', - threshold: 0.05, - severity: 'warning', - }, - { - id: 'overall_score', - description: 'Overall score ≥ 0.7', - metric: 'overallScore', - comparator: 'gte', - threshold: 0.7, - severity: 'critical', - }, -] diff --git a/src/ui-finding.ts b/src/ui-finding.ts deleted file mode 100644 index 0a34f4d1..00000000 --- a/src/ui-finding.ts +++ /dev/null @@ -1,139 +0,0 @@ -/** - * UI audit finding — substrate primitive for "what is wrong with the UI?" - * - * Used by: - * - `@tangle-network/agent-runtime` (ui-auditor profile + delegate) — - * produced as the canonical output of an audit iteration, persisted to - * disk as GitHub-issue Markdown, surfaced over MCP. - * - Downstream ship gates / dashboards / analyst consumers — load and - * transform findings without depending on the runtime. - * - * Repo layering: agent-eval is the substrate (no upward deps). Consumers - * read this type from here; the reverse is forbidden. See CLAUDE.md - * "Repo layering" for the rule. A UI finding makes sense WITHOUT a running - * agent loop (you can load a saved finding, ship-gate against a set of - * them, render them in a dashboard), which puts it firmly in substrate. - * - * The shape is intentionally minimal — runtime-shaped state (capture - * timestamps, OTel trace IDs, sandbox placement) lives on auxiliary - * runtime types in `agent-runtime`, not on the finding itself. - */ - -/** - * Canonical audit lenses. Each lens scopes a finding to a single class of - * problem so a single audit pass can iterate them without pile-on findings - * under a generic label. - * - * Naming is fixed for cross-package wire compatibility. Treat additions as - * a substrate-level decision — analysts, gates, and writers all branch on - * the lens. - */ -export type UiLens = - | 'consistency' - | 'hierarchy' - | 'layout' - | 'ux-flow' - | 'duplication' - | 'accessibility' - | 'responsive' - | 'states' - | 'content' - | 'interaction' - | 'performance-perceived' - | 'other' - -/** Frozen tuple of lenses for validation + iteration. */ -export const UI_LENSES: readonly UiLens[] = [ - 'consistency', - 'hierarchy', - 'layout', - 'ux-flow', - 'duplication', - 'accessibility', - 'responsive', - 'states', - 'content', - 'interaction', - 'performance-perceived', - 'other', -] as const - -/** - * Severity scale — intentionally narrow. - * - * - `critical` — blocks a core task or is an accessibility blocker. - * - `high` — confusing, broken-looking, or noticeable friction. - * - `med` — visible polish issue, would be caught in code review. - * - `low` — nitpick worth fixing eventually. - */ -export type UiFindingSeverity = 'low' | 'med' | 'high' | 'critical' - -/** Frozen severity tuple, ordered worst → least bad for sort/report. */ -export const UI_FINDING_SEVERITIES: readonly UiFindingSeverity[] = [ - 'critical', - 'high', - 'med', - 'low', -] as const - -/** - * Pointer to a screenshot referenced by the finding. The path is - * intentionally a relative string (relative to the audit workspace root) - * so findings remain portable across machines and into GitHub issues. - */ -export interface UiFindingScreenshot { - /** Workspace-relative path to the screenshot file (e.g. `screenshots/home--1280x800--...png`). */ - path: string - /** Optional viewport the screenshot was taken at, e.g. `1280x800`. */ - viewport?: string - /** Optional short label that disambiguates multiple captures of the same surface (e.g. `t0`, `step-1`). */ - label?: string -} - -/** - * A single UI audit finding — the unit of work a contributor can act on. - * - * Every field except the documented optionals is required. The shape is - * deliberately constraining: a finding without a screenshot, a lens, a - * concrete title, and a suggested fix is not actionable, and the auditor - * validator hard-fails on those gaps. - */ -export interface UiFinding { - /** - * Stable identifier within a single audit workspace. Monotonically - * increasing integer (1, 2, …) assigned by the writer when persisting. - * Optional in transit (before persistence) — undefined on freshly minted - * findings emitted from a loop iteration. - */ - id?: number - /** Concrete title — names the offending element AND what's wrong. */ - title: string - /** Lens this finding belongs to. */ - lens: UiLens - /** Severity. */ - severity: UiFindingSeverity - /** Logical route the finding was observed on (e.g. `home`, `checkout-step-2`). */ - route: string - /** Fully qualified URL the finding was observed at. */ - url?: string - /** Viewport string the offending capture was taken at (e.g. `1280x800`). */ - viewport?: string - /** CSS selector pinning the offending element, when one can be identified. */ - selector?: string - /** 1–3 sentences describing what the screenshot shows that is wrong. */ - observation: string - /** Who is affected and how. Concrete user impact. */ - impact: string - /** A specific change a contributor could apply without asking back. */ - suggestedFix: string - /** Optional explicit reproduction steps. Writer synthesizes from route/url/selector when omitted. */ - reproSteps?: string - /** Free-form tags. */ - tags?: readonly string[] - /** Screenshot references — required to be non-empty for actionable findings. */ - screenshots: readonly UiFindingScreenshot[] - /** Cross-references to similar findings already on file, by id. */ - similarTo?: readonly number[] - /** ISO-8601 creation timestamp set by the writer when persisted. */ - createdAt?: string -} diff --git a/src/worker-driver-seed.test.ts b/src/worker-driver-seed.test.ts deleted file mode 100644 index 039871f9..00000000 --- a/src/worker-driver-seed.test.ts +++ /dev/null @@ -1,71 +0,0 @@ -/** - * The worker-driver knowledge is DATA now. These tests pin the same - * contract the deleted `buildWorkerDriverSystemPrompt`'s tests pinned — a - * future edit cannot quietly soften the doctrine or strip a harness brief — - * plus the discriminating assertion that the deleted function stays deleted: - * if the role-as-function path returns to the public surface, this suite - * fails. - */ - -import { describe, expect, it } from 'vitest' -import * as api from './worker-driver-seed' -import { HARNESS_BRIEFS, WORKER_DRIVER_DOCTRINE } from './worker-driver-seed' - -describe('WORKER_DRIVER_DOCTRINE — the driving contract as seed data', () => { - it('demands rich, high-signal instructions and forbids thin steers', () => { - expect(WORKER_DRIVER_DOCTRINE).toMatch(/dense, specific/i) - expect(WORKER_DRIVER_DOCTRINE).toMatch(/thin steer/i) - expect(WORKER_DRIVER_DOCTRINE).toMatch(/out-drive a human/i) - }) - - it('drives the worker to exploit its harness — parallelize, sub-agents, run-to-completion', () => { - expect(WORKER_DRIVER_DOCTRINE).toMatch(/parallel/i) - expect(WORKER_DRIVER_DOCTRINE).toMatch(/sub-agent/i) - expect(WORKER_DRIVER_DOCTRINE).toMatch(/run to completion/i) - }) - - it('requires verification and refuses self-declared completion', () => { - expect(WORKER_DRIVER_DOCTRINE).toMatch(/verif/i) - expect(WORKER_DRIVER_DOCTRINE).toMatch(/never accept "done" without the check/i) - expect(WORKER_DRIVER_DOCTRINE).toMatch(/the deliverable's checker does/i) - }) - - it('reads the worker trace, decomposes, and names the traps each turn', () => { - expect(WORKER_DRIVER_DOCTRINE).toMatch(/not what it claims/i) - expect(WORKER_DRIVER_DOCTRINE).toMatch(/in sequence/i) - expect(WORKER_DRIVER_DOCTRINE).toMatch(/failure modes/i) - }) -}) - -describe('HARNESS_BRIEFS — capability briefs as seed data', () => { - it('covers the conventional harnesses', () => { - for (const harness of ['claude-code', 'codex', 'opencode', 'router-tools']) { - expect(HARNESS_BRIEFS[harness], `brief for ${harness}`).toBeTruthy() - } - }) - - it('claude-code brief names sub-agent fan-out, web access, and MCP', () => { - const brief = HARNESS_BRIEFS['claude-code']! - expect(brief).toMatch(/parallel/i) - expect(brief).toMatch(/sub-agent/i) - expect(brief).toMatch(/WebSearch/i) - expect(brief).toMatch(/MCP/i) - }) - - it('every brief carries a caveat, not just capabilities — no brief is a bare feature list', () => { - for (const [harness, brief] of Object.entries(HARNESS_BRIEFS)) { - expect(brief.length, `brief for ${harness}`).toBeGreaterThan(80) - expect(brief, `brief for ${harness} should mention a limit or a "never/no/not"`).toMatch( - /\b(no|not|never|cannot)\b/i, - ) - } - }) -}) - -describe('role-as-function stays deleted (discriminating)', () => { - it('the package no longer exports buildWorkerDriverSystemPrompt — the knowledge is data', () => { - expect(api).not.toHaveProperty('buildWorkerDriverSystemPrompt') - expect(api).toHaveProperty('WORKER_DRIVER_DOCTRINE') - expect(api).toHaveProperty('HARNESS_BRIEFS') - }) -}) diff --git a/src/worker-driver-seed.ts b/src/worker-driver-seed.ts deleted file mode 100644 index bb641729..00000000 --- a/src/worker-driver-seed.ts +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Worker-driver seed DATA — the knowledge that used to be hardcoded inside - * `buildWorkerDriverSystemPrompt` (deleted; zero callers), re-expressed as - * plain data so a driver PROFILE can seed its directive prompt from it and an - * optimizer (GEPA/DSPy) can then improve on it. A role expressed as a code - * function can never improve; a role expressed as versionable prompt data can - * (tangle-network/agent-runtime#694). - * - * Two exports: - * - `WORKER_DRIVER_DOCTRINE` — the "never write a thin steer" driving - * contract: the bar for every instruction a driver sends a worker. - * - `HARNESS_BRIEFS` — per-harness capability + caveat briefs the driver - * weaves into its instructions so it drives the worker to exploit what - * THAT harness can actually do, and never asks for a capability it lacks. - * - * Both are seeds, not law: a consumer copies them into its directive prompt - * registry (e.g. `prompts//v0.md`) and optimizes from there. - */ - -/** - * The driving contract a worker-driver directive seeds from. Every message the - * driver writes is held to this bar; a thin steer is the driver's failure, not - * the worker's. - */ -export const WORKER_DRIVER_DOCTRINE = `You are a DRIVER: a meta-agent whose entire job is to drive a capable coding worker to ACHIEVE A GOAL by writing it precise, high-signal instructions. You do not do the work yourself — you direct the worker to do it, brilliantly, and you hold it to a standard it would not hold itself to. - -THE BAR — non-negotiable. Every instruction you write is dense, specific, and complete. You write the message a world-class engineering lead writes to a strong report: the exact next objective, the concrete sub-steps, what to do in parallel vs in sequence, what to verify and how, what "done" looks like, and the failure modes to avoid. A thin steer — "try again", "fix the issues", a single vague sentence — is a failure on YOUR part, not the worker's. Out-drive a human power-user. - -HOW to drive (each turn): -1. Read what the worker actually did (its trace/state), not what it claims. Find the real gap between here and the goal. -2. Decide the next objective — the largest correct step the worker can take now. -3. Decompose it: name the sub-tasks that can run IN PARALLEL (independent files/checks/searches) and tell the worker to fan them out (sub-agents / parallel tool calls where its harness allows); name what must run in SEQUENCE and why. -4. Make it exploit the harness: drive it to run to completion under its own autonomy, spawn sub-agents for separable work, use its tools/MCP, and not stop at the first plausible stopping point. -5. Specify verification: the exact command / test / check that proves the step landed, and tell it to run that and report the result — never accept "done" without the check. -6. Name the traps: the specific failure modes for THIS step (editing the wrong file, a check that passes vacuously, scope creep) and forbid them. -7. As the task gets harder, decompose MORE, not less — more parallel branches, deeper sub-agent trees, tighter verification. - -COMPLETION: drive toward the goal's real acceptance check. Do not declare done — the deliverable's checker does. If the worker claims it is done, drive it to PROVE it with the check; if the check fails, drive the fix. - -Output ONLY your next instruction to the worker — direct, detailed, actionable, in the first person as the driver. No meta-commentary, no preamble.` - -/** - * Capability + caveat briefs per harness, keyed by harness id. Free text so - * the substrate stays decoupled from any runtime harness type — the keys are - * the conventional harness ids, but the record accepts any string so a - * consumer can extend it with its own. - */ -export const HARNESS_BRIEFS: Record = { - 'claude-code': [ - '- parallel tool calls in a single turn (batch independent reads/greps/commands)', - '- parallel Task sub-agents (~10 concurrent) for separable investigation or bulk work; sub-agents cannot spawn sub-agents', - '- native WebSearch/WebFetch plus MCP servers when configured', - '- skills, hooks, and slash commands when installed in the workspace', - '- runs to completion under its own autonomy; drive it NOT to stop at the first plausible stopping point', - ].join('\n'), - codex: [ - '- long-horizon autonomous runs (strong run-to-completion; suited to hours-long missions)', - '- sandboxed exec with full shell access; MCP servers when configured', - '- no native sub-agent fan-out: express parallelism as explicit backgrounded commands or sequential decomposition', - '- structured output via --output-schema when a machine-parsed result is needed', - ].join('\n'), - opencode: [ - '- router-backed: any model the router serves; model choice is a live lever', - '- tool calls and MCP servers when configured', - '- no native sub-agent tree; decompose into sequential objectives with explicit verification between them', - ].join('\n'), - 'router-tools': [ - '- a plain tool loop over router chat completions: no autonomy, no sub-agents, no workspace', - '- every capability must be driven explicitly, one tool call at a time', - '- keep objectives small and verification immediate; never assume it will continue on its own', - ].join('\n'), -} diff --git a/src/workspace-inspector.ts b/src/workspace-inspector.ts deleted file mode 100644 index 66a0bc0d..00000000 --- a/src/workspace-inspector.ts +++ /dev/null @@ -1,159 +0,0 @@ -/** - * Workspace inspector — score the persisted state of an agent after a run. - * - * Many evals don't ask "did the response say the right thing" but "did the - * agent put the right rows in the DB / files in the vault / entities on the - * canvas". This is the primitive for that. - * - * Implementations read from D1, KV, filesystem, or any store — the interface - * is deliberately small so consumers plug in their own backends. - */ - -export interface WorkspaceSnapshot { - /** Vault files: logical path → content */ - files: Record - /** DB rows: table name → array of rows (post-validation) */ - rows: Record>> - /** KV entries: key → value (scoped to whatever prefix the inspector chose) */ - kv: Record - /** Free-form blob metadata: for large binaries the inspector stores summary, not bytes */ - blobs?: Record -} - -export interface InspectorContext { - /** Workspace / agent / thread id — whatever the backend uses to scope the snapshot */ - scopeId: string - /** Optional scenario id — allows scenario-specific snapshot shaping */ - scenarioId?: string -} - -export interface WorkspaceInspector { - name: string - snapshot(context: InspectorContext): Promise -} - -// --------------------------------------------------------------------------- -// In-memory inspector — useful for tests, not for production -// --------------------------------------------------------------------------- - -export class InMemoryWorkspaceInspector implements WorkspaceInspector { - readonly name = 'in-memory' - private readonly snapshots = new Map() - - set(scopeId: string, snapshot: WorkspaceSnapshot): void { - this.snapshots.set(scopeId, snapshot) - } - - async snapshot(context: InspectorContext): Promise { - return this.snapshots.get(context.scopeId) ?? { files: {}, rows: {}, kv: {} } - } -} - -// --------------------------------------------------------------------------- -// Snapshot-level assertions -// --------------------------------------------------------------------------- - -export interface WorkspaceAssertion { - name: string - description?: string - check(snapshot: WorkspaceSnapshot): WorkspaceAssertionResult -} - -export interface WorkspaceAssertionResult { - pass: boolean - /** 0..1 — partial credit for assertions that admit it */ - score: number - detail?: string -} - -export function fileExists(path: string): WorkspaceAssertion { - return { - name: `file_exists:${path}`, - check(snapshot) { - const pass = path in snapshot.files - return { - pass, - score: pass ? 1 : 0, - detail: pass ? undefined : `No file at ${path}`, - } - }, - } -} - -export function fileContains(path: string, needle: string): WorkspaceAssertion { - return { - name: `file_contains:${path}:${needle}`, - check(snapshot) { - const content = snapshot.files[path] - if (content === undefined) { - return { pass: false, score: 0, detail: `File ${path} missing` } - } - const pass = content.includes(needle) - return { - pass, - score: pass ? 1 : 0, - detail: pass ? undefined : `File ${path} missing substring "${needle}"`, - } - }, - } -} - -export function rowCount(table: string, min: number, max?: number): WorkspaceAssertion { - return { - name: `row_count:${table}:[${min},${max ?? '∞'}]`, - check(snapshot) { - const rows = snapshot.rows[table] ?? [] - const count = rows.length - const upper = max ?? Infinity - const pass = count >= min && count <= upper - const score = pass ? 1 : count < min ? Math.max(0, count / min) : Math.max(0, upper / count) - return { - pass, - score, - detail: pass - ? undefined - : `Table ${table} has ${count} rows, expected [${min}, ${max ?? '∞'}]`, - } - }, - } -} - -export function rowWhere>( - table: string, - predicate: (row: T) => boolean, - options?: { min?: number }, -): WorkspaceAssertion { - const min = options?.min ?? 1 - return { - name: `row_where:${table}`, - check(snapshot) { - const rows = (snapshot.rows[table] ?? []) as T[] - const matching = rows.filter(predicate).length - const pass = matching >= min - return { - pass, - score: pass ? 1 : Math.max(0, matching / min), - detail: pass - ? undefined - : `Table ${table} has ${matching} matching rows, expected ≥ ${min}`, - } - }, - } -} - -/** Run many assertions; return aggregate pass + mean score + per-assertion details. */ -export function runAssertions( - snapshot: WorkspaceSnapshot, - assertions: WorkspaceAssertion[], -): { - pass: boolean - score: number - results: Array<{ assertion: string; result: WorkspaceAssertionResult }> -} { - const results = assertions.map((a) => ({ assertion: a.name, result: a.check(snapshot) })) - const pass = results.every((r) => r.result.pass) - const score = results.length - ? results.reduce((acc, r) => acc + r.result.score, 0) / results.length - : 1 - return { pass, score, results } -} diff --git a/tests/dual-agent-bench.test.ts b/tests/dual-agent-bench.test.ts deleted file mode 100644 index 7f8dd7bb..00000000 --- a/tests/dual-agent-bench.test.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { DualAgentBench } from '../src/dual-agent-bench' - -describe('DualAgentBench', () => { - it('converges when the critic returns threshold on round N', async () => { - const bench = new DualAgentBench() - const report = await bench.run({ - scenarios: [{ id: 's1', initialPrompt: 'draft a contract' }], - maxRounds: 5, - convergenceThreshold: 0.9, - propose: async ({ roundIndex }) => `proposal round ${roundIndex}`, - critique: async ({ roundIndex }) => ({ - critique: `critique ${roundIndex}`, - // Converge on round 2 (index 2 = third round) - convergenceScore: roundIndex >= 2 ? 1.0 : 0.5, - }), - }) - expect(report.scenarios[0].converged).toBe(true) - expect(report.scenarios[0].roundsToConverge).toBe(3) - expect(report.aggregate.convergenceRate).toBe(1) - }) - - it('records full history — regression: silent loss of intermediate rounds breaks forensics', async () => { - const bench = new DualAgentBench() - const report = await bench.run({ - scenarios: [{ id: 's1', initialPrompt: 'x' }], - maxRounds: 3, - convergenceThreshold: 2, // impossible; forces all rounds to run - propose: async ({ roundIndex }) => `p${roundIndex}`, - critique: async ({ roundIndex }) => ({ - critique: `c${roundIndex}`, - convergenceScore: 0.1, - }), - }) - expect(report.scenarios[0].history).toHaveLength(3) - expect(report.scenarios[0].history.map((r) => r.proposal)).toEqual(['p0', 'p1', 'p2']) - expect(report.scenarios[0].converged).toBe(false) - }) - - it('proposer sees prior critique — regression: proposer ignoring critique means no iteration', async () => { - const bench = new DualAgentBench() - const seenCritiques: (string | undefined)[] = [] - await bench.run({ - scenarios: [{ id: 's1', initialPrompt: 'x' }], - maxRounds: 3, - convergenceThreshold: 2, - propose: async ({ priorCritique }) => { - seenCritiques.push(priorCritique) - return 'proposal' - }, - critique: async ({ roundIndex }) => ({ - critique: `round ${roundIndex} critique`, - convergenceScore: 0.1, - }), - }) - expect(seenCritiques[0]).toBeUndefined() - expect(seenCritiques[1]).toBe('round 0 critique') - expect(seenCritiques[2]).toBe('round 1 critique') - }) - - it('rejects out-of-range convergenceScore — regression: >1 would lock max out of the aggregate', async () => { - const bench = new DualAgentBench() - await expect( - bench.run({ - scenarios: [{ id: 's1', initialPrompt: 'x' }], - propose: async () => 'p', - critique: async () => ({ critique: 'c', convergenceScore: 1.5 }), - }), - ).rejects.toThrow(/\[0,1\]/) - }) - - it('rejects empty scenario list', async () => { - const bench = new DualAgentBench() - await expect( - bench.run({ - scenarios: [], - propose: async () => 'p', - critique: async () => ({ critique: 'c', convergenceScore: 0 }), - }), - ).rejects.toThrow(/at least 1/) - }) - - it('fires onRoundComplete per round', async () => { - const bench = new DualAgentBench() - const events: Array<{ scenarioId: string; round: number }> = [] - await bench.run({ - scenarios: [{ id: 's1', initialPrompt: 'x' }], - maxRounds: 2, - convergenceThreshold: 2, - propose: async () => 'p', - critique: async () => ({ critique: 'c', convergenceScore: 0 }), - onRoundComplete: ({ scenarioId, round }) => - events.push({ scenarioId, round: round.roundIndex }), - }) - expect(events).toEqual([ - { scenarioId: 's1', round: 0 }, - { scenarioId: 's1', round: 1 }, - ]) - }) - - it('aggregate.convergenceRate is the fraction that converged', async () => { - const bench = new DualAgentBench() - const report = await bench.run({ - scenarios: [ - { id: 'a', initialPrompt: 'x' }, - { id: 'b', initialPrompt: 'x' }, - { id: 'c', initialPrompt: 'x' }, - { id: 'd', initialPrompt: 'x' }, - ], - maxRounds: 2, - convergenceThreshold: 0.5, - propose: async () => 'p', - critique: async ({ scenario }) => ({ - critique: 'c', - // a + b converge immediately; c + d never - convergenceScore: scenario.id < 'c' ? 1 : 0, - }), - }) - expect(report.aggregate.convergenceRate).toBe(0.5) - }) -}) diff --git a/tests/golden-matcher.test.ts b/tests/golden-matcher.test.ts deleted file mode 100644 index 9850ffee..00000000 --- a/tests/golden-matcher.test.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { describe, expect, it } from 'vitest' -import type { GoldenSpec } from '../src/golden-matcher' -import { - DEFAULT_SEVERITY_WEIGHTS, - precision as goldenPrecision, - matchGoldens, - weightedRecall, -} from '../src/golden-matcher' - -const goldens: GoldenSpec[] = [ - { id: 'a', severity: 'critical', any: ['primary action', 'no clear primary'], hint: '' }, - { id: 'b', severity: 'major', any: ['equal weight'], hint: '' }, - { id: 'c', severity: 'minor', any: ['next step'], hint: '' }, -] - -describe('matchGoldens', () => { - it('matches against a string-only candidate list', () => { - const r = matchGoldens(goldens, ['No clear PRIMARY ACTION on welcome screen']) - expect(r.matches).toEqual([true, false, false]) - expect(r.hits).toBe(1) - expect(r.total).toBe(3) - }) - - it('default extract concatenates string fields', () => { - const r = matchGoldens(goldens, [ - { description: 'buttons compete', location: 'equal weight grid' }, - ]) - expect(r.matches).toEqual([false, true, false]) - }) - - it('honours custom text() extractor', () => { - const r = matchGoldens(goldens, [{ x: 'PRIMARY ACTION missing' }], { text: (c) => c.x }) - expect(r.matches[0]).toBe(true) - }) - - it('handles regex via anyRegex', () => { - const re: GoldenSpec[] = [ - { id: 'r', severity: 'major', any: [], anyRegex: ['no\\s+primary'], hint: '' }, - ] - const r = matchGoldens(re, ['there is no primary CTA']) - expect(r.matches).toEqual([true]) - }) - - it('returns all-false on empty candidates', () => { - expect(matchGoldens(goldens, []).matches).toEqual([false, false, false]) - }) - - it('skips invalid regex without crashing', () => { - const bad: GoldenSpec[] = [{ id: 'b', severity: 'minor', any: [], anyRegex: ['['], hint: '' }] - expect(() => matchGoldens(bad, ['anything'])).not.toThrow() - expect(matchGoldens(bad, ['anything']).matches).toEqual([false]) - }) -}) - -describe('weightedRecall', () => { - it('weights critical 3x, major 2x, minor 1x', () => { - // total weight = 3 + 2 + 1 = 6 - expect( - weightedRecall(goldens, { matches: [true, false, false], hits: 1, total: 3 }), - ).toBeCloseTo(3 / 6) - expect( - weightedRecall(goldens, { matches: [false, true, false], hits: 1, total: 3 }), - ).toBeCloseTo(2 / 6) - expect(weightedRecall(goldens, { matches: [true, true, true], hits: 3, total: 3 })).toBe(1) - expect(weightedRecall(goldens, { matches: [false, false, false], hits: 0, total: 3 })).toBe(0) - }) - - it('returns 1 when no goldens (vacuous)', () => { - expect(weightedRecall([], { matches: [], hits: 0, total: 0 })).toBe(1) - }) - - it('respects custom weights', () => { - const custom = { ...DEFAULT_SEVERITY_WEIGHTS, critical: 10 } - // weight = 10 + 2 + 1 = 13; hit critical only → 10/13 - expect( - weightedRecall(goldens, { matches: [true, false, false], hits: 1, total: 3 }, custom), - ).toBeCloseTo(10 / 13) - }) -}) - -describe('goldenPrecision', () => { - it('returns 1 when no candidates', () => { - expect(goldenPrecision(goldens, [])).toBe(1) - }) - - it('counts the share that match a golden phrase', () => { - const cands = [ - 'Primary action unclear', - 'Some unrelated polish nit', - 'No clear primary visible', - ] - expect(goldenPrecision(goldens, cands)).toBeCloseTo(2 / 3) - }) - - it('honours regex goldens for precision too', () => { - const re: GoldenSpec[] = [ - { id: 'r', severity: 'major', any: [], anyRegex: ['p[a-z]+y action'], hint: '' }, - ] - expect(goldenPrecision(re, ['primary action found', 'noise'])).toBeCloseTo(1 / 2) - }) -}) diff --git a/tests/judge-runner.test.ts b/tests/judge-runner.test.ts deleted file mode 100644 index 53999bb2..00000000 --- a/tests/judge-runner.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { compilerJudge, JudgeRunner, runJudgeFleet, testJudge } from '../src/judge-runner' -import type { HarnessConfig, SandboxDriver, SandboxResult } from '../src/sandbox-harness' - -class FakeDriver implements SandboxDriver { - id = 'fake' - async exec( - phase: SandboxResult['phase'], - _command: string, - _config: HarnessConfig, - ): Promise { - return { - phase, - exitCode: 0, - stdout: phase === 'test' ? 'Tests 4 passed' : '', - stderr: '', - wallMs: 5, - ...(phase === 'test' ? { testsTotal: 4, testsPassed: 4 } : {}), - } - } -} - -describe('judge runner', () => { - it('runs named judges through the sandbox harness', async () => { - const runner = new JudgeRunner(new FakeDriver()) - const result = await runner.run(testJudge('tests', { testCommand: 'pnpm test' })) - expect(result.kind).toBe('test') - expect(result.passed).toBe(true) - expect(result.score).toBe(1) - }) - - it('runs a judge fleet in parallel by default', async () => { - const results = await runJudgeFleet( - [ - compilerJudge('compile', { runCommand: 'pnpm build' }), - testJudge('tests', { testCommand: 'pnpm test' }), - ], - { driver: new FakeDriver() }, - ) - expect(results).toHaveLength(2) - expect(results.every((result) => result.passed)).toBe(true) - }) -}) diff --git a/tests/ui-finding.test.ts b/tests/ui-finding.test.ts deleted file mode 100644 index 2021f57b..00000000 --- a/tests/ui-finding.test.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { - UI_FINDING_SEVERITIES, - UI_LENSES, - type UiFinding, - type UiFindingSeverity, - type UiLens, -} from '../src/ui-finding' - -describe('UI_LENSES', () => { - it('matches the UiLens union exhaustively', () => { - // The runtime tuple and the type must stay in sync — if a lens is added - // to the union without updating the tuple, a lens-keyed Record will - // silently lose dimensions in downstream code. Enforce both directions. - const witness: Record = { - consistency: true, - hierarchy: true, - layout: true, - 'ux-flow': true, - duplication: true, - accessibility: true, - responsive: true, - states: true, - content: true, - interaction: true, - 'performance-perceived': true, - other: true, - } - for (const lens of UI_LENSES) { - expect(witness[lens]).toBe(true) - } - expect(Object.keys(witness).sort()).toEqual([...UI_LENSES].sort()) - }) - - it('contains no duplicates', () => { - expect(new Set(UI_LENSES).size).toBe(UI_LENSES.length) - }) -}) - -describe('UI_FINDING_SEVERITIES', () => { - it('orders severities worst → least bad', () => { - expect(UI_FINDING_SEVERITIES).toEqual(['critical', 'high', 'med', 'low']) - }) - - it('matches the UiFindingSeverity union exhaustively', () => { - const witness: Record = { - critical: true, - high: true, - med: true, - low: true, - } - expect(Object.keys(witness).sort()).toEqual([...UI_FINDING_SEVERITIES].sort()) - }) -}) - -describe('UiFinding shape', () => { - it('accepts a minimally complete finding', () => { - const finding: UiFinding = { - title: 'Primary CTA invisible on hover state', - lens: 'interaction', - severity: 'med', - route: 'home', - observation: 'CTA background blends into card background on hover.', - impact: 'Users lose the affordance during the moment they need it most.', - suggestedFix: 'Add 2px border on hover instead of swapping background.', - screenshots: [{ path: 'screenshots/home--cta--hover.png', viewport: '1280x800' }], - } - expect(finding.title.length).toBeGreaterThan(0) - expect(finding.screenshots.length).toBe(1) - }) - - it('treats `screenshots` and `tags` as readonly arrays', () => { - const finding: UiFinding = { - title: 't', - lens: 'consistency', - severity: 'low', - route: 'r', - observation: 'o', - impact: 'i', - suggestedFix: 's', - screenshots: [{ path: 'a.png' }], - tags: ['nav', 'header'], - } - // @ts-expect-error — screenshots is readonly. - finding.screenshots.push({ path: 'b.png' }) - // @ts-expect-error — tags is readonly. - finding.tags?.push('extra') - }) -}) diff --git a/tests/workspace-inspector.test.ts b/tests/workspace-inspector.test.ts deleted file mode 100644 index 1a68a01b..00000000 --- a/tests/workspace-inspector.test.ts +++ /dev/null @@ -1,112 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { - fileContains, - fileExists, - InMemoryWorkspaceInspector, - rowCount, - rowWhere, - runAssertions, - type WorkspaceSnapshot, -} from '../src/workspace-inspector' - -function snap(partial: Partial = {}): WorkspaceSnapshot { - return { files: {}, rows: {}, kv: {}, ...partial } -} - -describe('InMemoryWorkspaceInspector', () => { - it('returns the snapshot set for a scope', async () => { - const insp = new InMemoryWorkspaceInspector() - insp.set('workspace_1', snap({ files: { 'vault/a.md': 'hi' } })) - const s = await insp.snapshot({ scopeId: 'workspace_1' }) - expect(s.files).toEqual({ 'vault/a.md': 'hi' }) - }) - - it('returns empty shape for unknown scope — regression: undefined would break downstream assertions', async () => { - const insp = new InMemoryWorkspaceInspector() - const s = await insp.snapshot({ scopeId: 'missing' }) - expect(s).toEqual({ files: {}, rows: {}, kv: {} }) - }) -}) - -describe('fileExists / fileContains', () => { - it('fileExists passes when file present', () => { - const r = fileExists('a.md').check(snap({ files: { 'a.md': 'x' } })) - expect(r.pass).toBe(true) - expect(r.score).toBe(1) - }) - it('fileExists fails when absent', () => { - const r = fileExists('a.md').check(snap()) - expect(r.pass).toBe(false) - expect(r.detail).toContain('a.md') - }) - it('fileContains passes on substring hit', () => { - const r = fileContains('a.md', 'hello').check(snap({ files: { 'a.md': 'well hello there' } })) - expect(r.pass).toBe(true) - }) - it('fileContains fails with a descriptive detail when the substring is missing', () => { - const r = fileContains('a.md', 'foo').check(snap({ files: { 'a.md': 'bar' } })) - expect(r.pass).toBe(false) - expect(r.detail).toMatch(/missing substring/) - }) -}) - -describe('rowCount', () => { - it('passes inside range', () => { - const r = rowCount('deals', 1, 10).check(snap({ rows: { deals: [{}, {}, {}] } })) - expect(r.pass).toBe(true) - expect(r.score).toBe(1) - }) - it('partial credit below min — regression: binary pass/fail loses tuning signal', () => { - const r = rowCount('deals', 10).check(snap({ rows: { deals: [{}, {}, {}] } })) - expect(r.pass).toBe(false) - expect(r.score).toBeCloseTo(0.3, 2) - }) - it('partial credit above max', () => { - const r = rowCount('deals', 1, 10).check(snap({ rows: { deals: new Array(20).fill({}) } })) - expect(r.pass).toBe(false) - expect(r.score).toBe(0.5) - }) - it('treats missing table as 0 rows', () => { - const r = rowCount('missing', 1).check(snap()) - expect(r.pass).toBe(false) - }) -}) - -describe('rowWhere', () => { - it('passes when predicate matches enough rows', () => { - const r = rowWhere<{ status: string }>('deals', (row) => row.status === 'closed', { - min: 2, - }).check( - snap({ rows: { deals: [{ status: 'open' }, { status: 'closed' }, { status: 'closed' }] } }), - ) - expect(r.pass).toBe(true) - }) - it('partial credit when min not met', () => { - const r = rowWhere<{ status: string }>('deals', (row) => row.status === 'closed', { - min: 3, - }).check(snap({ rows: { deals: [{ status: 'closed' }] } })) - expect(r.pass).toBe(false) - expect(r.score).toBeCloseTo(1 / 3, 2) - }) -}) - -describe('runAssertions', () => { - it('aggregates pass as AND and score as mean', () => { - const result = runAssertions( - snap({ files: { 'a.md': 'content' }, rows: { deals: [{}, {}, {}] } }), - [fileExists('a.md'), rowCount('deals', 1, 10)], - ) - expect(result.pass).toBe(true) - expect(result.score).toBe(1) - expect(result.results).toHaveLength(2) - }) - - it("aggregate pass is AND across assertions — regression: one fail doesn't sink the aggregate pass", () => { - const result = runAssertions(snap({ files: { 'a.md': 'content' } }), [ - fileExists('a.md'), - fileExists('missing.md'), - ]) - expect(result.pass).toBe(false) - expect(result.score).toBe(0.5) - }) -})