From 2bdb3bb1c9546643ca2aca836c8280dea08a962f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 16:07:46 +0000 Subject: [PATCH 1/7] feat(evals): semantic eval harness for bundled schemas Fixture spaces labelled with the verdict each node should get per rule, a live runner that reports agreement, stability, probability on the expected verdicts, reserved verdicts and errors per provider and rule, and free tests that keep the labels complete and consistent. Starts with a strict OST space for opportunity-customer-voice. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019Xphpdioyw95tAT6PEKGTU --- AGENTS.md | 2 + evals/semantic/README.md | 52 +++++ evals/semantic/harness.ts | 208 ++++++++++++++++++ evals/semantic/run.ts | 89 ++++++++ evals/semantic/strict-ost/eval.json5 | 24 ++ .../Add a progress bar to workspace setup.md | 8 + .../space/Build a Slack integration.md | 8 + ... tell which plan fits a team of my size.md | 8 + ...know whether my changes have been saved.md | 8 + ...t my teammates changed while I was away.md | 8 + ...I'll lose my work if my browser crashes.md | 8 + ...ncrease trial-to-paid conversion by 10%.md | 8 + .../space/Increase weekly active teams.md | 7 + ...users find the editor toolbar confusing.md | 8 + .../space/Offer a discount on annual plans.md | 8 + evals/semantic/strict-ost/space/Onboarding.md | 8 + .../space/Reduce churn among small teams.md | 8 + ... workspace takes so long that I give up.md | 8 + .../space/Show unsaved-change indicator.md | 7 + package.json | 1 + tests/evals/semantic.test.ts | 95 ++++++++ tsconfig.json | 10 +- 22 files changed, 590 insertions(+), 1 deletion(-) create mode 100644 evals/semantic/README.md create mode 100644 evals/semantic/harness.ts create mode 100644 evals/semantic/run.ts create mode 100644 evals/semantic/strict-ost/eval.json5 create mode 100644 evals/semantic/strict-ost/space/Add a progress bar to workspace setup.md create mode 100644 evals/semantic/strict-ost/space/Build a Slack integration.md create mode 100644 evals/semantic/strict-ost/space/I can't tell which plan fits a team of my size.md create mode 100644 evals/semantic/strict-ost/space/I don't know whether my changes have been saved.md create mode 100644 evals/semantic/strict-ost/space/I want to see what my teammates changed while I was away.md create mode 100644 evals/semantic/strict-ost/space/I'm afraid I'll lose my work if my browser crashes.md create mode 100644 evals/semantic/strict-ost/space/Increase trial-to-paid conversion by 10%.md create mode 100644 evals/semantic/strict-ost/space/Increase weekly active teams.md create mode 100644 evals/semantic/strict-ost/space/New users find the editor toolbar confusing.md create mode 100644 evals/semantic/strict-ost/space/Offer a discount on annual plans.md create mode 100644 evals/semantic/strict-ost/space/Onboarding.md create mode 100644 evals/semantic/strict-ost/space/Reduce churn among small teams.md create mode 100644 evals/semantic/strict-ost/space/Setting up a workspace takes so long that I give up.md create mode 100644 evals/semantic/strict-ost/space/Show unsaved-change indicator.md create mode 100644 tests/evals/semantic.test.ts diff --git a/AGENTS.md b/AGENTS.md index 6f02776..36cb3a8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,6 +45,7 @@ There are several places that need reviewing and updating with any new feature o |---|---| | CLI command, flag or output | `docs/*` for the command; `plugin/skills/structured-context/references/commands.md`; the command list in `plugin/skills/structured-context/SKILL.md`; `plugin/commands/*` | | Rule, schema or `$metadata` shape | `docs/schemas.md`, `docs/rules.md`, `docs/semantic-rules.md`; `references/schema-authoring.md`, `references/schema-design.md`; bundled `schemas/` | +| Bundled semantic rule | its eval cases in `evals/semantic/` | | User-visible error or warning message | the troubleshooting table in `SKILL.md` | | Config key or space config | `docs/config.md`; the config example in `references/commands.md` | | `validate --json` output | `plugin/scripts/state.ts` (the hooks parse it), `scripts/mod-contract.ts` and `bun run test:hook` | @@ -99,6 +100,7 @@ For most development only the main unit tests need re-running regularly. - `bun run test:package` — builds and packs the package, installs the tarball, and runs the installed CLI. Runs in CI and before every version bump; run it after changing the build, `tsconfig*.json` or `files`. - `bun run test:judge` — live judge tests (`tests/judge/live.test.ts`): one real provider call per provider whose API key is set. Costs money; use sparingly when changing judges or semantic rule behaviour. - `bun run test:models` — check that every bundled provider's configured model exists (no API keys) +- `bun run eval:semantic --provider ` — semantic evals (`evals/semantic/`): how often a provider's verdicts match labelled fixture spaces for the bundled schemas' semantic rules, and how stable they are. Costs money; run it when changing a bundled semantic rule, the schema descriptions it draws on, or a judge. ## Debugging diff --git a/evals/semantic/README.md b/evals/semantic/README.md new file mode 100644 index 0000000..bd33d8f --- /dev/null +++ b/evals/semantic/README.md @@ -0,0 +1,52 @@ +# Semantic evals + +Semantic evals measure how well model providers judge the semantic rules of the bundled schemas. Each eval space is a small fixture space with a label per node and rule: the verdict the node should get. + +## Run the evals + +The run calls real providers, so it costs money. Name each provider to run; config `modelProviders` entries work as well as bundled providers. + +```bash +bun run eval:semantic --provider openrouter-jev --provider anthropic +``` + +| Flag | Purpose | +|---|---| +| `--provider ` | The provider to judge with. Repeat it, or give a comma-separated list, to compare providers | +| `--space ` | Run only this eval space. Repeatable | +| `--runs ` | Judge each space this many times, to measure stability. Default: 3 | +| `--per-node` | Send one request per node, even to a provider that packs several nodes into one request | +| `--json` | Print every judgement and score as JSON instead of the report | + +The report has one row per space, provider and rule: + +| Column | Meaning | +|---|---| +| Agreement | Share of judgements, over every case and run, that match the label | +| Stable | Share of cases that got the same verdict on every run | +| p(expected) | Mean probability the provider gave the labelled verdicts. Only decision models report it | +| Reserved | Judgements with a reserved verdict, such as `missing-context` | +| Errors | Cases that got an evaluation error, or no judgement, on some run | + +Each run validates the space through the same pipeline as `sctx validate --semantic`, with only the labelled rules, and evaluates gated nodes too. + +## Add a case + +An eval space is a directory here with a `space/` directory of nodes and an `eval.json5`: + +```json5 +{ + schema: 'strict_ost.json', + expected: { + 'opportunity-customer-voice': { + 'Build a Slack integration.md': 'fail', + // A list accepts any of its verdicts. + 'New users find the editor toolbar confusing.md': ['pass', 'fail'], + }, + }, +} +``` + +`expected` maps a rule id to node ids (as `sctx dump` shows them) and their verdicts. Label every node a labelled rule judges. Write a comment on a case whose label is not obvious. + +`bun run test` checks each eval space for free: the space is valid, every label names a real node, semantic rule and verdict, and every node a labelled rule judges has a label. diff --git a/evals/semantic/harness.ts b/evals/semantic/harness.ts new file mode 100644 index 0000000..0fd50aa --- /dev/null +++ b/evals/semantic/harness.ts @@ -0,0 +1,208 @@ +import { readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import JSON5 from 'json5'; +import type { Config } from '../../src/config'; +import type { SemanticJudgeProvider, SemanticJudgeResult } from '../../src/judge/types'; +import { resolveSemanticRule } from '../../src/judge/verdicts'; +import { readSpace } from '../../src/read/read-space'; +import { RESERVED_VERDICTS } from '../../src/schema/metadata-contract'; +import { createSpaceContext } from '../../src/space-context'; +import type { SpaceContext } from '../../src/types'; +import { validateNodes } from '../../src/validate'; + +export const EVALS_DIR = import.meta.dir; + +/** Rule id → node id → the verdicts that count as agreeing. */ +export type Expected = Record>; + +/** A fixture space and the verdict each of its nodes should get per rule. */ +export interface EvalSpace { + name: string; + context: SpaceContext; + expected: Expected; +} + +/** One verdict (or error) the judge gave for a node and rule. */ +export interface Judgement { + nodeId: string; + ruleId: string; + result: SemanticJudgeResult; +} + +type EvalFile = { schema: string; expected: Record> }; + +/** The eval spaces under `root`: each directory with an `eval.json5` beside a `space/` directory. */ +export function loadEvalSpaces(root: string = EVALS_DIR, only?: string[]): EvalSpace[] { + const names = readdirSync(root, { withFileTypes: true }) + .filter((d) => d.isDirectory() && (!only || only.includes(d.name))) + .map((d) => d.name) + .sort(); + const unknown = only?.filter((name) => !names.includes(name)) ?? []; + if (unknown.length > 0) throw new Error(`Unknown eval space: ${unknown.join(', ')}. Known: ${names.join(', ')}`); + return names.map((name) => { + const dir = join(root, name); + const file = JSON5.parse(readFileSync(join(dir, 'eval.json5'), 'utf8')) as EvalFile; + const config: Config = { spaces: [{ name, path: join(dir, 'space'), schema: file.schema }] }; + const expected: Expected = {}; + for (const [ruleId, nodes] of Object.entries(file.expected)) { + expected[ruleId] = Object.fromEntries( + Object.entries(nodes).map(([id, v]) => [id, typeof v === 'string' ? [v] : v]), + ); + } + return { name, context: createSpaceContext(name, config, { configDir: dir }), expected }; + }); +} + +/** + * Problems that would make an eval space's scores meaningless: an invalid space (gating and schema failures + * change what is judged), or a label naming an unknown node, a rule with no semantic part, or a verdict the + * rule cannot return. Checked before any judge call. + */ +export async function checkEvalSpace(space: EvalSpace): Promise { + const problems: string[] = []; + const read = await readSpace(space.context); + const result = await validateNodes(space.context, read.nodes, read); + for (const issue of result.issues) { + if (issue.severity === 'error') problems.push(`${issue.nodeId ?? ''}: ${issue.message}`); + } + const ids = new Set(read.nodes.map((n) => n.id as string)); + const rules = space.context.schema.metadata.rules ?? []; + for (const [ruleId, labels] of Object.entries(space.expected)) { + const rule = rules.find((r) => r.id === ruleId); + if (!rule?.semantic) { + problems.push(`rule "${ruleId}": not a semantic rule of the schema`); + continue; + } + const verdicts = Object.keys(resolveSemanticRule(rule).spec.verdicts); + for (const [nodeId, expected] of Object.entries(labels)) { + if (!ids.has(nodeId)) problems.push(`rule "${ruleId}": unknown node "${nodeId}"`); + for (const verdict of expected) { + if (!verdicts.includes(verdict)) { + problems.push(`rule "${ruleId}", node "${nodeId}": "${verdict}" is not a verdict (${verdicts.join(', ')})`); + } + } + } + } + return problems.map((p) => `${space.name}: ${p}`); +} + +/** + * Validate the space with its labelled rules through the real pipeline, recording every result the provider + * returns, passing verdicts included. Gated nodes are judged too. `perNode` hides `evaluateNodes`, so a + * provider that packs nodes into one request gets one request per node instead. + */ +export async function judgeEvalSpace( + space: EvalSpace, + provider: SemanticJudgeProvider, + options: { perNode?: boolean } = {}, +): Promise { + const judgements: Judgement[] = []; + const record = ( + input: Parameters[0], + results: Record, + ) => { + for (const rule of input.rules) { + const result = results[rule.id] ?? { error: 'no result for this rule' }; + judgements.push({ nodeId: input.node.id, ruleId: rule.id, result }); + } + }; + const recording: SemanticJudgeProvider = { + name: provider.name, + async evaluate(input) { + const results = await provider.evaluate(input); + record(input, results); + return results; + }, + }; + const { evaluateNodes } = provider; + if (evaluateNodes && !options.perNode) { + recording.evaluateNodes = async (inputs) => { + const results = await evaluateNodes.call(provider, inputs); + for (const [i, input] of inputs.entries()) record(input, results[i] ?? {}); + return results; + }; + } + const read = await readSpace(space.context); + await validateNodes(space.context, read.nodes, read, { + includeGated: true, + ruleFilter: { id: Object.keys(space.expected) }, + judge: { provider: recording }, + }); + return judgements; +} + +/** How well a provider's verdicts on one rule match the labels, over every run. */ +export interface RuleScore { + ruleId: string; + cases: number; + runs: number; + /** Share of judgements, over cases and runs, whose verdict is an expected one. */ + agreement: number; + /** Share of cases that got the same verdict on every run, with no errors. */ + stable: number; + /** Mean probability given to the expected verdicts, over judgements that report probabilities. */ + expectedProbability?: number; + /** Judgements with a reserved verdict, such as `missing-context`. */ + reserved: number; + /** Cases that got an error, or no judgement, on some run. */ + errors: number; + /** Cases where some run disagreed with the label, with the verdict (or error) each run gave. */ + misses: Array<{ nodeId: string; expected: string[]; got: string[]; evidence?: string }>; + /** Nodes judged on this rule with no label. */ + unlabelled: string[]; +} + +const describe = (result: SemanticJudgeResult | undefined) => + !result ? 'not judged' : 'error' in result ? `error: ${result.error}` : result.verdict; + +/** Score each labelled rule over the runs of one eval space with one provider. */ +export function scoreEvalSpace(space: EvalSpace, runs: Judgement[][]): RuleScore[] { + const reserved: readonly string[] = RESERVED_VERDICTS; + return Object.entries(space.expected).map(([ruleId, labels]) => { + const perRun = runs.map((run) => new Map(run.filter((j) => j.ruleId === ruleId).map((j) => [j.nodeId, j.result]))); + let agreeing = 0; + let stable = 0; + let errors = 0; + let reservedCount = 0; + const probabilities: number[] = []; + const misses: RuleScore['misses'] = []; + for (const [nodeId, expected] of Object.entries(labels)) { + const results = perRun.map((run) => run.get(nodeId)); + const verdicts = results.map((r) => (r && !('error' in r) ? r.verdict : undefined)); + agreeing += verdicts.filter((v) => v !== undefined && expected.includes(v)).length; + reservedCount += verdicts.filter((v) => v !== undefined && reserved.includes(v)).length; + if (verdicts.includes(undefined)) errors++; + else if (new Set(verdicts).size === 1) stable++; + for (const r of results) { + if (r && !('error' in r) && r.probabilities) { + probabilities.push(expected.reduce((sum, v) => sum + (r.probabilities?.[v] ?? 0), 0)); + } + } + if (verdicts.some((v) => v === undefined || !expected.includes(v))) { + const miss = results.find((r) => !r || 'error' in r || !expected.includes(r.verdict)); + misses.push({ + nodeId, + expected, + got: results.map(describe), + ...(miss && !('error' in miss) && miss.evidence !== undefined && { evidence: miss.evidence }), + }); + } + } + const cases = Object.keys(labels).length; + const unlabelled = new Set(perRun.flatMap((run) => [...run.keys()]).filter((id) => !(id in labels))); + return { + ruleId, + cases, + runs: runs.length, + agreement: cases && runs.length ? agreeing / (cases * runs.length) : 0, + stable: cases ? stable / cases : 0, + ...(probabilities.length > 0 && { + expectedProbability: probabilities.reduce((a, b) => a + b, 0) / probabilities.length, + }), + reserved: reservedCount, + errors, + misses, + unlabelled: [...unlabelled].sort(), + }; + }); +} diff --git a/evals/semantic/run.ts b/evals/semantic/run.ts new file mode 100644 index 0000000..b34ff6d --- /dev/null +++ b/evals/semantic/run.ts @@ -0,0 +1,89 @@ +/** + * Live semantic evals: judge the eval spaces with real providers and report agreement with the labels. + * Costs money. See README.md. + */ +import { parseArgs } from 'node:util'; +import { loadConfigOrEmpty } from '../../src/config'; +import { createJudgeProvider } from '../../src/judge/factory'; +import { + checkEvalSpace, + type Judgement, + judgeEvalSpace, + loadEvalSpaces, + type RuleScore, + scoreEvalSpace, +} from './harness'; + +const { values } = parseArgs({ + options: { + provider: { type: 'string', multiple: true }, + space: { type: 'string', multiple: true }, + runs: { type: 'string', default: '3' }, + 'per-node': { type: 'boolean', default: false }, + json: { type: 'boolean', default: false }, + }, +}); + +const providerNames = (values.provider ?? []).flatMap((p) => p.split(',')).filter(Boolean); +const runs = Number(values.runs); +if (providerNames.length === 0) throw new Error('Name at least one model provider with --provider .'); +if (!Number.isInteger(runs) || runs < 1) { + throw new Error(`--runs must be a positive whole number, not "${values.runs}".`); +} + +// Build every provider and check every space before the first paid call. +const config = loadConfigOrEmpty(); +const providers = providerNames.map((name) => createJudgeProvider(config, {}, name)!); +const spaces = loadEvalSpaces(undefined, values.space); +const problems = (await Promise.all(spaces.map(checkEvalSpace))).flat(); +if (problems.length > 0) throw new Error(`Eval spaces have problems:\n${problems.join('\n')}`); + +const results: Array<{ space: string; provider: string; scores: RuleScore[]; runs: Judgement[][] }> = []; +for (const space of spaces) { + for (const [i, provider] of providers.entries()) { + const judged: Judgement[][] = []; + for (let run = 0; run < runs; run++) { + judged.push(await judgeEvalSpace(space, provider, { perNode: values['per-node'] })); + } + results.push({ + space: space.name, + provider: providerNames[i]!, + scores: scoreEvalSpace(space, judged), + runs: judged, + }); + } +} + +if (values.json) { + console.log(JSON.stringify(results, null, 2)); +} else { + const pct = (n: number | undefined) => (n === undefined ? '-' : `${Math.round(n * 100)}%`); + const lines = [ + `Semantic evals: ${runs} run(s) per provider${values['per-node'] ? ', one request per node' : ''}`, + '', + '| Space | Provider | Rule | Cases | Agreement | Stable | p(expected) | Reserved | Errors |', + '|---|---|---|---|---|---|---|---|---|', + ]; + for (const { space, provider, scores } of results) { + for (const s of scores) { + lines.push( + `| ${space} | ${provider} | ${s.ruleId} | ${s.cases} | ${pct(s.agreement)} | ${runs > 1 ? pct(s.stable) : '-'} | ${pct(s.expectedProbability)} | ${s.reserved} | ${s.errors} |`, + ); + } + } + const notes: string[] = []; + for (const { space, provider, scores } of results) { + for (const s of scores) { + for (const m of s.misses) { + notes.push( + `- ${space} · ${provider} · ${s.ruleId} · ${m.nodeId}: expected ${m.expected.join(' or ')}, got ${m.got.join(', ')}`, + ); + if (m.evidence) notes.push(` ${m.evidence}`); + } + if (s.unlabelled.length > 0) + notes.push(`- ${space} · ${provider} · ${s.ruleId}: judged with no label: ${s.unlabelled.join(', ')}`); + } + } + if (notes.length > 0) lines.push('', 'Misses:', ...notes); + console.log(lines.join('\n')); +} diff --git a/evals/semantic/strict-ost/eval.json5 b/evals/semantic/strict-ost/eval.json5 new file mode 100644 index 0000000..835d431 --- /dev/null +++ b/evals/semantic/strict-ost/eval.json5 @@ -0,0 +1,24 @@ +{ + schema: 'strict_ost.json', + // Rule id → node id → the verdict the node should get. A list accepts any of its verdicts. + expected: { + 'opportunity-customer-voice': { + 'Setting up a workspace takes so long that I give up.md': 'pass', + "I can't tell which plan fits a team of my size.md": 'pass', + "I'm afraid I'll lose my work if my browser crashes.md": 'pass', + "I don't know whether my changes have been saved.md": 'pass', + 'I want to see what my teammates changed while I was away.md': 'pass', + // A solution in disguise. + 'Add a progress bar to workspace setup.md': 'fail', + 'Build a Slack integration.md': 'fail', + 'Offer a discount on annual plans.md': 'fail', + // A business metric. + 'Increase trial-to-paid conversion by 10%.md': 'fail', + 'Reduce churn among small teams.md': 'fail', + // A topic, not a need. + 'Onboarding.md': 'fail', + // A real pain point, but in the third person rather than the customer's own words. + 'New users find the editor toolbar confusing.md': ['pass', 'fail'], + }, + }, +} diff --git a/evals/semantic/strict-ost/space/Add a progress bar to workspace setup.md b/evals/semantic/strict-ost/space/Add a progress bar to workspace setup.md new file mode 100644 index 0000000..c7e28b9 --- /dev/null +++ b/evals/semantic/strict-ost/space/Add a progress bar to workspace setup.md @@ -0,0 +1,8 @@ +--- +type: opportunity +status: identified +parent: "[[Increase weekly active teams]]" +source: Product team brainstorm +--- + +Show users how many setup steps remain. diff --git a/evals/semantic/strict-ost/space/Build a Slack integration.md b/evals/semantic/strict-ost/space/Build a Slack integration.md new file mode 100644 index 0000000..98db002 --- /dev/null +++ b/evals/semantic/strict-ost/space/Build a Slack integration.md @@ -0,0 +1,8 @@ +--- +type: opportunity +status: identified +parent: "[[Increase weekly active teams]]" +source: Competitor review +--- + +Post page updates to a Slack channel. diff --git a/evals/semantic/strict-ost/space/I can't tell which plan fits a team of my size.md b/evals/semantic/strict-ost/space/I can't tell which plan fits a team of my size.md new file mode 100644 index 0000000..10e8fd9 --- /dev/null +++ b/evals/semantic/strict-ost/space/I can't tell which plan fits a team of my size.md @@ -0,0 +1,8 @@ +--- +type: opportunity +status: identified +parent: "[[Increase weekly active teams]]" +source: Sales call notes, 2026-09-10 +--- + +Prospects asked which tier covers twelve seats with guest access. diff --git a/evals/semantic/strict-ost/space/I don't know whether my changes have been saved.md b/evals/semantic/strict-ost/space/I don't know whether my changes have been saved.md new file mode 100644 index 0000000..866a1fd --- /dev/null +++ b/evals/semantic/strict-ost/space/I don't know whether my changes have been saved.md @@ -0,0 +1,8 @@ +--- +type: opportunity +status: identified +parent: "[[I'm afraid I'll lose my work if my browser crashes]]" +source: Usability test round 3 +--- + +Four of six participants pressed the save shortcut repeatedly. diff --git a/evals/semantic/strict-ost/space/I want to see what my teammates changed while I was away.md b/evals/semantic/strict-ost/space/I want to see what my teammates changed while I was away.md new file mode 100644 index 0000000..bbe2c6e --- /dev/null +++ b/evals/semantic/strict-ost/space/I want to see what my teammates changed while I was away.md @@ -0,0 +1,8 @@ +--- +type: opportunity +status: identified +parent: "[[Increase weekly active teams]]" +source: Interview with Tomas, 2026-09-15 +--- + +Tomas reads every page again after a holiday to find what is new. diff --git a/evals/semantic/strict-ost/space/I'm afraid I'll lose my work if my browser crashes.md b/evals/semantic/strict-ost/space/I'm afraid I'll lose my work if my browser crashes.md new file mode 100644 index 0000000..d806b16 --- /dev/null +++ b/evals/semantic/strict-ost/space/I'm afraid I'll lose my work if my browser crashes.md @@ -0,0 +1,8 @@ +--- +type: opportunity +status: identified +parent: "[[Increase weekly active teams]]" +source: Support tickets, September 2026 +--- + +Several tickets describe retyping long notes after a crash. diff --git a/evals/semantic/strict-ost/space/Increase trial-to-paid conversion by 10%.md b/evals/semantic/strict-ost/space/Increase trial-to-paid conversion by 10%.md new file mode 100644 index 0000000..621c236 --- /dev/null +++ b/evals/semantic/strict-ost/space/Increase trial-to-paid conversion by 10%.md @@ -0,0 +1,8 @@ +--- +type: opportunity +status: identified +parent: "[[Increase weekly active teams]]" +source: Quarterly planning +--- + +The board wants conversion to rise this quarter. diff --git a/evals/semantic/strict-ost/space/Increase weekly active teams.md b/evals/semantic/strict-ost/space/Increase weekly active teams.md new file mode 100644 index 0000000..ef84cd0 --- /dev/null +++ b/evals/semantic/strict-ost/space/Increase weekly active teams.md @@ -0,0 +1,7 @@ +--- +type: outcome +status: identified +metric: Increase % of trial teams active in their second week +--- + +Teams that use the product every week renew. diff --git a/evals/semantic/strict-ost/space/New users find the editor toolbar confusing.md b/evals/semantic/strict-ost/space/New users find the editor toolbar confusing.md new file mode 100644 index 0000000..1f1e585 --- /dev/null +++ b/evals/semantic/strict-ost/space/New users find the editor toolbar confusing.md @@ -0,0 +1,8 @@ +--- +type: opportunity +status: identified +parent: "[[Increase weekly active teams]]" +source: Usability test round 2 +--- + +Participants hovered over icons without knowing what they did. diff --git a/evals/semantic/strict-ost/space/Offer a discount on annual plans.md b/evals/semantic/strict-ost/space/Offer a discount on annual plans.md new file mode 100644 index 0000000..c70b935 --- /dev/null +++ b/evals/semantic/strict-ost/space/Offer a discount on annual plans.md @@ -0,0 +1,8 @@ +--- +type: opportunity +status: identified +parent: "[[Increase weekly active teams]]" +source: Pricing workshop +--- + +Ten percent off when paying yearly. diff --git a/evals/semantic/strict-ost/space/Onboarding.md b/evals/semantic/strict-ost/space/Onboarding.md new file mode 100644 index 0000000..1fc9d5a --- /dev/null +++ b/evals/semantic/strict-ost/space/Onboarding.md @@ -0,0 +1,8 @@ +--- +type: opportunity +status: identified +parent: "[[Increase weekly active teams]]" +source: Interview synthesis +--- + +Everything about how new users get started. diff --git a/evals/semantic/strict-ost/space/Reduce churn among small teams.md b/evals/semantic/strict-ost/space/Reduce churn among small teams.md new file mode 100644 index 0000000..6184ca8 --- /dev/null +++ b/evals/semantic/strict-ost/space/Reduce churn among small teams.md @@ -0,0 +1,8 @@ +--- +type: opportunity +status: identified +parent: "[[Increase weekly active teams]]" +source: Retention dashboard +--- + +Teams of fewer than five seats churn twice as often. diff --git a/evals/semantic/strict-ost/space/Setting up a workspace takes so long that I give up.md b/evals/semantic/strict-ost/space/Setting up a workspace takes so long that I give up.md new file mode 100644 index 0000000..900f090 --- /dev/null +++ b/evals/semantic/strict-ost/space/Setting up a workspace takes so long that I give up.md @@ -0,0 +1,8 @@ +--- +type: opportunity +status: identified +parent: "[[Increase weekly active teams]]" +source: Interview with Priya, 2026-09-02 +--- + +Priya abandoned setup twice before her team lead finished it for her. diff --git a/evals/semantic/strict-ost/space/Show unsaved-change indicator.md b/evals/semantic/strict-ost/space/Show unsaved-change indicator.md new file mode 100644 index 0000000..cf4748e --- /dev/null +++ b/evals/semantic/strict-ost/space/Show unsaved-change indicator.md @@ -0,0 +1,7 @@ +--- +type: solution +parent: "[[I don't know whether my changes have been saved]]" +status: exploring +--- + +A dot on the tab while changes are pending. diff --git a/package.json b/package.json index 73db597..41aad0a 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "test:smoke": "bun test smoke/", "test:package": "bash scripts/test-package.sh", "test:judge": "SCTX_LIVE_JUDGE_TESTS=1 bun test tests/judge/live.test.ts", + "eval:semantic": "bun run evals/semantic/run.ts", "test:models": "SCTX_MODEL_VALIDATION=1 bun test tests/judge/model-validation.test.ts", "test:hook": "bun test hook-test/unit/", "test:mod": "claude plugin test plugin", diff --git a/tests/evals/semantic.test.ts b/tests/evals/semantic.test.ts new file mode 100644 index 0000000..7548dfc --- /dev/null +++ b/tests/evals/semantic.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, test } from 'bun:test'; +import { + checkEvalSpace, + type EvalSpace, + type Judgement, + judgeEvalSpace, + loadEvalSpaces, + scoreEvalSpace, +} from '../../evals/semantic/harness'; +import { FakeJudgeProvider } from '../../src/judge/provider/fake'; + +const spaces = loadEvalSpaces(); + +/** A judge that gives every labelled case its first expected verdict. */ +const labelJudge = (space: EvalSpace) => + new FakeJudgeProvider(({ node, rule }) => ({ verdict: space.expected[rule.id]?.[node.id]?.[0] ?? 'unlabelled' })); + +describe.each(spaces.map((s) => [s.name, s] as const))('eval space %s', (_name, space) => { + test('is valid and its labels name real nodes, semantic rules and verdicts', async () => { + expect(await checkEvalSpace(space)).toEqual([]); + }); + + test('every judged node is labelled, and every label is judged', async () => { + const scores = scoreEvalSpace(space, [await judgeEvalSpace(space, labelJudge(space))]); + for (const score of scores) { + expect(score.unlabelled).toEqual([]); + expect(score.misses).toEqual([]); + expect(score.agreement).toBe(1); + } + }); +}); + +describe('judgeEvalSpace', () => { + const [space] = spaces; + /** A judge that packs nodes, counting its calls of each kind. */ + class PackingJudge extends FakeJudgeProvider { + calls = { evaluate: 0, evaluateNodes: 0 }; + override evaluate(input: Parameters[0]) { + this.calls.evaluate++; + return super.evaluate(input); + } + async evaluateNodes(inputs: Parameters[0][]) { + this.calls.evaluateNodes++; + return Promise.all(inputs.map((input) => super.evaluate(input))); + } + } + + test('passes packed inputs through, or one input per call with perNode', async () => { + const packed = new PackingJudge(); + const perNode = new PackingJudge(); + const a = await judgeEvalSpace(space!, packed); + const b = await judgeEvalSpace(space!, perNode, { perNode: true }); + expect(packed.calls).toEqual({ evaluate: 0, evaluateNodes: expect.any(Number) }); + expect(perNode.calls).toEqual({ evaluate: b.length, evaluateNodes: 0 }); + expect(a.length).toBe(b.length); + }); +}); + +describe('scoreEvalSpace', () => { + const space = { + name: 'x', + expected: { r: { a: ['pass'], b: ['fail'], c: ['pass', 'fail'] } }, + } as unknown as EvalSpace; + const j = (nodeId: string, verdict: string, probabilities?: Record): Judgement => ({ + nodeId, + ruleId: 'r', + result: { verdict, ...(probabilities && { probabilities }) }, + }); + + test('scores agreement, stability, probability, errors and unlabelled nodes over runs', () => { + const runs = [ + [ + j('a', 'pass', { pass: 0.9, fail: 0.1 }), + j('b', 'pass', { pass: 0.6, fail: 0.4 }), + j('c', 'fail'), + j('d', 'pass'), + ], + [j('a', 'pass', { pass: 0.7, fail: 0.3 }), j('b', 'fail', { pass: 0.2, fail: 0.8 })], + ]; + const [score] = scoreEvalSpace(space, runs); + expect(score).toMatchObject({ cases: 3, runs: 2, stable: 1 / 3, errors: 1, reserved: 0, unlabelled: ['d'] }); + expect(score!.agreement).toBeCloseTo(4 / 6); + expect(score!.expectedProbability).toBeCloseTo((0.9 + 0.4 + 0.7 + 0.8) / 4); + expect(score!.misses).toEqual([ + { nodeId: 'b', expected: ['fail'], got: ['pass', 'fail'] }, + { nodeId: 'c', expected: ['pass', 'fail'], got: ['fail', 'not judged'] }, + ]); + }); + + test('counts reserved verdicts and errors', () => { + const runs = [[j('a', 'missing-context'), { nodeId: 'b', ruleId: 'r', result: { error: 'boom' } }, j('c', 'pass')]]; + const [score] = scoreEvalSpace(space, runs); + expect(score).toMatchObject({ reserved: 1, errors: 1, agreement: 1 / 3 }); + }); +}); diff --git a/tsconfig.json b/tsconfig.json index 7a7bd17..50ca1dd 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -29,6 +29,14 @@ "noUnusedParameters": false, "noPropertyAccessFromIndexSignature": false }, - "include": ["src/**/*", "tests/**/*", "smoke/**/*", "hook-test/**/*", "plugin/scripts/**/*", "scripts/**/*"], + "include": [ + "src/**/*", + "tests/**/*", + "smoke/**/*", + "hook-test/**/*", + "plugin/scripts/**/*", + "scripts/**/*", + "evals/**/*" + ], "exclude": ["node_modules", "dist"] } From 35df2d2aaae0f27252fb6ec7a5fc5ffb10fd270a Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 16:11:22 +0000 Subject: [PATCH 2/7] build: exclude evals from the package build Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019Xphpdioyw95tAT6PEKGTU --- tsconfig.build.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tsconfig.build.json b/tsconfig.build.json index 55948f3..81aa233 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -11,6 +11,7 @@ "hook-test", "plugin/scripts", "scripts", + "evals", "**/*.test.ts", "**/*.spec.ts" ] From 1dc574a72b94438f2b2a4db86ad005300bbb96a7 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 22:23:17 +0000 Subject: [PATCH 3/7] feat(evals): eval spaces from plugins, local configs and configured spaces --from picks where eval spaces come from: core, a plugin's evals/semantic/, evals/semantic/ beside the config, or a path. An eval.json5 either names a schema for its own fixture nodes or labels nodes of a configured space. Only labelled nodes reach the provider, and --check runs the free checks. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019Xphpdioyw95tAT6PEKGTU --- AGENTS.md | 2 +- evals/semantic/README.md | 37 ++++++-- evals/semantic/harness.ts | 178 ++++++++++++++++++++++++++--------- evals/semantic/run.ts | 36 +++++-- tests/evals/semantic.test.ts | 55 ++++++++--- 5 files changed, 236 insertions(+), 72 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 36cb3a8..52cbac0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -100,7 +100,7 @@ For most development only the main unit tests need re-running regularly. - `bun run test:package` — builds and packs the package, installs the tarball, and runs the installed CLI. Runs in CI and before every version bump; run it after changing the build, `tsconfig*.json` or `files`. - `bun run test:judge` — live judge tests (`tests/judge/live.test.ts`): one real provider call per provider whose API key is set. Costs money; use sparingly when changing judges or semantic rule behaviour. - `bun run test:models` — check that every bundled provider's configured model exists (no API keys) -- `bun run eval:semantic --provider ` — semantic evals (`evals/semantic/`): how often a provider's verdicts match labelled fixture spaces for the bundled schemas' semantic rules, and how stable they are. Costs money; run it when changing a bundled semantic rule, the schema descriptions it draws on, or a judge. +- `bun run eval:semantic --provider ` — semantic evals (`evals/semantic/`): how often a provider's verdicts match labelled eval spaces, and how stable they are. Core's cover the bundled schemas; `--from` runs a plugin's or a local config's. Costs money (`--check` doesn't); run it when changing a bundled semantic rule, the schema descriptions it draws on, or a judge. ## Debugging diff --git a/evals/semantic/README.md b/evals/semantic/README.md index bd33d8f..08a9aab 100644 --- a/evals/semantic/README.md +++ b/evals/semantic/README.md @@ -1,6 +1,6 @@ # Semantic evals -Semantic evals measure how well model providers judge the semantic rules of the bundled schemas. Each eval space is a small fixture space with a label per node and rule: the verdict the node should get. +Semantic evals measure how well model providers judge semantic rules. An eval space is a space with a label per node and rule: the verdict the node should get. Core's eval spaces here cover the bundled schemas. A plugin, or a local config, can keep its own eval spaces for its schemas and spaces. ## Run the evals @@ -10,12 +10,16 @@ The run calls real providers, so it costs money. Name each provider to run; conf bun run eval:semantic --provider openrouter-jev --provider anthropic ``` +From outside this repo, run `bun run /evals/semantic/run.ts` with the same flags. + | Flag | Purpose | |---|---| | `--provider ` | The provider to judge with. Repeat it, or give a comma-separated list, to compare providers | -| `--space ` | Run only this eval space. Repeatable | +| `--from ` | Where to find eval spaces. Repeatable. Default: `core`. See [Where eval spaces live](#where-eval-spaces-live) | +| `--space ` | Run only this eval space, by name (`core/strict-ost`) or directory (`strict-ost`). Repeatable | | `--runs ` | Judge each space this many times, to measure stability. Default: 3 | | `--per-node` | Send one request per node, even to a provider that packs several nodes into one request | +| `--check` | Check the eval spaces and stop, with no provider calls | | `--json` | Print every judgement and score as JSON instead of the report | The report has one row per space, provider and rule: @@ -28,11 +32,24 @@ The report has one row per space, provider and rule: | Reserved | Judgements with a reserved verdict, such as `missing-context` | | Errors | Cases that got an evaluation error, or no judgement, on some run | -Each run validates the space through the same pipeline as `sctx validate --semantic`, with only the labelled rules, and evaluates gated nodes too. +Each run validates the space through the same pipeline as `sctx validate --semantic`, with only the labelled rules, and evaluates gated nodes too. Only labelled nodes go to the provider. + +## Where eval spaces live + +Each source is a directory of eval spaces: + +| `--from` | Directory | Schema names resolve from | +|---|---|---| +| `core` | `evals/semantic/` in this repo | the bundled schemas | +| `` | `evals/semantic/` in the plugin, found as the config finds plugins | the config that finds the plugin | +| `config` | `evals/semantic/` beside the config file, and beside each file it includes | that config | +| a path | that directory | that directory | + +## Add an eval space -## Add a case +An eval space is a directory with an `eval.json5`. It is one of two kinds. -An eval space is a directory here with a `space/` directory of nodes and an `eval.json5`: +A **fixture** eval space has its own nodes in a `space/` directory beside `eval.json5`. It names its `schema`, and optionally `plugins`, as a space config does: ```json5 { @@ -47,6 +64,12 @@ An eval space is a directory here with a `space/` directory of nodes and an `eva } ``` -`expected` maps a rule id to node ids (as `sctx dump` shows them) and their verdicts. Label every node a labelled rule judges. Write a comment on a case whose label is not obvious. +An eval space on a **configured space** names a space from the config instead, and labels some of its nodes: + +```json5 +{ space: 'work', expected: { 'opportunity-customer-voice': { 'Onboarding.md': 'fail' } } } +``` + +`expected` maps a rule id to node ids (as `sctx dump` shows them) and their verdicts. In a fixture, label every node a labelled rule judges. Write a comment on a case whose label is not obvious. -`bun run test` checks each eval space for free: the space is valid, every label names a real node, semantic rule and verdict, and every node a labelled rule judges has a label. +`--check` finds the problems that would make scores meaningless: a label that names an unknown node, rule or verdict; a labelled node that fails validation; and, in a fixture, a judged node with no label. A run checks this before its first provider call. `bun run test` checks core's eval spaces. diff --git a/evals/semantic/harness.ts b/evals/semantic/harness.ts index 0fd50aa..3d2fc4f 100644 --- a/evals/semantic/harness.ts +++ b/evals/semantic/harness.ts @@ -1,23 +1,32 @@ -import { readdirSync, readFileSync } from 'node:fs'; -import { join } from 'node:path'; +import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { basename, join, resolve } from 'node:path'; import JSON5 from 'json5'; -import type { Config } from '../../src/config'; +import type { Config, SpaceConfig } from '../../src/config'; +import { FakeJudgeProvider } from '../../src/judge/provider/fake'; import type { SemanticJudgeProvider, SemanticJudgeResult } from '../../src/judge/types'; import { resolveSemanticRule } from '../../src/judge/verdicts'; +import { resolvePluginDir } from '../../src/plugins/util'; import { readSpace } from '../../src/read/read-space'; import { RESERVED_VERDICTS } from '../../src/schema/metadata-contract'; import { createSpaceContext } from '../../src/space-context'; import type { SpaceContext } from '../../src/types'; import { validateNodes } from '../../src/validate'; -export const EVALS_DIR = import.meta.dir; +/** Core's eval spaces, for the bundled schemas. */ +export const CORE_EVALS_DIR = import.meta.dir; + +/** Where eval spaces live, relative to a plugin's root or a config file's directory. */ +export const EVALS_SUBDIR = join('evals', 'semantic'); /** Rule id → node id → the verdicts that count as agreeing. */ export type Expected = Record>; -/** A fixture space and the verdict each of its nodes should get per rule. */ +/** A space and the verdict each of its nodes should get per rule. */ export interface EvalSpace { + /** `/`, e.g. `core/strict-ost`. */ name: string; + /** A fixture space has its own nodes, so every node a labelled rule judges must have a label. */ + fixture: boolean; context: SpaceContext; expected: Expected; } @@ -29,43 +38,105 @@ export interface Judgement { result: SemanticJudgeResult; } -type EvalFile = { schema: string; expected: Record> }; +/** A directory of eval spaces, and the directory their schema names resolve from. */ +export interface EvalSource { + label: string; + dir: string; + configDir: string; +} -/** The eval spaces under `root`: each directory with an `eval.json5` beside a `space/` directory. */ -export function loadEvalSpaces(root: string = EVALS_DIR, only?: string[]): EvalSpace[] { - const names = readdirSync(root, { withFileTypes: true }) - .filter((d) => d.isDirectory() && (!only || only.includes(d.name))) - .map((d) => d.name) - .sort(); - const unknown = only?.filter((name) => !names.includes(name)) ?? []; - if (unknown.length > 0) throw new Error(`Unknown eval space: ${unknown.join(', ')}. Known: ${names.join(', ')}`); - return names.map((name) => { - const dir = join(root, name); - const file = JSON5.parse(readFileSync(join(dir, 'eval.json5'), 'utf8')) as EvalFile; - const config: Config = { spaces: [{ name, path: join(dir, 'space'), schema: file.schema }] }; - const expected: Expected = {}; - for (const [ruleId, nodes] of Object.entries(file.expected)) { - expected[ruleId] = Object.fromEntries( - Object.entries(nodes).map(([id, v]) => [id, typeof v === 'string' ? [v] : v]), - ); +/** + * Find a source: `core`; `config` (an `evals/semantic/` directory beside each loaded config file); a plugin name + * (its `evals/semantic/`); or a path to a directory of eval spaces. A plugin's and a config's schema names + * resolve as they would in that config. + */ +export function resolveEvalSources(source: string, configDirs: string[]): EvalSource[] { + if (source === 'core') return [{ label: 'core', dir: CORE_EVALS_DIR, configDir: CORE_EVALS_DIR }]; + if (source === 'config') { + return configDirs + .map((configDir) => ({ label: 'config', dir: join(configDir, EVALS_SUBDIR), configDir })) + .filter(({ dir }) => existsSync(dir)); + } + if (source.includes('/') || source.includes('\\') || source.startsWith('.')) { + const dir = resolve(source); + if (!existsSync(dir)) throw new Error(`Eval source "${source}": no directory at ${dir}`); + return [{ label: basename(dir), dir, configDir: dir }]; + } + for (const configDir of configDirs) { + const pluginDir = resolvePluginDir(source, configDir); + if (pluginDir) { + const dir = join(pluginDir, EVALS_SUBDIR); + if (!existsSync(dir)) throw new Error(`Plugin "${source}" has no ${EVALS_SUBDIR}/ directory (${pluginDir})`); + return [{ label: source, dir, configDir }]; } - return { name, context: createSpaceContext(name, config, { configDir: dir }), expected }; - }); + } + throw new Error(`Eval source "${source}" is not core, config, a path, or a plugin found from the config.`); } /** - * Problems that would make an eval space's scores meaningless: an invalid space (gating and schema failures - * change what is judged), or a label naming an unknown node, a rule with no semantic part, or a verdict the - * rule cannot return. Checked before any judge call. + * An eval space's `eval.json5`. A fixture eval space names a `schema` (and optional `plugins`), as a space config + * does, and keeps its nodes in `space/` beside the file. An eval space on a configured space names the `space` + * instead, and labels its nodes. + */ +type EvalFile = Pick & { + space?: string; + expected: Record>; +}; + +/** The eval spaces in each source: every directory with an `eval.json5`. `config` holds configured spaces. */ +export function loadEvalSpaces(sources: EvalSource[], config: Config): EvalSpace[] { + return sources.flatMap((source) => + readdirSync(source.dir, { withFileTypes: true }) + .filter((d) => d.isDirectory() && existsSync(join(source.dir, d.name, 'eval.json5'))) + .map((d) => d.name) + .sort() + .map((dirName) => { + const name = `${source.label}/${dirName}`; + const dir = join(source.dir, dirName); + const file = JSON5.parse(readFileSync(join(dir, 'eval.json5'), 'utf8')) as EvalFile; + if ((file.space === undefined) === (file.schema === undefined)) { + throw new Error(`Eval space ${name}: eval.json5 needs either "schema" (with a space/ directory) or "space".`); + } + let context: SpaceContext; + try { + context = + file.space !== undefined + ? createSpaceContext(file.space, config) + : createSpaceContext( + name, + { spaces: [{ name, path: join(dir, 'space'), schema: file.schema, plugins: file.plugins }] }, + { configDir: source.configDir }, + ); + } catch (err) { + throw new Error(`Eval space ${name}: ${err instanceof Error ? err.message : String(err)}`); + } + const expected: Expected = {}; + for (const [ruleId, nodes] of Object.entries(file.expected)) { + expected[ruleId] = Object.fromEntries( + Object.entries(nodes).map(([id, v]) => [id, typeof v === 'string' ? [v] : v]), + ); + } + return { name, fixture: file.space === undefined, context, expected }; + }), + ); +} + +/** + * Problems that would make an eval space's scores meaningless, checked before any judge call: a label naming an + * unknown node, a rule with no semantic part, or a verdict the rule cannot return; a labelled node that fails + * validation; and, in a fixture space, a node a labelled rule judges with no label, or a label never judged. */ export async function checkEvalSpace(space: EvalSpace): Promise { const problems: string[] = []; const read = await readSpace(space.context); + const ids = new Set(read.nodes.map((n) => n.id as string)); + const labelled = new Set(Object.values(space.expected).flatMap((labels) => Object.keys(labels))); const result = await validateNodes(space.context, read.nodes, read); for (const issue of result.issues) { - if (issue.severity === 'error') problems.push(`${issue.nodeId ?? ''}: ${issue.message}`); + if (issue.severity === 'error' && issue.nodeId && labelled.has(issue.nodeId)) { + problems.push(`${issue.nodeId}: ${issue.message}`); + } } - const ids = new Set(read.nodes.map((n) => n.id as string)); const rules = space.context.schema.metadata.rules ?? []; for (const [ruleId, labels] of Object.entries(space.expected)) { const rule = rules.find((r) => r.id === ruleId); @@ -83,24 +154,38 @@ export async function checkEvalSpace(space: EvalSpace): Promise { } } } + if (space.fixture && problems.length === 0) { + const judge = new FakeJudgeProvider(({ node, rule }) => ({ + verdict: space.expected[rule.id]?.[node.id]?.[0] ?? Object.keys(rule.verdicts)[0]!, + })); + for (const score of scoreEvalSpace(space, [await judgeEvalSpace(space, judge, { unlabelled: true })])) { + for (const nodeId of score.unlabelled) problems.push(`rule "${score.ruleId}": node "${nodeId}" has no label`); + for (const { nodeId } of score.misses) problems.push(`rule "${score.ruleId}": node "${nodeId}" is not judged`); + } + } return problems.map((p) => `${space.name}: ${p}`); } +type JudgeInput = Parameters[0]; + /** * Validate the space with its labelled rules through the real pipeline, recording every result the provider - * returns, passing verdicts included. Gated nodes are judged too. `perNode` hides `evaluateNodes`, so a - * provider that packs nodes into one request gets one request per node instead. + * returns, passing verdicts included. Gated nodes are judged too. Only labelled nodes reach the provider, unless + * `unlabelled` is set. `perNode` hides `evaluateNodes`, so a provider that packs nodes into one request gets one + * request per node instead. */ export async function judgeEvalSpace( space: EvalSpace, provider: SemanticJudgeProvider, - options: { perNode?: boolean } = {}, + options: { perNode?: boolean; unlabelled?: boolean } = {}, ): Promise { const judgements: Judgement[] = []; - const record = ( - input: Parameters[0], - results: Record, - ) => { + const toJudge = (input: JudgeInput): JudgeInput | undefined => { + if (options.unlabelled) return input; + const rules = input.rules.filter((rule) => space.expected[rule.id]?.[input.node.id] !== undefined); + return rules.length > 0 ? { ...input, rules } : undefined; + }; + const record = (input: JudgeInput, results: Record) => { for (const rule of input.rules) { const result = results[rule.id] ?? { error: 'no result for this rule' }; judgements.push({ nodeId: input.node.id, ruleId: rule.id, result }); @@ -109,17 +194,26 @@ export async function judgeEvalSpace( const recording: SemanticJudgeProvider = { name: provider.name, async evaluate(input) { - const results = await provider.evaluate(input); - record(input, results); + const sent = toJudge(input); + if (!sent) return {}; + const results = await provider.evaluate(sent); + record(sent, results); return results; }, }; const { evaluateNodes } = provider; if (evaluateNodes && !options.perNode) { recording.evaluateNodes = async (inputs) => { - const results = await evaluateNodes.call(provider, inputs); - for (const [i, input] of inputs.entries()) record(input, results[i] ?? {}); - return results; + const sent = inputs.map(toJudge); + const kept = sent.filter((input) => input !== undefined); + const results = kept.length > 0 ? await evaluateNodes.call(provider, kept) : []; + let k = 0; + return sent.map((input) => { + if (!input) return {}; + const result = results[k++] ?? {}; + record(input, result); + return result; + }); }; } const read = await readSpace(space.context); diff --git a/evals/semantic/run.ts b/evals/semantic/run.ts index b34ff6d..55b22f9 100644 --- a/evals/semantic/run.ts +++ b/evals/semantic/run.ts @@ -1,9 +1,11 @@ /** - * Live semantic evals: judge the eval spaces with real providers and report agreement with the labels. - * Costs money. See README.md. + * Semantic evals: judge eval spaces with real providers and report agreement with the labels. Costs money, + * except with `--check`. See README.md. */ +import { existsSync } from 'node:fs'; +import { dirname } from 'node:path'; import { parseArgs } from 'node:util'; -import { loadConfigOrEmpty } from '../../src/config'; +import { configPath, getConfigSourceFiles, loadConfigOrEmpty } from '../../src/config'; import { createJudgeProvider } from '../../src/judge/factory'; import { checkEvalSpace, @@ -11,32 +13,52 @@ import { judgeEvalSpace, loadEvalSpaces, type RuleScore, + resolveEvalSources, scoreEvalSpace, } from './harness'; const { values } = parseArgs({ options: { + from: { type: 'string', multiple: true }, provider: { type: 'string', multiple: true }, space: { type: 'string', multiple: true }, runs: { type: 'string', default: '3' }, 'per-node': { type: 'boolean', default: false }, json: { type: 'boolean', default: false }, + check: { type: 'boolean', default: false }, }, }); -const providerNames = (values.provider ?? []).flatMap((p) => p.split(',')).filter(Boolean); +const list = (values: string[] | undefined) => (values ?? []).flatMap((v) => v.split(',')).filter(Boolean); +const providerNames = list(values.provider); const runs = Number(values.runs); -if (providerNames.length === 0) throw new Error('Name at least one model provider with --provider .'); +if (providerNames.length === 0 && !values.check) { + throw new Error('Name at least one model provider with --provider , or pass --check.'); +} if (!Number.isInteger(runs) || runs < 1) { throw new Error(`--runs must be a positive whole number, not "${values.runs}".`); } // Build every provider and check every space before the first paid call. const config = loadConfigOrEmpty(); +// The main config file and every file it includes. +const configFiles = [configPath(), ...getConfigSourceFiles()].filter((file) => existsSync(file)); +const configDirs = [...new Set(configFiles.map((file) => dirname(file)))]; const providers = providerNames.map((name) => createJudgeProvider(config, {}, name)!); -const spaces = loadEvalSpaces(undefined, values.space); +const sources = (list(values.from).length > 0 ? list(values.from) : ['core']).flatMap((source) => + resolveEvalSources(source, configDirs), +); +const only = list(values.space); +const spaces = loadEvalSpaces(sources, config).filter( + (space) => only.length === 0 || only.some((name) => space.name === name || space.name.endsWith(`/${name}`)), +); +if (spaces.length === 0) throw new Error('No eval spaces found.'); const problems = (await Promise.all(spaces.map(checkEvalSpace))).flat(); if (problems.length > 0) throw new Error(`Eval spaces have problems:\n${problems.join('\n')}`); +if (values.check) { + console.log(`Checked ${spaces.length} eval space(s): ${spaces.map((s) => s.name).join(', ')}`); + process.exit(0); +} const results: Array<{ space: string; provider: string; scores: RuleScore[]; runs: Judgement[][] }> = []; for (const space of spaces) { @@ -80,8 +102,6 @@ if (values.json) { ); if (m.evidence) notes.push(` ${m.evidence}`); } - if (s.unlabelled.length > 0) - notes.push(`- ${space} · ${provider} · ${s.ruleId}: judged with no label: ${s.unlabelled.join(', ')}`); } } if (notes.length > 0) lines.push('', 'Misses:', ...notes); diff --git a/tests/evals/semantic.test.ts b/tests/evals/semantic.test.ts index 7548dfc..e7a3c62 100644 --- a/tests/evals/semantic.test.ts +++ b/tests/evals/semantic.test.ts @@ -1,32 +1,59 @@ -import { describe, expect, test } from 'bun:test'; +import { afterAll, describe, expect, test } from 'bun:test'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, join } from 'node:path'; import { + CORE_EVALS_DIR, checkEvalSpace, type EvalSpace, type Judgement, judgeEvalSpace, loadEvalSpaces, + resolveEvalSources, scoreEvalSpace, } from '../../evals/semantic/harness'; +import type { Config } from '../../src/config'; import { FakeJudgeProvider } from '../../src/judge/provider/fake'; -const spaces = loadEvalSpaces(); - -/** A judge that gives every labelled case its first expected verdict. */ -const labelJudge = (space: EvalSpace) => - new FakeJudgeProvider(({ node, rule }) => ({ verdict: space.expected[rule.id]?.[node.id]?.[0] ?? 'unlabelled' })); +const spaces = loadEvalSpaces(resolveEvalSources('core', []), { spaces: [] }); describe.each(spaces.map((s) => [s.name, s] as const))('eval space %s', (_name, space) => { - test('is valid and its labels name real nodes, semantic rules and verdicts', async () => { + test('passes its checks: labels name real nodes, semantic rules and verdicts, and cover every judged node', async () => { expect(await checkEvalSpace(space)).toEqual([]); }); +}); - test('every judged node is labelled, and every label is judged', async () => { - const scores = scoreEvalSpace(space, [await judgeEvalSpace(space, labelJudge(space))]); - for (const score of scores) { - expect(score.unlabelled).toEqual([]); - expect(score.misses).toEqual([]); - expect(score.agreement).toBe(1); - } +describe('eval spaces on a configured space', () => { + const dir = mkdtempSync(join(tmpdir(), 'sctx-evals-')); + const fixture = join(CORE_EVALS_DIR, 'strict-ost', 'space'); + mkdirSync(join(dir, 'mine')); + writeFileSync( + join(dir, 'mine', 'eval.json5'), + JSON.stringify({ + space: 'work', + expected: { 'opportunity-customer-voice': { 'Onboarding.md': 'fail', 'Build a Slack integration.md': 'fail' } }, + }), + ); + const config: Config = { spaces: [{ name: 'work', path: fixture, schema: 'strict_ost.json', sourceDir: dir }] }; + const [space] = loadEvalSpaces(resolveEvalSources(dir, []), config); + afterAll(() => rmSync(dir, { recursive: true, force: true })); + + test('loads the configured space, named by its source directory', () => { + expect(space).toMatchObject({ name: `${basename(dir)}/mine`, fixture: false }); + expect(space!.context.space.name).toBe('work'); + }); + + test('passes its checks without labelling every node', async () => { + expect(await checkEvalSpace(space!)).toEqual([]); + }); + + test('sends only labelled nodes to the provider', async () => { + const judge = new FakeJudgeProvider({ verdict: 'fail' }); + const judged = await judgeEvalSpace(space!, judge); + expect(judged.map((j) => j.nodeId).sort()).toEqual(['Build a Slack integration.md', 'Onboarding.md']); + expect(judge.callHistory.map((input) => input.node.id as string).sort()).toEqual( + judged.map((j) => j.nodeId).sort(), + ); }); }); From d7ca624b79e62384f698df0a4c2ed58194c73036 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 22:57:45 +0000 Subject: [PATCH 4/7] feat(evals): fixture nodes in JSONL; drop the config source Fixture eval spaces keep their nodes in nodes.jsonl (parsed nodes, one per line), so editors and spaces never read them as markdown. The config source waits for the .sctx/ layout; --from covers local eval spaces until then. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019Xphpdioyw95tAT6PEKGTU --- AGENTS.md | 2 +- evals/semantic/README.md | 11 ++-- evals/semantic/harness.ts | 51 ++++++++++++------- evals/semantic/strict-ost/eval.json5 | 24 ++++----- evals/semantic/strict-ost/nodes.jsonl | 14 +++++ .../Add a progress bar to workspace setup.md | 8 --- .../space/Build a Slack integration.md | 8 --- ... tell which plan fits a team of my size.md | 8 --- ...know whether my changes have been saved.md | 8 --- ...t my teammates changed while I was away.md | 8 --- ...I'll lose my work if my browser crashes.md | 8 --- ...ncrease trial-to-paid conversion by 10%.md | 8 --- .../space/Increase weekly active teams.md | 7 --- ...users find the editor toolbar confusing.md | 8 --- .../space/Offer a discount on annual plans.md | 8 --- evals/semantic/strict-ost/space/Onboarding.md | 8 --- .../space/Reduce churn among small teams.md | 8 --- ... workspace takes so long that I give up.md | 8 --- .../space/Show unsaved-change indicator.md | 7 --- tests/evals/semantic.test.ts | 10 +++- 20 files changed, 73 insertions(+), 149 deletions(-) create mode 100644 evals/semantic/strict-ost/nodes.jsonl delete mode 100644 evals/semantic/strict-ost/space/Add a progress bar to workspace setup.md delete mode 100644 evals/semantic/strict-ost/space/Build a Slack integration.md delete mode 100644 evals/semantic/strict-ost/space/I can't tell which plan fits a team of my size.md delete mode 100644 evals/semantic/strict-ost/space/I don't know whether my changes have been saved.md delete mode 100644 evals/semantic/strict-ost/space/I want to see what my teammates changed while I was away.md delete mode 100644 evals/semantic/strict-ost/space/I'm afraid I'll lose my work if my browser crashes.md delete mode 100644 evals/semantic/strict-ost/space/Increase trial-to-paid conversion by 10%.md delete mode 100644 evals/semantic/strict-ost/space/Increase weekly active teams.md delete mode 100644 evals/semantic/strict-ost/space/New users find the editor toolbar confusing.md delete mode 100644 evals/semantic/strict-ost/space/Offer a discount on annual plans.md delete mode 100644 evals/semantic/strict-ost/space/Onboarding.md delete mode 100644 evals/semantic/strict-ost/space/Reduce churn among small teams.md delete mode 100644 evals/semantic/strict-ost/space/Setting up a workspace takes so long that I give up.md delete mode 100644 evals/semantic/strict-ost/space/Show unsaved-change indicator.md diff --git a/AGENTS.md b/AGENTS.md index 52cbac0..74f4166 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -100,7 +100,7 @@ For most development only the main unit tests need re-running regularly. - `bun run test:package` — builds and packs the package, installs the tarball, and runs the installed CLI. Runs in CI and before every version bump; run it after changing the build, `tsconfig*.json` or `files`. - `bun run test:judge` — live judge tests (`tests/judge/live.test.ts`): one real provider call per provider whose API key is set. Costs money; use sparingly when changing judges or semantic rule behaviour. - `bun run test:models` — check that every bundled provider's configured model exists (no API keys) -- `bun run eval:semantic --provider ` — semantic evals (`evals/semantic/`): how often a provider's verdicts match labelled eval spaces, and how stable they are. Core's cover the bundled schemas; `--from` runs a plugin's or a local config's. Costs money (`--check` doesn't); run it when changing a bundled semantic rule, the schema descriptions it draws on, or a judge. +- `bun run eval:semantic --provider ` — semantic evals (`evals/semantic/`): how often a provider's verdicts match labelled eval spaces, and how stable they are. Core's cover the bundled schemas; `--from` runs a plugin's, or a directory's. Costs money (`--check` doesn't); run it when changing a bundled semantic rule, the schema descriptions it draws on, or a judge. ## Debugging diff --git a/evals/semantic/README.md b/evals/semantic/README.md index 08a9aab..249b9a8 100644 --- a/evals/semantic/README.md +++ b/evals/semantic/README.md @@ -1,6 +1,6 @@ # Semantic evals -Semantic evals measure how well model providers judge semantic rules. An eval space is a space with a label per node and rule: the verdict the node should get. Core's eval spaces here cover the bundled schemas. A plugin, or a local config, can keep its own eval spaces for its schemas and spaces. +Semantic evals measure how well model providers judge semantic rules. An eval space is a space with a label per node and rule: the verdict the node should get. Core's eval spaces here cover the bundled schemas. A plugin can keep its own eval spaces for its schemas, and an eval space can label nodes of a configured space. ## Run the evals @@ -42,23 +42,22 @@ Each source is a directory of eval spaces: |---|---|---| | `core` | `evals/semantic/` in this repo | the bundled schemas | | `` | `evals/semantic/` in the plugin, found as the config finds plugins | the config that finds the plugin | -| `config` | `evals/semantic/` beside the config file, and beside each file it includes | that config | | a path | that directory | that directory | ## Add an eval space An eval space is a directory with an `eval.json5`. It is one of two kinds. -A **fixture** eval space has its own nodes in a `space/` directory beside `eval.json5`. It names its `schema`, and optionally `plugins`, as a space config does: +A **fixture** eval space has its own nodes in a `nodes.jsonl` beside `eval.json5`: one parsed node per line, as a plugin's parse hook returns it (`id`, `title`, `type`, `linkTargets` and `schemaData`). JSON keeps fixtures out of the markdown that editors and spaces read. To start from markdown notes, `sctx dump | jq -c '.nodes[]'` prints nodes in this shape. The eval space names its `schema`, and optionally `plugins`, as a space config does: ```json5 { schema: 'strict_ost.json', expected: { 'opportunity-customer-voice': { - 'Build a Slack integration.md': 'fail', + 'Build a Slack integration': 'fail', // A list accepts any of its verdicts. - 'New users find the editor toolbar confusing.md': ['pass', 'fail'], + 'New users find the editor toolbar confusing': ['pass', 'fail'], }, }, } @@ -70,6 +69,6 @@ An eval space on a **configured space** names a space from the config instead, a { space: 'work', expected: { 'opportunity-customer-voice': { 'Onboarding.md': 'fail' } } } ``` -`expected` maps a rule id to node ids (as `sctx dump` shows them) and their verdicts. In a fixture, label every node a labelled rule judges. Write a comment on a case whose label is not obvious. +`expected` maps a rule id to node ids and their verdicts. In a fixture, label every node a labelled rule judges. Write a comment on a case whose label is not obvious. `--check` finds the problems that would make scores meaningless: a label that names an unknown node, rule or verdict; a labelled node that fails validation; and, in a fixture, a judged node with no label. A run checks this before its first provider call. `bun run test` checks core's eval spaces. diff --git a/evals/semantic/harness.ts b/evals/semantic/harness.ts index 3d2fc4f..6532ac1 100644 --- a/evals/semantic/harness.ts +++ b/evals/semantic/harness.ts @@ -7,15 +7,16 @@ import type { SemanticJudgeProvider, SemanticJudgeResult } from '../../src/judge import { resolveSemanticRule } from '../../src/judge/verdicts'; import { resolvePluginDir } from '../../src/plugins/util'; import { readSpace } from '../../src/read/read-space'; +import { resolveGraphEdges } from '../../src/read/resolve-graph-edges'; import { RESERVED_VERDICTS } from '../../src/schema/metadata-contract'; import { createSpaceContext } from '../../src/space-context'; -import type { SpaceContext } from '../../src/types'; +import type { BaseNode, ReadSpaceResult, SpaceContext } from '../../src/types'; import { validateNodes } from '../../src/validate'; /** Core's eval spaces, for the bundled schemas. */ export const CORE_EVALS_DIR = import.meta.dir; -/** Where eval spaces live, relative to a plugin's root or a config file's directory. */ +/** Where a plugin keeps its eval spaces. */ export const EVALS_SUBDIR = join('evals', 'semantic'); /** Rule id → node id → the verdicts that count as agreeing. */ @@ -29,6 +30,8 @@ export interface EvalSpace { fixture: boolean; context: SpaceContext; expected: Expected; + /** The space's nodes, as validation reads them. */ + read: () => Promise>; } /** One verdict (or error) the judge gave for a node and rule. */ @@ -46,17 +49,11 @@ export interface EvalSource { } /** - * Find a source: `core`; `config` (an `evals/semantic/` directory beside each loaded config file); a plugin name - * (its `evals/semantic/`); or a path to a directory of eval spaces. A plugin's and a config's schema names - * resolve as they would in that config. + * Find a source: `core`, a plugin name (its `evals/semantic/`), or a path to a directory of eval spaces. A + * plugin's schema names resolve as they would in the config that finds the plugin. */ export function resolveEvalSources(source: string, configDirs: string[]): EvalSource[] { if (source === 'core') return [{ label: 'core', dir: CORE_EVALS_DIR, configDir: CORE_EVALS_DIR }]; - if (source === 'config') { - return configDirs - .map((configDir) => ({ label: 'config', dir: join(configDir, EVALS_SUBDIR), configDir })) - .filter(({ dir }) => existsSync(dir)); - } if (source.includes('/') || source.includes('\\') || source.startsWith('.')) { const dir = resolve(source); if (!existsSync(dir)) throw new Error(`Eval source "${source}": no directory at ${dir}`); @@ -70,13 +67,13 @@ export function resolveEvalSources(source: string, configDirs: string[]): EvalSo return [{ label: source, dir, configDir }]; } } - throw new Error(`Eval source "${source}" is not core, config, a path, or a plugin found from the config.`); + throw new Error(`Eval source "${source}" is not core, a path, or a plugin found from the config.`); } /** * An eval space's `eval.json5`. A fixture eval space names a `schema` (and optional `plugins`), as a space config - * does, and keeps its nodes in `space/` beside the file. An eval space on a configured space names the `space` - * instead, and labels its nodes. + * does, and keeps its nodes in `nodes.jsonl` beside the file, one parsed node per line. An eval space on a + * configured space names the `space` instead, and labels some of its nodes. */ type EvalFile = Pick & { space?: string; @@ -95,7 +92,7 @@ export function loadEvalSpaces(sources: EvalSource[], config: Config): EvalSpace const dir = join(source.dir, dirName); const file = JSON5.parse(readFileSync(join(dir, 'eval.json5'), 'utf8')) as EvalFile; if ((file.space === undefined) === (file.schema === undefined)) { - throw new Error(`Eval space ${name}: eval.json5 needs either "schema" (with a space/ directory) or "space".`); + throw new Error(`Eval space ${name}: eval.json5 needs either "schema" (with nodes.jsonl) or "space".`); } let context: SpaceContext; try { @@ -104,7 +101,7 @@ export function loadEvalSpaces(sources: EvalSource[], config: Config): EvalSpace ? createSpaceContext(file.space, config) : createSpaceContext( name, - { spaces: [{ name, path: join(dir, 'space'), schema: file.schema, plugins: file.plugins }] }, + { spaces: [{ name, path: dir, schema: file.schema, plugins: file.plugins }] }, { configDir: source.configDir }, ); } catch (err) { @@ -116,11 +113,29 @@ export function loadEvalSpaces(sources: EvalSource[], config: Config): EvalSpace Object.entries(nodes).map(([id, v]) => [id, typeof v === 'string' ? [v] : v]), ); } - return { name, fixture: file.space === undefined, context, expected }; + const read = + file.space !== undefined + ? () => readSpace(context) + : async () => resolveGraphEdges(readNodes(join(dir, 'nodes.jsonl')), context.schema.metadata); + return { name, fixture: file.space === undefined, context, expected, read }; }), ); } +/** Parsed nodes, one JSON object per line: the nodes a plugin's parse hook returns. */ +function readNodes(path: string): BaseNode[] { + return readFileSync(path, 'utf8') + .split('\n') + .filter((line) => line.trim() !== '') + .map((line, i) => { + const node = JSON.parse(line) as BaseNode; + for (const key of ['id', 'title', 'type', 'linkTargets', 'schemaData'] as const) { + if (node[key] === undefined) throw new Error(`${path}:${i + 1}: node has no "${key}"`); + } + return node; + }); +} + /** * Problems that would make an eval space's scores meaningless, checked before any judge call: a label naming an * unknown node, a rule with no semantic part, or a verdict the rule cannot return; a labelled node that fails @@ -128,7 +143,7 @@ export function loadEvalSpaces(sources: EvalSource[], config: Config): EvalSpace */ export async function checkEvalSpace(space: EvalSpace): Promise { const problems: string[] = []; - const read = await readSpace(space.context); + const read = await space.read(); const ids = new Set(read.nodes.map((n) => n.id as string)); const labelled = new Set(Object.values(space.expected).flatMap((labels) => Object.keys(labels))); const result = await validateNodes(space.context, read.nodes, read); @@ -216,7 +231,7 @@ export async function judgeEvalSpace( }); }; } - const read = await readSpace(space.context); + const read = await space.read(); await validateNodes(space.context, read.nodes, read, { includeGated: true, ruleFilter: { id: Object.keys(space.expected) }, diff --git a/evals/semantic/strict-ost/eval.json5 b/evals/semantic/strict-ost/eval.json5 index 835d431..6c8bc4a 100644 --- a/evals/semantic/strict-ost/eval.json5 +++ b/evals/semantic/strict-ost/eval.json5 @@ -3,22 +3,22 @@ // Rule id → node id → the verdict the node should get. A list accepts any of its verdicts. expected: { 'opportunity-customer-voice': { - 'Setting up a workspace takes so long that I give up.md': 'pass', - "I can't tell which plan fits a team of my size.md": 'pass', - "I'm afraid I'll lose my work if my browser crashes.md": 'pass', - "I don't know whether my changes have been saved.md": 'pass', - 'I want to see what my teammates changed while I was away.md': 'pass', + 'Setting up a workspace takes so long that I give up': 'pass', + "I can't tell which plan fits a team of my size": 'pass', + "I'm afraid I'll lose my work if my browser crashes": 'pass', + "I don't know whether my changes have been saved": 'pass', + 'I want to see what my teammates changed while I was away': 'pass', // A solution in disguise. - 'Add a progress bar to workspace setup.md': 'fail', - 'Build a Slack integration.md': 'fail', - 'Offer a discount on annual plans.md': 'fail', + 'Add a progress bar to workspace setup': 'fail', + 'Build a Slack integration': 'fail', + 'Offer a discount on annual plans': 'fail', // A business metric. - 'Increase trial-to-paid conversion by 10%.md': 'fail', - 'Reduce churn among small teams.md': 'fail', + 'Increase trial-to-paid conversion by 10%': 'fail', + 'Reduce churn among small teams': 'fail', // A topic, not a need. - 'Onboarding.md': 'fail', + 'Onboarding': 'fail', // A real pain point, but in the third person rather than the customer's own words. - 'New users find the editor toolbar confusing.md': ['pass', 'fail'], + 'New users find the editor toolbar confusing': ['pass', 'fail'], }, }, } diff --git a/evals/semantic/strict-ost/nodes.jsonl b/evals/semantic/strict-ost/nodes.jsonl new file mode 100644 index 0000000..30fe631 --- /dev/null +++ b/evals/semantic/strict-ost/nodes.jsonl @@ -0,0 +1,14 @@ +{"id":"Increase weekly active teams","title":"Increase weekly active teams","type":"outcome","linkTargets":["Increase weekly active teams"],"schemaData":{"title":"Increase weekly active teams","type":"outcome","status":"identified","metric":"Increase % of trial teams active in their second week","content":"Teams that use the product every week renew."}} +{"id":"New users find the editor toolbar confusing","title":"New users find the editor toolbar confusing","type":"opportunity","linkTargets":["New users find the editor toolbar confusing"],"schemaData":{"title":"New users find the editor toolbar confusing","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Usability test round 2","content":"Participants hovered over icons without knowing what they did."}} +{"id":"Onboarding","title":"Onboarding","type":"opportunity","linkTargets":["Onboarding"],"schemaData":{"title":"Onboarding","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Interview synthesis","content":"Everything about how new users get started."}} +{"id":"I don't know whether my changes have been saved","title":"I don't know whether my changes have been saved","type":"opportunity","linkTargets":["I don't know whether my changes have been saved"],"schemaData":{"title":"I don't know whether my changes have been saved","type":"opportunity","status":"identified","parent":"[[I'm afraid I'll lose my work if my browser crashes]]","source":"Usability test round 3","content":"Four of six participants pressed the save shortcut repeatedly."}} +{"id":"Add a progress bar to workspace setup","title":"Add a progress bar to workspace setup","type":"opportunity","linkTargets":["Add a progress bar to workspace setup"],"schemaData":{"title":"Add a progress bar to workspace setup","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Product team brainstorm","content":"Show users how many setup steps remain."}} +{"id":"Build a Slack integration","title":"Build a Slack integration","type":"opportunity","linkTargets":["Build a Slack integration"],"schemaData":{"title":"Build a Slack integration","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Competitor review","content":"Post page updates to a Slack channel."}} +{"id":"Setting up a workspace takes so long that I give up","title":"Setting up a workspace takes so long that I give up","type":"opportunity","linkTargets":["Setting up a workspace takes so long that I give up"],"schemaData":{"title":"Setting up a workspace takes so long that I give up","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Interview with Priya, 2026-09-02","content":"Priya abandoned setup twice before her team lead finished it for her."}} +{"id":"Increase trial-to-paid conversion by 10%","title":"Increase trial-to-paid conversion by 10%","type":"opportunity","linkTargets":["Increase trial-to-paid conversion by 10%"],"schemaData":{"title":"Increase trial-to-paid conversion by 10%","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Quarterly planning","content":"The board wants conversion to rise this quarter."}} +{"id":"I can't tell which plan fits a team of my size","title":"I can't tell which plan fits a team of my size","type":"opportunity","linkTargets":["I can't tell which plan fits a team of my size"],"schemaData":{"title":"I can't tell which plan fits a team of my size","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Sales call notes, 2026-09-10","content":"Prospects asked which tier covers twelve seats with guest access."}} +{"id":"I want to see what my teammates changed while I was away","title":"I want to see what my teammates changed while I was away","type":"opportunity","linkTargets":["I want to see what my teammates changed while I was away"],"schemaData":{"title":"I want to see what my teammates changed while I was away","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Interview with Tomas, 2026-09-15","content":"Tomas reads every page again after a holiday to find what is new."}} +{"id":"I'm afraid I'll lose my work if my browser crashes","title":"I'm afraid I'll lose my work if my browser crashes","type":"opportunity","linkTargets":["I'm afraid I'll lose my work if my browser crashes"],"schemaData":{"title":"I'm afraid I'll lose my work if my browser crashes","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Support tickets, September 2026","content":"Several tickets describe retyping long notes after a crash."}} +{"id":"Offer a discount on annual plans","title":"Offer a discount on annual plans","type":"opportunity","linkTargets":["Offer a discount on annual plans"],"schemaData":{"title":"Offer a discount on annual plans","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Pricing workshop","content":"Ten percent off when paying yearly."}} +{"id":"Reduce churn among small teams","title":"Reduce churn among small teams","type":"opportunity","linkTargets":["Reduce churn among small teams"],"schemaData":{"title":"Reduce churn among small teams","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Retention dashboard","content":"Teams of fewer than five seats churn twice as often."}} +{"id":"Show unsaved-change indicator","title":"Show unsaved-change indicator","type":"solution","linkTargets":["Show unsaved-change indicator"],"schemaData":{"title":"Show unsaved-change indicator","type":"solution","parent":"[[I don't know whether my changes have been saved]]","status":"exploring","content":"A dot on the tab while changes are pending."}} diff --git a/evals/semantic/strict-ost/space/Add a progress bar to workspace setup.md b/evals/semantic/strict-ost/space/Add a progress bar to workspace setup.md deleted file mode 100644 index c7e28b9..0000000 --- a/evals/semantic/strict-ost/space/Add a progress bar to workspace setup.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -type: opportunity -status: identified -parent: "[[Increase weekly active teams]]" -source: Product team brainstorm ---- - -Show users how many setup steps remain. diff --git a/evals/semantic/strict-ost/space/Build a Slack integration.md b/evals/semantic/strict-ost/space/Build a Slack integration.md deleted file mode 100644 index 98db002..0000000 --- a/evals/semantic/strict-ost/space/Build a Slack integration.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -type: opportunity -status: identified -parent: "[[Increase weekly active teams]]" -source: Competitor review ---- - -Post page updates to a Slack channel. diff --git a/evals/semantic/strict-ost/space/I can't tell which plan fits a team of my size.md b/evals/semantic/strict-ost/space/I can't tell which plan fits a team of my size.md deleted file mode 100644 index 10e8fd9..0000000 --- a/evals/semantic/strict-ost/space/I can't tell which plan fits a team of my size.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -type: opportunity -status: identified -parent: "[[Increase weekly active teams]]" -source: Sales call notes, 2026-09-10 ---- - -Prospects asked which tier covers twelve seats with guest access. diff --git a/evals/semantic/strict-ost/space/I don't know whether my changes have been saved.md b/evals/semantic/strict-ost/space/I don't know whether my changes have been saved.md deleted file mode 100644 index 866a1fd..0000000 --- a/evals/semantic/strict-ost/space/I don't know whether my changes have been saved.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -type: opportunity -status: identified -parent: "[[I'm afraid I'll lose my work if my browser crashes]]" -source: Usability test round 3 ---- - -Four of six participants pressed the save shortcut repeatedly. diff --git a/evals/semantic/strict-ost/space/I want to see what my teammates changed while I was away.md b/evals/semantic/strict-ost/space/I want to see what my teammates changed while I was away.md deleted file mode 100644 index bbe2c6e..0000000 --- a/evals/semantic/strict-ost/space/I want to see what my teammates changed while I was away.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -type: opportunity -status: identified -parent: "[[Increase weekly active teams]]" -source: Interview with Tomas, 2026-09-15 ---- - -Tomas reads every page again after a holiday to find what is new. diff --git a/evals/semantic/strict-ost/space/I'm afraid I'll lose my work if my browser crashes.md b/evals/semantic/strict-ost/space/I'm afraid I'll lose my work if my browser crashes.md deleted file mode 100644 index d806b16..0000000 --- a/evals/semantic/strict-ost/space/I'm afraid I'll lose my work if my browser crashes.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -type: opportunity -status: identified -parent: "[[Increase weekly active teams]]" -source: Support tickets, September 2026 ---- - -Several tickets describe retyping long notes after a crash. diff --git a/evals/semantic/strict-ost/space/Increase trial-to-paid conversion by 10%.md b/evals/semantic/strict-ost/space/Increase trial-to-paid conversion by 10%.md deleted file mode 100644 index 621c236..0000000 --- a/evals/semantic/strict-ost/space/Increase trial-to-paid conversion by 10%.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -type: opportunity -status: identified -parent: "[[Increase weekly active teams]]" -source: Quarterly planning ---- - -The board wants conversion to rise this quarter. diff --git a/evals/semantic/strict-ost/space/Increase weekly active teams.md b/evals/semantic/strict-ost/space/Increase weekly active teams.md deleted file mode 100644 index ef84cd0..0000000 --- a/evals/semantic/strict-ost/space/Increase weekly active teams.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: outcome -status: identified -metric: Increase % of trial teams active in their second week ---- - -Teams that use the product every week renew. diff --git a/evals/semantic/strict-ost/space/New users find the editor toolbar confusing.md b/evals/semantic/strict-ost/space/New users find the editor toolbar confusing.md deleted file mode 100644 index 1f1e585..0000000 --- a/evals/semantic/strict-ost/space/New users find the editor toolbar confusing.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -type: opportunity -status: identified -parent: "[[Increase weekly active teams]]" -source: Usability test round 2 ---- - -Participants hovered over icons without knowing what they did. diff --git a/evals/semantic/strict-ost/space/Offer a discount on annual plans.md b/evals/semantic/strict-ost/space/Offer a discount on annual plans.md deleted file mode 100644 index c70b935..0000000 --- a/evals/semantic/strict-ost/space/Offer a discount on annual plans.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -type: opportunity -status: identified -parent: "[[Increase weekly active teams]]" -source: Pricing workshop ---- - -Ten percent off when paying yearly. diff --git a/evals/semantic/strict-ost/space/Onboarding.md b/evals/semantic/strict-ost/space/Onboarding.md deleted file mode 100644 index 1fc9d5a..0000000 --- a/evals/semantic/strict-ost/space/Onboarding.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -type: opportunity -status: identified -parent: "[[Increase weekly active teams]]" -source: Interview synthesis ---- - -Everything about how new users get started. diff --git a/evals/semantic/strict-ost/space/Reduce churn among small teams.md b/evals/semantic/strict-ost/space/Reduce churn among small teams.md deleted file mode 100644 index 6184ca8..0000000 --- a/evals/semantic/strict-ost/space/Reduce churn among small teams.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -type: opportunity -status: identified -parent: "[[Increase weekly active teams]]" -source: Retention dashboard ---- - -Teams of fewer than five seats churn twice as often. diff --git a/evals/semantic/strict-ost/space/Setting up a workspace takes so long that I give up.md b/evals/semantic/strict-ost/space/Setting up a workspace takes so long that I give up.md deleted file mode 100644 index 900f090..0000000 --- a/evals/semantic/strict-ost/space/Setting up a workspace takes so long that I give up.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -type: opportunity -status: identified -parent: "[[Increase weekly active teams]]" -source: Interview with Priya, 2026-09-02 ---- - -Priya abandoned setup twice before her team lead finished it for her. diff --git a/evals/semantic/strict-ost/space/Show unsaved-change indicator.md b/evals/semantic/strict-ost/space/Show unsaved-change indicator.md deleted file mode 100644 index cf4748e..0000000 --- a/evals/semantic/strict-ost/space/Show unsaved-change indicator.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: solution -parent: "[[I don't know whether my changes have been saved]]" -status: exploring ---- - -A dot on the tab while changes are pending. diff --git a/tests/evals/semantic.test.ts b/tests/evals/semantic.test.ts index e7a3c62..c266e29 100644 --- a/tests/evals/semantic.test.ts +++ b/tests/evals/semantic.test.ts @@ -3,7 +3,6 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { basename, join } from 'node:path'; import { - CORE_EVALS_DIR, checkEvalSpace, type EvalSpace, type Judgement, @@ -25,7 +24,14 @@ describe.each(spaces.map((s) => [s.name, s] as const))('eval space %s', (_name, describe('eval spaces on a configured space', () => { const dir = mkdtempSync(join(tmpdir(), 'sctx-evals-')); - const fixture = join(CORE_EVALS_DIR, 'strict-ost', 'space'); + const fixture = join(dir, 'space'); + const note = (title: string, frontmatter: string) => + writeFileSync(join(fixture, `${title}.md`), `---\nstatus: identified\n${frontmatter}\n---\n`); + mkdirSync(fixture); + note('Grow usage', 'type: outcome\nmetric: Weekly active teams'); + for (const title of ['Onboarding', 'Build a Slack integration', 'I lose track of changes']) { + note(title, 'type: opportunity\nparent: "[[Grow usage]]"\nsource: Interviews'); + } mkdirSync(join(dir, 'mine')); writeFileSync( join(dir, 'mine', 'eval.json5'), From 3e82c141d9af87179244d038d743afe20a476d2e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 23:02:23 +0000 Subject: [PATCH 5/7] feat(evals): fixture lines are plain schema objects Each line of nodes.jsonl holds the fields frontmatter would, with its title as node id, matching the JSON node idea in #112. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019Xphpdioyw95tAT6PEKGTU --- evals/semantic/README.md | 2 +- evals/semantic/harness.ts | 23 ++++++++++++---------- evals/semantic/strict-ost/nodes.jsonl | 28 +++++++++++++-------------- 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/evals/semantic/README.md b/evals/semantic/README.md index 249b9a8..ebe3174 100644 --- a/evals/semantic/README.md +++ b/evals/semantic/README.md @@ -48,7 +48,7 @@ Each source is a directory of eval spaces: An eval space is a directory with an `eval.json5`. It is one of two kinds. -A **fixture** eval space has its own nodes in a `nodes.jsonl` beside `eval.json5`: one parsed node per line, as a plugin's parse hook returns it (`id`, `title`, `type`, `linkTargets` and `schemaData`). JSON keeps fixtures out of the markdown that editors and spaces read. To start from markdown notes, `sctx dump | jq -c '.nodes[]'` prints nodes in this shape. The eval space names its `schema`, and optionally `plugins`, as a space config does: +A **fixture** eval space has its own nodes in a `nodes.jsonl` beside `eval.json5`: one JSON object per line, with the fields frontmatter would hold. Each needs a `title`, which is also its node id, and a `type`. A body goes in `content`. JSON keeps fixtures out of the markdown that editors and spaces read. To start from markdown notes, `sctx dump | jq -c '.nodes[].schemaData'` prints them in this shape. The eval space names its `schema`, and optionally `plugins`, as a space config does: ```json5 { diff --git a/evals/semantic/harness.ts b/evals/semantic/harness.ts index 6532ac1..fcfcd2a 100644 --- a/evals/semantic/harness.ts +++ b/evals/semantic/harness.ts @@ -10,7 +10,7 @@ import { readSpace } from '../../src/read/read-space'; import { resolveGraphEdges } from '../../src/read/resolve-graph-edges'; import { RESERVED_VERDICTS } from '../../src/schema/metadata-contract'; import { createSpaceContext } from '../../src/space-context'; -import type { BaseNode, ReadSpaceResult, SpaceContext } from '../../src/types'; +import type { BaseNode, NodeId, ReadSpaceResult, SpaceContext } from '../../src/types'; import { validateNodes } from '../../src/validate'; /** Core's eval spaces, for the bundled schemas. */ @@ -72,7 +72,7 @@ export function resolveEvalSources(source: string, configDirs: string[]): EvalSo /** * An eval space's `eval.json5`. A fixture eval space names a `schema` (and optional `plugins`), as a space config - * does, and keeps its nodes in `nodes.jsonl` beside the file, one parsed node per line. An eval space on a + * does, and keeps its nodes in `nodes.jsonl` beside the file, one schema object per line. An eval space on a * configured space names the `space` instead, and labels some of its nodes. */ type EvalFile = Pick & { @@ -122,17 +122,20 @@ export function loadEvalSpaces(sources: EvalSource[], config: Config): EvalSpace ); } -/** Parsed nodes, one JSON object per line: the nodes a plugin's parse hook returns. */ +/** + * Fixture nodes: one schema object per line, as frontmatter would hold it, with its `title` as the node id and + * link target. + */ function readNodes(path: string): BaseNode[] { return readFileSync(path, 'utf8') .split('\n') - .filter((line) => line.trim() !== '') - .map((line, i) => { - const node = JSON.parse(line) as BaseNode; - for (const key of ['id', 'title', 'type', 'linkTargets', 'schemaData'] as const) { - if (node[key] === undefined) throw new Error(`${path}:${i + 1}: node has no "${key}"`); - } - return node; + .map((line, i) => ({ line, at: `${path}:${i + 1}` })) + .filter(({ line }) => line.trim() !== '') + .map(({ line, at }) => { + const schemaData = JSON.parse(line) as Record; + const { title, type } = schemaData; + if (typeof title !== 'string' || typeof type !== 'string') throw new Error(`${at}: needs a "title" and a "type"`); + return { id: title as NodeId, title, type, linkTargets: [title], schemaData }; }); } diff --git a/evals/semantic/strict-ost/nodes.jsonl b/evals/semantic/strict-ost/nodes.jsonl index 30fe631..745676e 100644 --- a/evals/semantic/strict-ost/nodes.jsonl +++ b/evals/semantic/strict-ost/nodes.jsonl @@ -1,14 +1,14 @@ -{"id":"Increase weekly active teams","title":"Increase weekly active teams","type":"outcome","linkTargets":["Increase weekly active teams"],"schemaData":{"title":"Increase weekly active teams","type":"outcome","status":"identified","metric":"Increase % of trial teams active in their second week","content":"Teams that use the product every week renew."}} -{"id":"New users find the editor toolbar confusing","title":"New users find the editor toolbar confusing","type":"opportunity","linkTargets":["New users find the editor toolbar confusing"],"schemaData":{"title":"New users find the editor toolbar confusing","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Usability test round 2","content":"Participants hovered over icons without knowing what they did."}} -{"id":"Onboarding","title":"Onboarding","type":"opportunity","linkTargets":["Onboarding"],"schemaData":{"title":"Onboarding","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Interview synthesis","content":"Everything about how new users get started."}} -{"id":"I don't know whether my changes have been saved","title":"I don't know whether my changes have been saved","type":"opportunity","linkTargets":["I don't know whether my changes have been saved"],"schemaData":{"title":"I don't know whether my changes have been saved","type":"opportunity","status":"identified","parent":"[[I'm afraid I'll lose my work if my browser crashes]]","source":"Usability test round 3","content":"Four of six participants pressed the save shortcut repeatedly."}} -{"id":"Add a progress bar to workspace setup","title":"Add a progress bar to workspace setup","type":"opportunity","linkTargets":["Add a progress bar to workspace setup"],"schemaData":{"title":"Add a progress bar to workspace setup","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Product team brainstorm","content":"Show users how many setup steps remain."}} -{"id":"Build a Slack integration","title":"Build a Slack integration","type":"opportunity","linkTargets":["Build a Slack integration"],"schemaData":{"title":"Build a Slack integration","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Competitor review","content":"Post page updates to a Slack channel."}} -{"id":"Setting up a workspace takes so long that I give up","title":"Setting up a workspace takes so long that I give up","type":"opportunity","linkTargets":["Setting up a workspace takes so long that I give up"],"schemaData":{"title":"Setting up a workspace takes so long that I give up","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Interview with Priya, 2026-09-02","content":"Priya abandoned setup twice before her team lead finished it for her."}} -{"id":"Increase trial-to-paid conversion by 10%","title":"Increase trial-to-paid conversion by 10%","type":"opportunity","linkTargets":["Increase trial-to-paid conversion by 10%"],"schemaData":{"title":"Increase trial-to-paid conversion by 10%","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Quarterly planning","content":"The board wants conversion to rise this quarter."}} -{"id":"I can't tell which plan fits a team of my size","title":"I can't tell which plan fits a team of my size","type":"opportunity","linkTargets":["I can't tell which plan fits a team of my size"],"schemaData":{"title":"I can't tell which plan fits a team of my size","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Sales call notes, 2026-09-10","content":"Prospects asked which tier covers twelve seats with guest access."}} -{"id":"I want to see what my teammates changed while I was away","title":"I want to see what my teammates changed while I was away","type":"opportunity","linkTargets":["I want to see what my teammates changed while I was away"],"schemaData":{"title":"I want to see what my teammates changed while I was away","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Interview with Tomas, 2026-09-15","content":"Tomas reads every page again after a holiday to find what is new."}} -{"id":"I'm afraid I'll lose my work if my browser crashes","title":"I'm afraid I'll lose my work if my browser crashes","type":"opportunity","linkTargets":["I'm afraid I'll lose my work if my browser crashes"],"schemaData":{"title":"I'm afraid I'll lose my work if my browser crashes","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Support tickets, September 2026","content":"Several tickets describe retyping long notes after a crash."}} -{"id":"Offer a discount on annual plans","title":"Offer a discount on annual plans","type":"opportunity","linkTargets":["Offer a discount on annual plans"],"schemaData":{"title":"Offer a discount on annual plans","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Pricing workshop","content":"Ten percent off when paying yearly."}} -{"id":"Reduce churn among small teams","title":"Reduce churn among small teams","type":"opportunity","linkTargets":["Reduce churn among small teams"],"schemaData":{"title":"Reduce churn among small teams","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Retention dashboard","content":"Teams of fewer than five seats churn twice as often."}} -{"id":"Show unsaved-change indicator","title":"Show unsaved-change indicator","type":"solution","linkTargets":["Show unsaved-change indicator"],"schemaData":{"title":"Show unsaved-change indicator","type":"solution","parent":"[[I don't know whether my changes have been saved]]","status":"exploring","content":"A dot on the tab while changes are pending."}} +{"title": "Increase weekly active teams", "type": "outcome", "status": "identified", "metric": "Increase % of trial teams active in their second week", "content": "Teams that use the product every week renew."} +{"title": "New users find the editor toolbar confusing", "type": "opportunity", "status": "identified", "parent": "[[Increase weekly active teams]]", "source": "Usability test round 2", "content": "Participants hovered over icons without knowing what they did."} +{"title": "Onboarding", "type": "opportunity", "status": "identified", "parent": "[[Increase weekly active teams]]", "source": "Interview synthesis", "content": "Everything about how new users get started."} +{"title": "I don't know whether my changes have been saved", "type": "opportunity", "status": "identified", "parent": "[[I'm afraid I'll lose my work if my browser crashes]]", "source": "Usability test round 3", "content": "Four of six participants pressed the save shortcut repeatedly."} +{"title": "Add a progress bar to workspace setup", "type": "opportunity", "status": "identified", "parent": "[[Increase weekly active teams]]", "source": "Product team brainstorm", "content": "Show users how many setup steps remain."} +{"title": "Build a Slack integration", "type": "opportunity", "status": "identified", "parent": "[[Increase weekly active teams]]", "source": "Competitor review", "content": "Post page updates to a Slack channel."} +{"title": "Setting up a workspace takes so long that I give up", "type": "opportunity", "status": "identified", "parent": "[[Increase weekly active teams]]", "source": "Interview with Priya, 2026-09-02", "content": "Priya abandoned setup twice before her team lead finished it for her."} +{"title": "Increase trial-to-paid conversion by 10%", "type": "opportunity", "status": "identified", "parent": "[[Increase weekly active teams]]", "source": "Quarterly planning", "content": "The board wants conversion to rise this quarter."} +{"title": "I can't tell which plan fits a team of my size", "type": "opportunity", "status": "identified", "parent": "[[Increase weekly active teams]]", "source": "Sales call notes, 2026-09-10", "content": "Prospects asked which tier covers twelve seats with guest access."} +{"title": "I want to see what my teammates changed while I was away", "type": "opportunity", "status": "identified", "parent": "[[Increase weekly active teams]]", "source": "Interview with Tomas, 2026-09-15", "content": "Tomas reads every page again after a holiday to find what is new."} +{"title": "I'm afraid I'll lose my work if my browser crashes", "type": "opportunity", "status": "identified", "parent": "[[Increase weekly active teams]]", "source": "Support tickets, September 2026", "content": "Several tickets describe retyping long notes after a crash."} +{"title": "Offer a discount on annual plans", "type": "opportunity", "status": "identified", "parent": "[[Increase weekly active teams]]", "source": "Pricing workshop", "content": "Ten percent off when paying yearly."} +{"title": "Reduce churn among small teams", "type": "opportunity", "status": "identified", "parent": "[[Increase weekly active teams]]", "source": "Retention dashboard", "content": "Teams of fewer than five seats churn twice as often."} +{"title": "Show unsaved-change indicator", "type": "solution", "parent": "[[I don't know whether my changes have been saved]]", "status": "exploring", "content": "A dot on the tab while changes are pending."} From 887bbd3611924bf1dc007a28386359f56bb1102e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 03:05:32 +0000 Subject: [PATCH 6/7] refactor(evals): fixtures, evals and suites Replaces "eval spaces" with three parts: a fixture (schema and nodes, shared by evals), an eval (one file of labelled cases on a fixture or configured space), and a suite (the directory --from names). Harness code moves out of the suite directory and splits into load, judge and score. - Unlabelled nodes are allowed as context; the check now reports labels the rule never judges and labels that accept every verdict, from one fake-judge validation pass. - The report lists the agreeing cases with the lowest p(expected). - Fixtures are JSON5, so scenarios can carry comments. - Problems print without a stack trace. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019XphpdrUMvqf8fuj8uG7JZ --- AGENTS.md | 4 +- evals/README.md | 85 +++++ evals/judge.ts | 98 ++++++ evals/load.ts | 107 ++++++ evals/run.ts | 99 ++++++ evals/score.ts | 79 +++++ evals/semantic/README.md | 74 ---- evals/semantic/fixtures/strict-ost.json5 | 26 ++ evals/semantic/harness.ts | 320 ------------------ ...json5 => opportunity-customer-voice.json5} | 5 +- evals/semantic/run.ts | 109 ------ evals/semantic/strict-ost/nodes.jsonl | 14 - package.json | 2 +- tests/evals/semantic.test.ts | 141 ++++---- 14 files changed, 576 insertions(+), 587 deletions(-) create mode 100644 evals/README.md create mode 100644 evals/judge.ts create mode 100644 evals/load.ts create mode 100644 evals/run.ts create mode 100644 evals/score.ts delete mode 100644 evals/semantic/README.md create mode 100644 evals/semantic/fixtures/strict-ost.json5 delete mode 100644 evals/semantic/harness.ts rename evals/semantic/{strict-ost/eval.json5 => opportunity-customer-voice.json5} (73%) delete mode 100644 evals/semantic/run.ts delete mode 100644 evals/semantic/strict-ost/nodes.jsonl diff --git a/AGENTS.md b/AGENTS.md index 74f4166..ad2aac8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,7 +45,7 @@ There are several places that need reviewing and updating with any new feature o |---|---| | CLI command, flag or output | `docs/*` for the command; `plugin/skills/structured-context/references/commands.md`; the command list in `plugin/skills/structured-context/SKILL.md`; `plugin/commands/*` | | Rule, schema or `$metadata` shape | `docs/schemas.md`, `docs/rules.md`, `docs/semantic-rules.md`; `references/schema-authoring.md`, `references/schema-design.md`; bundled `schemas/` | -| Bundled semantic rule | its eval cases in `evals/semantic/` | +| Bundled semantic rule | its eval in `evals/semantic/` | | User-visible error or warning message | the troubleshooting table in `SKILL.md` | | Config key or space config | `docs/config.md`; the config example in `references/commands.md` | | `validate --json` output | `plugin/scripts/state.ts` (the hooks parse it), `scripts/mod-contract.ts` and `bun run test:hook` | @@ -100,7 +100,7 @@ For most development only the main unit tests need re-running regularly. - `bun run test:package` — builds and packs the package, installs the tarball, and runs the installed CLI. Runs in CI and before every version bump; run it after changing the build, `tsconfig*.json` or `files`. - `bun run test:judge` — live judge tests (`tests/judge/live.test.ts`): one real provider call per provider whose API key is set. Costs money; use sparingly when changing judges or semantic rule behaviour. - `bun run test:models` — check that every bundled provider's configured model exists (no API keys) -- `bun run eval:semantic --provider ` — semantic evals (`evals/semantic/`): how often a provider's verdicts match labelled eval spaces, and how stable they are. Core's cover the bundled schemas; `--from` runs a plugin's, or a directory's. Costs money (`--check` doesn't); run it when changing a bundled semantic rule, the schema descriptions it draws on, or a judge. +- `bun run eval:semantic --provider ` — semantic evals (`evals/`): how often a provider's verdicts match labelled cases. Costs money (`--check` doesn't). Run it when changing a bundled semantic rule, the schema descriptions it draws on, or a judge. ## Debugging diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 0000000..32dbe4c --- /dev/null +++ b/evals/README.md @@ -0,0 +1,85 @@ +# Semantic evals + +Semantic evals measure how well model providers judge semantic rules. A case is a node, a rule, and the verdict the node should get. An eval holds the cases on one set of nodes: a fixture, or a configured space. + +## Run the evals + +The run calls real providers, so it costs money. Name each provider to run. Config `modelProviders` entries work as well as bundled providers. + +```bash +bun run eval:semantic --provider openrouter-jev --provider anthropic +``` + +From outside this repo, run `bun run /evals/run.ts` with the same flags. + +| Flag | Purpose | +|---|---| +| `--provider ` | The provider to judge with. Repeat it, or give a comma-separated list, to compare providers | +| `--from ` | The suite to run. Repeatable. Default: `core`. See [Suites](#suites) | +| `--eval ` | Run only this eval, as `core/opportunity-customer-voice` or `opportunity-customer-voice`. Repeatable | +| `--runs ` | Judge each eval this many times, to measure stability. Default: 3 | +| `--per-node` | Send one request per node, even to a provider that packs several nodes into one request | +| `--check` | Check the evals and stop, with no provider calls | +| `--json` | Print every judgement and score as JSON instead of the report | + +The report has one row per eval, provider and rule: + +| Column | Meaning | +|---|---| +| Agreement | Share of judgements, over every case and run, that match the label | +| Stable | Share of cases that got the same verdict on every run | +| p(expected) | Mean probability the provider gave the labelled verdicts. Only decision models report it | +| Reserved | Judgements with a reserved verdict, such as `missing-context` | +| Errors | Cases that got an evaluation error, or no judgement, on some run | + +Below the table, the report lists each miss, and the agreeing cases with the lowest p(expected). Those are the cases nearest to a miss. + +Each run validates through the same pipeline as `sctx validate --semantic`, with only the labelled rules, and evaluates gated nodes too. Only labelled nodes are judged. Other nodes still reach the model as related nodes, when a rule's `scope` includes them. + +## Suites + +A suite is a directory of evals, with their fixtures in `fixtures/`: + +``` +evals/semantic/ + opportunity-customer-voice.json5 + fixtures/ + strict-ost.json5 +``` + +| `--from` | Directory | Schema names resolve from | +|---|---|---| +| `core` | `evals/semantic/` in this repo | the bundled schemas | +| `` | `evals/semantic/` in the plugin, found as the config finds plugins | the config that finds the plugin | +| a path | that directory | that directory | + +## Add an eval + +An eval is a `.json5` file in the suite. It names a `fixture` in the suite, or a configured `space`. `expected` maps a rule id to node ids and their verdicts: + +```json5 +{ + fixture: 'strict-ost', + expected: { + 'opportunity-customer-voice': { + 'Build a Slack integration': 'fail', + // A solution in disguise. + 'Add a progress bar to workspace setup': 'fail', + }, + }, +} +``` + +A list of verdicts accepts any of them. On a configured space, node ids are file paths: `{ space: 'work', expected: { 'opportunity-customer-voice': { 'Onboarding.md': 'fail' } } }`. + +Write a comment on a case whose label is not obvious. We recommend one eval per rule, so each rule's cases are easy to find and to run alone. + +## Add a fixture + +A fixture is `fixtures/.json5`: a `schema`, optional `plugins` as in a space config, and `nodes`. Each node holds the fields its frontmatter would, with a body in `content`. Its `title` is its node id and link target. `sctx dump | jq '.nodes | map(.schemaData)'` prints notes in this shape. + +Several evals can share a fixture. For a rule whose `scope` sends related nodes, such as siblings, give each scenario its own small tree, so that one case's related nodes do not change another's. + +## Checks + +`--check` finds the problems that would make scores meaningless: a label on a node the rule does not judge, an unknown rule or verdict, a label that accepts every verdict, and a labelled node that fails validation. Every run checks first. `bun run test` checks core's evals. diff --git a/evals/judge.ts b/evals/judge.ts new file mode 100644 index 0000000..0f299f0 --- /dev/null +++ b/evals/judge.ts @@ -0,0 +1,98 @@ +import { FakeJudgeProvider } from '../src/judge/provider/fake'; +import type { SemanticJudgeInput, SemanticJudgeProvider, SemanticJudgeResult } from '../src/judge/types'; +import { resolveSemanticRule } from '../src/judge/verdicts'; +import { validateNodes } from '../src/validate'; +import type { Eval } from './load'; + +/** One verdict (or error) the judge gave for a node and rule. */ +export interface Judgement { + nodeId: string; + ruleId: string; + result: SemanticJudgeResult; +} + +/** Validate with only the eval's labelled rules, judging gated nodes too, as `sctx validate --semantic` would. */ +const validate = async (evaluation: Eval, provider: SemanticJudgeProvider) => { + const read = await evaluation.read(); + return validateNodes(evaluation.context, read.nodes, read, { + includeGated: true, + ruleFilter: { id: Object.keys(evaluation.expected) }, + judge: { provider }, + }); +}; + +/** + * Problems that would make an eval's scores meaningless, found before any paid call. A labelled node's + * validation errors count, because its rule input would not match a real space. + */ +export async function checkEval(evaluation: Eval): Promise { + const problems: string[] = []; + const judge = new FakeJudgeProvider(); + const result = await validate(evaluation, judge); + const judged = new Set(judge.callHistory.flatMap((input) => input.rules.map((r) => `${r.id}\0${input.node.id}`))); + const labelled = new Set(Object.values(evaluation.expected).flatMap(Object.keys)); + for (const issue of result.issues) { + if (issue.severity === 'error' && issue.nodeId && labelled.has(issue.nodeId)) { + problems.push(`${issue.nodeId}: ${issue.message}`); + } + } + const rules = evaluation.context.schema.metadata.rules ?? []; + for (const [ruleId, labels] of Object.entries(evaluation.expected)) { + const rule = rules.find((r) => r.id === ruleId); + if (!rule?.semantic) { + problems.push(`rule "${ruleId}": not a semantic rule of the schema`); + continue; + } + const verdicts = Object.keys(resolveSemanticRule(rule).spec.verdicts); + for (const [nodeId, expected] of Object.entries(labels)) { + const at = `rule "${ruleId}", node "${nodeId}"`; + if (!judged.has(`${ruleId}\0${nodeId}`)) problems.push(`${at}: not judged (unknown node, or the rule skips it)`); + const unknown = expected.filter((v) => !verdicts.includes(v)); + if (unknown.length > 0) problems.push(`${at}: ${unknown.join(', ')} not a verdict (${verdicts.join(', ')})`); + else if (verdicts.every((v) => expected.includes(v))) problems.push(`${at}: accepts every verdict`); + } + } + return problems.map((p) => `${evaluation.name}: ${p}`); +} + +/** + * Judge the eval's labelled cases, recording every result the provider returns. Unlabelled nodes still reach + * the input as related nodes but are not judged themselves. `perNode` hides `evaluateNodes`, so a provider that + * packs nodes into one request gets one request per node instead. + */ +export async function judgeEval( + evaluation: Eval, + provider: SemanticJudgeProvider, + options: { perNode?: boolean } = {}, +): Promise { + const judgements: Judgement[] = []; + const labelledOnly = (input: SemanticJudgeInput): SemanticJudgeInput | undefined => { + const rules = input.rules.filter((rule) => evaluation.expected[rule.id]?.[input.node.id] !== undefined); + return rules.length > 0 ? { ...input, rules } : undefined; + }; + const record = (input: SemanticJudgeInput, results: Record) => { + for (const rule of input.rules) { + judgements.push({ nodeId: input.node.id, ruleId: rule.id, result: results[rule.id] ?? { error: 'no result' } }); + } + return results; + }; + const recording: SemanticJudgeProvider = { + name: provider.name, + async evaluate(input) { + const sent = labelledOnly(input); + return sent ? record(sent, await provider.evaluate(sent)) : {}; + }, + }; + const { evaluateNodes } = provider; + if (evaluateNodes && !options.perNode) { + recording.evaluateNodes = async (inputs) => { + const sent = inputs.map(labelledOnly); + const kept = sent.filter((input) => input !== undefined); + const results = kept.length > 0 ? await evaluateNodes.call(provider, kept) : []; + let k = 0; + return sent.map((input) => (input ? record(input, results[k++] ?? {}) : {})); + }; + } + await validate(evaluation, recording); + return judgements; +} diff --git a/evals/load.ts b/evals/load.ts new file mode 100644 index 0000000..1993815 --- /dev/null +++ b/evals/load.ts @@ -0,0 +1,107 @@ +import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { basename, join, resolve } from 'node:path'; +import JSON5 from 'json5'; +import type { Config, SpaceConfig } from '../src/config'; +import { resolvePluginDir } from '../src/plugins/util'; +import { readSpace } from '../src/read/read-space'; +import { resolveGraphEdges } from '../src/read/resolve-graph-edges'; +import { createSpaceContext } from '../src/space-context'; +import type { BaseNode, NodeId, SpaceContext } from '../src/types'; + +/** Where core and each plugin keep their semantic suite. */ +export const SUITE_SUBDIR = join('evals', 'semantic'); + +const CORE_SUITE_DIR = join(import.meta.dir, 'semantic'); + +/** A directory of evals and their fixtures, and the directory schema names resolve from. */ +export interface Suite { + name: string; + dir: string; + configDir: string; +} + +/** Rule id → node id → the verdicts that count as agreeing. */ +export type Expected = Record>; + +/** Labelled cases on the nodes of one fixture or configured space. */ +export interface Eval { + /** `/`, e.g. `core/opportunity-customer-voice`. */ + name: string; + context: SpaceContext; + expected: Expected; + read: () => Promise>; +} + +/** `core`, a plugin name, or a path to a suite directory. */ +export function resolveSuite(source: string, configDirs: string[]): Suite { + if (source === 'core') return { name: 'core', dir: CORE_SUITE_DIR, configDir: CORE_SUITE_DIR }; + if (source.includes('/') || source.includes('\\') || source.startsWith('.')) { + const dir = resolve(source); + if (!existsSync(dir)) throw new Error(`Suite "${source}": no directory at ${dir}`); + return { name: basename(dir), dir, configDir: dir }; + } + for (const configDir of configDirs) { + const pluginDir = resolvePluginDir(source, configDir); + if (!pluginDir) continue; + const dir = join(pluginDir, SUITE_SUBDIR); + if (!existsSync(dir)) throw new Error(`Plugin "${source}" has no ${SUITE_SUBDIR}/ directory (${pluginDir})`); + return { name: source, dir, configDir }; + } + throw new Error(`Suite "${source}" is not core, a path, or a plugin found from the config.`); +} + +type EvalFile = { fixture?: string; space?: string; expected: Record> }; +type FixtureFile = Pick & { nodes: Record[] }; + +/** Every `*.json5` at the top of each suite is an eval. `config` holds the spaces evals name. */ +export function loadEvals(suites: Suite[], config: Config): Eval[] { + return suites.flatMap((suite) => { + const fixtures = new Map>(); + const fixture = (name: string) => { + if (!fixtures.has(name)) fixtures.set(name, loadFixture(suite, name)); + return fixtures.get(name)!; + }; + return readdirSync(suite.dir) + .filter((file) => file.endsWith('.json5')) + .sort() + .map((file) => { + const name = `${suite.name}/${basename(file, '.json5')}`; + try { + const evalFile = JSON5.parse(readFileSync(join(suite.dir, file), 'utf8')) as EvalFile; + if ((evalFile.fixture === undefined) === (evalFile.space === undefined)) { + throw new Error('needs either "fixture" or "space"'); + } + const expected: Expected = {}; + for (const [ruleId, labels] of Object.entries(evalFile.expected)) { + expected[ruleId] = Object.fromEntries(Object.entries(labels).map(([id, v]) => [id, [v].flat()])); + } + if (evalFile.fixture !== undefined) return { name, expected, ...fixture(evalFile.fixture) }; + const context = createSpaceContext(evalFile.space!, config); + return { name, expected, context, read: () => readSpace(context) }; + } catch (err) { + throw new Error(`Eval ${name}: ${err instanceof Error ? err.message : String(err)}`); + } + }); + }); +} + +/** `fixtures/.json5`: a schema and the nodes' schema data, each with its `title` as node id and link target. */ +function loadFixture(suite: Suite, name: string): Pick { + const path = join(suite.dir, 'fixtures', `${name}.json5`); + if (!existsSync(path)) throw new Error(`no fixture at ${path}`); + const file = JSON5.parse(readFileSync(path, 'utf8')) as FixtureFile; + const spaceName = `${suite.name}/fixtures/${name}`; + const context = createSpaceContext( + spaceName, + { spaces: [{ name: spaceName, path: suite.dir, schema: file.schema, plugins: file.plugins }] }, + { configDir: suite.configDir }, + ); + const nodes: BaseNode[] = file.nodes.map((schemaData, i) => { + const { title, type } = schemaData; + if (typeof title !== 'string' || typeof type !== 'string') { + throw new Error(`${path}: node ${i + 1} needs a "title" and a "type"`); + } + return { id: title as NodeId, title, type, linkTargets: [title], schemaData }; + }); + return { context, read: async () => resolveGraphEdges(nodes, context.schema.metadata) }; +} diff --git a/evals/run.ts b/evals/run.ts new file mode 100644 index 0000000..3a4581a --- /dev/null +++ b/evals/run.ts @@ -0,0 +1,99 @@ +/** Semantic evals: judge labelled cases with real providers and report agreement. See README.md. */ +import { dirname } from 'node:path'; +import { parseArgs } from 'node:util'; +import { configPath, loadConfigOrEmpty } from '../src/config'; +import { createJudgeProvider } from '../src/judge/factory'; +import { checkEval, type Judgement, judgeEval } from './judge'; +import { loadEvals, resolveSuite } from './load'; +import { type RuleScore, scoreEval } from './score'; + +const { values } = parseArgs({ + options: { + from: { type: 'string', multiple: true }, + provider: { type: 'string', multiple: true }, + eval: { type: 'string', multiple: true }, + runs: { type: 'string', default: '3' }, + 'per-node': { type: 'boolean', default: false }, + json: { type: 'boolean', default: false }, + check: { type: 'boolean', default: false }, + }, +}); + +const fail = (message: string): never => { + console.error(message); + process.exit(1); +}; + +const list = (values: string[] | undefined) => (values ?? []).flatMap((v) => v.split(',')).filter(Boolean); +const providerNames = list(values.provider); +const runs = Number(values.runs); +if (providerNames.length === 0 && !values.check) fail('Name a model provider with --provider , or pass --check.'); +if (!Number.isInteger(runs) || runs < 1) fail(`--runs must be a positive whole number, not "${values.runs}".`); + +// Build every provider and check every eval before the first paid call. +const config = loadConfigOrEmpty(); +// Plugins resolve from the main config file's directory and from each included file's. +const configDirs = [ + ...new Set([dirname(configPath()), ...config.spaces.flatMap((s) => (s.sourceDir ? [s.sourceDir] : []))]), +]; +const providers = providerNames.map((name) => createJudgeProvider(config, {}, name)!); +const suites = (list(values.from).length > 0 ? list(values.from) : ['core']).map((s) => resolveSuite(s, configDirs)); +const only = list(values.eval); +const evals = loadEvals(suites, config).filter( + (e) => only.length === 0 || only.some((name) => e.name === name || e.name.endsWith(`/${name}`)), +); +if (evals.length === 0) fail('No evals found.'); +const problems = (await Promise.all(evals.map(checkEval))).flat(); +if (problems.length > 0) fail(`Evals have problems:\n${problems.join('\n')}`); +if (values.check) { + console.log(`Checked ${evals.length} eval(s): ${evals.map((e) => e.name).join(', ')}`); + process.exit(0); +} + +const results: Array<{ eval: string; provider: string; scores: RuleScore[]; runs: Judgement[][] }> = []; +for (const evaluation of evals) { + for (const [i, provider] of providers.entries()) { + const judged: Judgement[][] = []; + for (let run = 0; run < runs; run++) { + judged.push(await judgeEval(evaluation, provider, { perNode: values['per-node'] })); + } + results.push({ + eval: evaluation.name, + provider: providerNames[i]!, + scores: scoreEval(evaluation, judged), + runs: judged, + }); + } +} + +if (values.json) { + console.log(JSON.stringify(results, null, 2)); + process.exit(0); +} + +const pct = (n: number | undefined) => (n === undefined ? '-' : `${Math.round(n * 100)}%`); +const lines = [ + `Semantic evals: ${runs} run(s) per provider${values['per-node'] ? ', one request per node' : ''}`, + '', + '| Eval | Provider | Rule | Cases | Agreement | Stable | p(expected) | Reserved | Errors |', + '|---|---|---|---|---|---|---|---|---|', +]; +const misses: string[] = []; +const closest: string[] = []; +for (const { eval: name, provider, scores } of results) { + for (const s of scores) { + const stable = runs > 1 ? pct(s.stable) : '-'; + lines.push( + `| ${name} | ${provider} | ${s.ruleId} | ${s.cases} | ${pct(s.agreement)} | ${stable} | ${pct(s.expectedProbability)} | ${s.reserved} | ${s.errors} |`, + ); + const at = `${name} · ${provider} · ${s.ruleId}`; + for (const m of s.misses) { + misses.push(`- ${at} · ${m.nodeId}: expected ${m.expected.join(' or ')}, got ${m.got.join(', ')}`); + if (m.evidence) misses.push(` ${m.evidence}`); + } + for (const c of s.closest) closest.push(`- ${at} · ${c.nodeId}: ${pct(c.probability)}`); + } +} +if (misses.length > 0) lines.push('', 'Misses:', ...misses); +if (closest.length > 0) lines.push('', 'Closest agreeing cases, by p(expected):', ...closest); +console.log(lines.join('\n')); diff --git a/evals/score.ts b/evals/score.ts new file mode 100644 index 0000000..2399900 --- /dev/null +++ b/evals/score.ts @@ -0,0 +1,79 @@ +import { RESERVED_VERDICTS } from '../src/schema/metadata-contract'; +import type { Judgement } from './judge'; +import type { Eval } from './load'; + +/** How well a provider's verdicts on one rule match the labels, over every run. README.md explains each score. */ +export interface RuleScore { + ruleId: string; + cases: number; + runs: number; + agreement: number; + stable: number; + expectedProbability?: number; + reserved: number; + errors: number; + /** Cases where some run disagreed with the label, with the verdict (or error) each run gave. */ + misses: Array<{ nodeId: string; expected: string[]; got: string[]; evidence?: string }>; + /** The agreeing cases with the lowest mean p(expected): the ones nearest to a miss. */ + closest: Array<{ nodeId: string; probability: number }>; +} + +const CLOSEST = 3; +const reserved: readonly string[] = RESERVED_VERDICTS; + +const describe = (result: Judgement['result'] | undefined) => + !result ? 'not judged' : 'error' in result ? `error: ${result.error}` : result.verdict; + +/** Score each labelled rule over the runs of one eval with one provider. */ +export function scoreEval(evaluation: Pick, runs: Judgement[][]): RuleScore[] { + return Object.entries(evaluation.expected).map(([ruleId, labels]) => { + const perRun = runs.map((run) => new Map(run.filter((j) => j.ruleId === ruleId).map((j) => [j.nodeId, j.result]))); + let agreeing = 0; + let stable = 0; + let errors = 0; + let reservedCount = 0; + const probabilities: number[] = []; + const misses: RuleScore['misses'] = []; + const agreed: RuleScore['closest'] = []; + for (const [nodeId, expected] of Object.entries(labels)) { + const results = perRun.map((run) => run.get(nodeId)); + const verdicts = results.map((r) => (r && !('error' in r) ? r.verdict : undefined)); + agreeing += verdicts.filter((v) => v !== undefined && expected.includes(v)).length; + reservedCount += verdicts.filter((v) => v !== undefined && reserved.includes(v)).length; + if (verdicts.includes(undefined)) errors++; + else if (new Set(verdicts).size === 1) stable++; + const caseProbabilities = results.flatMap((r) => + r && !('error' in r) && r.probabilities + ? [expected.reduce((sum, v) => sum + (r.probabilities?.[v] ?? 0), 0)] + : [], + ); + probabilities.push(...caseProbabilities); + if (verdicts.some((v) => v === undefined || !expected.includes(v))) { + const miss = results.find((r) => !r || 'error' in r || !expected.includes(r.verdict)); + misses.push({ + nodeId, + expected, + got: results.map(describe), + ...(miss && !('error' in miss) && miss.evidence !== undefined && { evidence: miss.evidence }), + }); + } else if (caseProbabilities.length > 0) { + agreed.push({ nodeId, probability: mean(caseProbabilities) }); + } + } + const cases = Object.keys(labels).length; + return { + ruleId, + cases, + runs: runs.length, + agreement: cases && runs.length ? agreeing / (cases * runs.length) : 0, + stable: cases ? stable / cases : 0, + ...(probabilities.length > 0 && { expectedProbability: mean(probabilities) }), + reserved: reservedCount, + errors, + misses, + closest: agreed.sort((a, b) => a.probability - b.probability).slice(0, CLOSEST), + }; + }); +} + +const mean = (values: number[]) => values.reduce((a, b) => a + b, 0) / values.length; diff --git a/evals/semantic/README.md b/evals/semantic/README.md deleted file mode 100644 index ebe3174..0000000 --- a/evals/semantic/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# Semantic evals - -Semantic evals measure how well model providers judge semantic rules. An eval space is a space with a label per node and rule: the verdict the node should get. Core's eval spaces here cover the bundled schemas. A plugin can keep its own eval spaces for its schemas, and an eval space can label nodes of a configured space. - -## Run the evals - -The run calls real providers, so it costs money. Name each provider to run; config `modelProviders` entries work as well as bundled providers. - -```bash -bun run eval:semantic --provider openrouter-jev --provider anthropic -``` - -From outside this repo, run `bun run /evals/semantic/run.ts` with the same flags. - -| Flag | Purpose | -|---|---| -| `--provider ` | The provider to judge with. Repeat it, or give a comma-separated list, to compare providers | -| `--from ` | Where to find eval spaces. Repeatable. Default: `core`. See [Where eval spaces live](#where-eval-spaces-live) | -| `--space ` | Run only this eval space, by name (`core/strict-ost`) or directory (`strict-ost`). Repeatable | -| `--runs ` | Judge each space this many times, to measure stability. Default: 3 | -| `--per-node` | Send one request per node, even to a provider that packs several nodes into one request | -| `--check` | Check the eval spaces and stop, with no provider calls | -| `--json` | Print every judgement and score as JSON instead of the report | - -The report has one row per space, provider and rule: - -| Column | Meaning | -|---|---| -| Agreement | Share of judgements, over every case and run, that match the label | -| Stable | Share of cases that got the same verdict on every run | -| p(expected) | Mean probability the provider gave the labelled verdicts. Only decision models report it | -| Reserved | Judgements with a reserved verdict, such as `missing-context` | -| Errors | Cases that got an evaluation error, or no judgement, on some run | - -Each run validates the space through the same pipeline as `sctx validate --semantic`, with only the labelled rules, and evaluates gated nodes too. Only labelled nodes go to the provider. - -## Where eval spaces live - -Each source is a directory of eval spaces: - -| `--from` | Directory | Schema names resolve from | -|---|---|---| -| `core` | `evals/semantic/` in this repo | the bundled schemas | -| `` | `evals/semantic/` in the plugin, found as the config finds plugins | the config that finds the plugin | -| a path | that directory | that directory | - -## Add an eval space - -An eval space is a directory with an `eval.json5`. It is one of two kinds. - -A **fixture** eval space has its own nodes in a `nodes.jsonl` beside `eval.json5`: one JSON object per line, with the fields frontmatter would hold. Each needs a `title`, which is also its node id, and a `type`. A body goes in `content`. JSON keeps fixtures out of the markdown that editors and spaces read. To start from markdown notes, `sctx dump | jq -c '.nodes[].schemaData'` prints them in this shape. The eval space names its `schema`, and optionally `plugins`, as a space config does: - -```json5 -{ - schema: 'strict_ost.json', - expected: { - 'opportunity-customer-voice': { - 'Build a Slack integration': 'fail', - // A list accepts any of its verdicts. - 'New users find the editor toolbar confusing': ['pass', 'fail'], - }, - }, -} -``` - -An eval space on a **configured space** names a space from the config instead, and labels some of its nodes: - -```json5 -{ space: 'work', expected: { 'opportunity-customer-voice': { 'Onboarding.md': 'fail' } } } -``` - -`expected` maps a rule id to node ids and their verdicts. In a fixture, label every node a labelled rule judges. Write a comment on a case whose label is not obvious. - -`--check` finds the problems that would make scores meaningless: a label that names an unknown node, rule or verdict; a labelled node that fails validation; and, in a fixture, a judged node with no label. A run checks this before its first provider call. `bun run test` checks core's eval spaces. diff --git a/evals/semantic/fixtures/strict-ost.json5 b/evals/semantic/fixtures/strict-ost.json5 new file mode 100644 index 0000000..1d7a0a8 --- /dev/null +++ b/evals/semantic/fixtures/strict-ost.json5 @@ -0,0 +1,26 @@ +// A small product's opportunity solution tree. Each node holds the fields its frontmatter would. +{ + schema: 'strict_ost.json', + nodes: [ + { title: 'Increase weekly active teams', type: 'outcome', status: 'identified', metric: 'Increase % of trial teams active in their second week', content: 'Teams that use the product every week renew.' }, + + // Customer needs, in the customer's words. + { title: 'Setting up a workspace takes so long that I give up', type: 'opportunity', status: 'identified', parent: '[[Increase weekly active teams]]', source: 'Interview with Priya, 2026-09-02', content: 'Priya abandoned setup twice before her team lead finished it for her.' }, + { title: "I can't tell which plan fits a team of my size", type: 'opportunity', status: 'identified', parent: '[[Increase weekly active teams]]', source: 'Sales call notes, 2026-09-10', content: 'Prospects asked which tier covers twelve seats with guest access.' }, + { title: 'I want to see what my teammates changed while I was away', type: 'opportunity', status: 'identified', parent: '[[Increase weekly active teams]]', source: 'Interview with Tomas, 2026-09-15', content: 'Tomas reads every page again after a holiday to find what is new.' }, + { title: "I'm afraid I'll lose my work if my browser crashes", type: 'opportunity', status: 'identified', parent: '[[Increase weekly active teams]]', source: 'Support tickets, September 2026', content: 'Several tickets describe retyping long notes after a crash.' }, + { title: "I don't know whether my changes have been saved", type: 'opportunity', status: 'identified', parent: "[[I'm afraid I'll lose my work if my browser crashes]]", source: 'Usability test round 3', content: 'Four of six participants pressed the save shortcut repeatedly.' }, + { title: 'Show unsaved-change indicator', type: 'solution', status: 'exploring', parent: "[[I don't know whether my changes have been saved]]", content: 'A dot on the tab while changes are pending.' }, + + // Solutions, metrics and topics posing as opportunities. + { title: 'Add a progress bar to workspace setup', type: 'opportunity', status: 'identified', parent: '[[Increase weekly active teams]]', source: 'Product team brainstorm', content: 'Show users how many setup steps remain.' }, + { title: 'Build a Slack integration', type: 'opportunity', status: 'identified', parent: '[[Increase weekly active teams]]', source: 'Competitor review', content: 'Post page updates to a Slack channel.' }, + { title: 'Offer a discount on annual plans', type: 'opportunity', status: 'identified', parent: '[[Increase weekly active teams]]', source: 'Pricing workshop', content: 'Ten percent off when paying yearly.' }, + { title: 'Increase trial-to-paid conversion by 10%', type: 'opportunity', status: 'identified', parent: '[[Increase weekly active teams]]', source: 'Quarterly planning', content: 'The board wants conversion to rise this quarter.' }, + { title: 'Reduce churn among small teams', type: 'opportunity', status: 'identified', parent: '[[Increase weekly active teams]]', source: 'Retention dashboard', content: 'Teams of fewer than five seats churn twice as often.' }, + { title: 'Onboarding', type: 'opportunity', status: 'identified', parent: '[[Increase weekly active teams]]', source: 'Interview synthesis', content: 'Everything about how new users get started.' }, + + // A real pain point, but reported in the third person. + { title: 'New users find the editor toolbar confusing', type: 'opportunity', status: 'identified', parent: '[[Increase weekly active teams]]', source: 'Usability test round 2', content: 'Participants hovered over icons without knowing what they did.' }, + ], +} diff --git a/evals/semantic/harness.ts b/evals/semantic/harness.ts deleted file mode 100644 index fcfcd2a..0000000 --- a/evals/semantic/harness.ts +++ /dev/null @@ -1,320 +0,0 @@ -import { existsSync, readdirSync, readFileSync } from 'node:fs'; -import { basename, join, resolve } from 'node:path'; -import JSON5 from 'json5'; -import type { Config, SpaceConfig } from '../../src/config'; -import { FakeJudgeProvider } from '../../src/judge/provider/fake'; -import type { SemanticJudgeProvider, SemanticJudgeResult } from '../../src/judge/types'; -import { resolveSemanticRule } from '../../src/judge/verdicts'; -import { resolvePluginDir } from '../../src/plugins/util'; -import { readSpace } from '../../src/read/read-space'; -import { resolveGraphEdges } from '../../src/read/resolve-graph-edges'; -import { RESERVED_VERDICTS } from '../../src/schema/metadata-contract'; -import { createSpaceContext } from '../../src/space-context'; -import type { BaseNode, NodeId, ReadSpaceResult, SpaceContext } from '../../src/types'; -import { validateNodes } from '../../src/validate'; - -/** Core's eval spaces, for the bundled schemas. */ -export const CORE_EVALS_DIR = import.meta.dir; - -/** Where a plugin keeps its eval spaces. */ -export const EVALS_SUBDIR = join('evals', 'semantic'); - -/** Rule id → node id → the verdicts that count as agreeing. */ -export type Expected = Record>; - -/** A space and the verdict each of its nodes should get per rule. */ -export interface EvalSpace { - /** `/`, e.g. `core/strict-ost`. */ - name: string; - /** A fixture space has its own nodes, so every node a labelled rule judges must have a label. */ - fixture: boolean; - context: SpaceContext; - expected: Expected; - /** The space's nodes, as validation reads them. */ - read: () => Promise>; -} - -/** One verdict (or error) the judge gave for a node and rule. */ -export interface Judgement { - nodeId: string; - ruleId: string; - result: SemanticJudgeResult; -} - -/** A directory of eval spaces, and the directory their schema names resolve from. */ -export interface EvalSource { - label: string; - dir: string; - configDir: string; -} - -/** - * Find a source: `core`, a plugin name (its `evals/semantic/`), or a path to a directory of eval spaces. A - * plugin's schema names resolve as they would in the config that finds the plugin. - */ -export function resolveEvalSources(source: string, configDirs: string[]): EvalSource[] { - if (source === 'core') return [{ label: 'core', dir: CORE_EVALS_DIR, configDir: CORE_EVALS_DIR }]; - if (source.includes('/') || source.includes('\\') || source.startsWith('.')) { - const dir = resolve(source); - if (!existsSync(dir)) throw new Error(`Eval source "${source}": no directory at ${dir}`); - return [{ label: basename(dir), dir, configDir: dir }]; - } - for (const configDir of configDirs) { - const pluginDir = resolvePluginDir(source, configDir); - if (pluginDir) { - const dir = join(pluginDir, EVALS_SUBDIR); - if (!existsSync(dir)) throw new Error(`Plugin "${source}" has no ${EVALS_SUBDIR}/ directory (${pluginDir})`); - return [{ label: source, dir, configDir }]; - } - } - throw new Error(`Eval source "${source}" is not core, a path, or a plugin found from the config.`); -} - -/** - * An eval space's `eval.json5`. A fixture eval space names a `schema` (and optional `plugins`), as a space config - * does, and keeps its nodes in `nodes.jsonl` beside the file, one schema object per line. An eval space on a - * configured space names the `space` instead, and labels some of its nodes. - */ -type EvalFile = Pick & { - space?: string; - expected: Record>; -}; - -/** The eval spaces in each source: every directory with an `eval.json5`. `config` holds configured spaces. */ -export function loadEvalSpaces(sources: EvalSource[], config: Config): EvalSpace[] { - return sources.flatMap((source) => - readdirSync(source.dir, { withFileTypes: true }) - .filter((d) => d.isDirectory() && existsSync(join(source.dir, d.name, 'eval.json5'))) - .map((d) => d.name) - .sort() - .map((dirName) => { - const name = `${source.label}/${dirName}`; - const dir = join(source.dir, dirName); - const file = JSON5.parse(readFileSync(join(dir, 'eval.json5'), 'utf8')) as EvalFile; - if ((file.space === undefined) === (file.schema === undefined)) { - throw new Error(`Eval space ${name}: eval.json5 needs either "schema" (with nodes.jsonl) or "space".`); - } - let context: SpaceContext; - try { - context = - file.space !== undefined - ? createSpaceContext(file.space, config) - : createSpaceContext( - name, - { spaces: [{ name, path: dir, schema: file.schema, plugins: file.plugins }] }, - { configDir: source.configDir }, - ); - } catch (err) { - throw new Error(`Eval space ${name}: ${err instanceof Error ? err.message : String(err)}`); - } - const expected: Expected = {}; - for (const [ruleId, nodes] of Object.entries(file.expected)) { - expected[ruleId] = Object.fromEntries( - Object.entries(nodes).map(([id, v]) => [id, typeof v === 'string' ? [v] : v]), - ); - } - const read = - file.space !== undefined - ? () => readSpace(context) - : async () => resolveGraphEdges(readNodes(join(dir, 'nodes.jsonl')), context.schema.metadata); - return { name, fixture: file.space === undefined, context, expected, read }; - }), - ); -} - -/** - * Fixture nodes: one schema object per line, as frontmatter would hold it, with its `title` as the node id and - * link target. - */ -function readNodes(path: string): BaseNode[] { - return readFileSync(path, 'utf8') - .split('\n') - .map((line, i) => ({ line, at: `${path}:${i + 1}` })) - .filter(({ line }) => line.trim() !== '') - .map(({ line, at }) => { - const schemaData = JSON.parse(line) as Record; - const { title, type } = schemaData; - if (typeof title !== 'string' || typeof type !== 'string') throw new Error(`${at}: needs a "title" and a "type"`); - return { id: title as NodeId, title, type, linkTargets: [title], schemaData }; - }); -} - -/** - * Problems that would make an eval space's scores meaningless, checked before any judge call: a label naming an - * unknown node, a rule with no semantic part, or a verdict the rule cannot return; a labelled node that fails - * validation; and, in a fixture space, a node a labelled rule judges with no label, or a label never judged. - */ -export async function checkEvalSpace(space: EvalSpace): Promise { - const problems: string[] = []; - const read = await space.read(); - const ids = new Set(read.nodes.map((n) => n.id as string)); - const labelled = new Set(Object.values(space.expected).flatMap((labels) => Object.keys(labels))); - const result = await validateNodes(space.context, read.nodes, read); - for (const issue of result.issues) { - if (issue.severity === 'error' && issue.nodeId && labelled.has(issue.nodeId)) { - problems.push(`${issue.nodeId}: ${issue.message}`); - } - } - const rules = space.context.schema.metadata.rules ?? []; - for (const [ruleId, labels] of Object.entries(space.expected)) { - const rule = rules.find((r) => r.id === ruleId); - if (!rule?.semantic) { - problems.push(`rule "${ruleId}": not a semantic rule of the schema`); - continue; - } - const verdicts = Object.keys(resolveSemanticRule(rule).spec.verdicts); - for (const [nodeId, expected] of Object.entries(labels)) { - if (!ids.has(nodeId)) problems.push(`rule "${ruleId}": unknown node "${nodeId}"`); - for (const verdict of expected) { - if (!verdicts.includes(verdict)) { - problems.push(`rule "${ruleId}", node "${nodeId}": "${verdict}" is not a verdict (${verdicts.join(', ')})`); - } - } - } - } - if (space.fixture && problems.length === 0) { - const judge = new FakeJudgeProvider(({ node, rule }) => ({ - verdict: space.expected[rule.id]?.[node.id]?.[0] ?? Object.keys(rule.verdicts)[0]!, - })); - for (const score of scoreEvalSpace(space, [await judgeEvalSpace(space, judge, { unlabelled: true })])) { - for (const nodeId of score.unlabelled) problems.push(`rule "${score.ruleId}": node "${nodeId}" has no label`); - for (const { nodeId } of score.misses) problems.push(`rule "${score.ruleId}": node "${nodeId}" is not judged`); - } - } - return problems.map((p) => `${space.name}: ${p}`); -} - -type JudgeInput = Parameters[0]; - -/** - * Validate the space with its labelled rules through the real pipeline, recording every result the provider - * returns, passing verdicts included. Gated nodes are judged too. Only labelled nodes reach the provider, unless - * `unlabelled` is set. `perNode` hides `evaluateNodes`, so a provider that packs nodes into one request gets one - * request per node instead. - */ -export async function judgeEvalSpace( - space: EvalSpace, - provider: SemanticJudgeProvider, - options: { perNode?: boolean; unlabelled?: boolean } = {}, -): Promise { - const judgements: Judgement[] = []; - const toJudge = (input: JudgeInput): JudgeInput | undefined => { - if (options.unlabelled) return input; - const rules = input.rules.filter((rule) => space.expected[rule.id]?.[input.node.id] !== undefined); - return rules.length > 0 ? { ...input, rules } : undefined; - }; - const record = (input: JudgeInput, results: Record) => { - for (const rule of input.rules) { - const result = results[rule.id] ?? { error: 'no result for this rule' }; - judgements.push({ nodeId: input.node.id, ruleId: rule.id, result }); - } - }; - const recording: SemanticJudgeProvider = { - name: provider.name, - async evaluate(input) { - const sent = toJudge(input); - if (!sent) return {}; - const results = await provider.evaluate(sent); - record(sent, results); - return results; - }, - }; - const { evaluateNodes } = provider; - if (evaluateNodes && !options.perNode) { - recording.evaluateNodes = async (inputs) => { - const sent = inputs.map(toJudge); - const kept = sent.filter((input) => input !== undefined); - const results = kept.length > 0 ? await evaluateNodes.call(provider, kept) : []; - let k = 0; - return sent.map((input) => { - if (!input) return {}; - const result = results[k++] ?? {}; - record(input, result); - return result; - }); - }; - } - const read = await space.read(); - await validateNodes(space.context, read.nodes, read, { - includeGated: true, - ruleFilter: { id: Object.keys(space.expected) }, - judge: { provider: recording }, - }); - return judgements; -} - -/** How well a provider's verdicts on one rule match the labels, over every run. */ -export interface RuleScore { - ruleId: string; - cases: number; - runs: number; - /** Share of judgements, over cases and runs, whose verdict is an expected one. */ - agreement: number; - /** Share of cases that got the same verdict on every run, with no errors. */ - stable: number; - /** Mean probability given to the expected verdicts, over judgements that report probabilities. */ - expectedProbability?: number; - /** Judgements with a reserved verdict, such as `missing-context`. */ - reserved: number; - /** Cases that got an error, or no judgement, on some run. */ - errors: number; - /** Cases where some run disagreed with the label, with the verdict (or error) each run gave. */ - misses: Array<{ nodeId: string; expected: string[]; got: string[]; evidence?: string }>; - /** Nodes judged on this rule with no label. */ - unlabelled: string[]; -} - -const describe = (result: SemanticJudgeResult | undefined) => - !result ? 'not judged' : 'error' in result ? `error: ${result.error}` : result.verdict; - -/** Score each labelled rule over the runs of one eval space with one provider. */ -export function scoreEvalSpace(space: EvalSpace, runs: Judgement[][]): RuleScore[] { - const reserved: readonly string[] = RESERVED_VERDICTS; - return Object.entries(space.expected).map(([ruleId, labels]) => { - const perRun = runs.map((run) => new Map(run.filter((j) => j.ruleId === ruleId).map((j) => [j.nodeId, j.result]))); - let agreeing = 0; - let stable = 0; - let errors = 0; - let reservedCount = 0; - const probabilities: number[] = []; - const misses: RuleScore['misses'] = []; - for (const [nodeId, expected] of Object.entries(labels)) { - const results = perRun.map((run) => run.get(nodeId)); - const verdicts = results.map((r) => (r && !('error' in r) ? r.verdict : undefined)); - agreeing += verdicts.filter((v) => v !== undefined && expected.includes(v)).length; - reservedCount += verdicts.filter((v) => v !== undefined && reserved.includes(v)).length; - if (verdicts.includes(undefined)) errors++; - else if (new Set(verdicts).size === 1) stable++; - for (const r of results) { - if (r && !('error' in r) && r.probabilities) { - probabilities.push(expected.reduce((sum, v) => sum + (r.probabilities?.[v] ?? 0), 0)); - } - } - if (verdicts.some((v) => v === undefined || !expected.includes(v))) { - const miss = results.find((r) => !r || 'error' in r || !expected.includes(r.verdict)); - misses.push({ - nodeId, - expected, - got: results.map(describe), - ...(miss && !('error' in miss) && miss.evidence !== undefined && { evidence: miss.evidence }), - }); - } - } - const cases = Object.keys(labels).length; - const unlabelled = new Set(perRun.flatMap((run) => [...run.keys()]).filter((id) => !(id in labels))); - return { - ruleId, - cases, - runs: runs.length, - agreement: cases && runs.length ? agreeing / (cases * runs.length) : 0, - stable: cases ? stable / cases : 0, - ...(probabilities.length > 0 && { - expectedProbability: probabilities.reduce((a, b) => a + b, 0) / probabilities.length, - }), - reserved: reservedCount, - errors, - misses, - unlabelled: [...unlabelled].sort(), - }; - }); -} diff --git a/evals/semantic/strict-ost/eval.json5 b/evals/semantic/opportunity-customer-voice.json5 similarity index 73% rename from evals/semantic/strict-ost/eval.json5 rename to evals/semantic/opportunity-customer-voice.json5 index 6c8bc4a..c6394eb 100644 --- a/evals/semantic/strict-ost/eval.json5 +++ b/evals/semantic/opportunity-customer-voice.json5 @@ -1,6 +1,5 @@ { - schema: 'strict_ost.json', - // Rule id → node id → the verdict the node should get. A list accepts any of its verdicts. + fixture: 'strict-ost', expected: { 'opportunity-customer-voice': { 'Setting up a workspace takes so long that I give up': 'pass', @@ -17,8 +16,6 @@ 'Reduce churn among small teams': 'fail', // A topic, not a need. 'Onboarding': 'fail', - // A real pain point, but in the third person rather than the customer's own words. - 'New users find the editor toolbar confusing': ['pass', 'fail'], }, }, } diff --git a/evals/semantic/run.ts b/evals/semantic/run.ts deleted file mode 100644 index 55b22f9..0000000 --- a/evals/semantic/run.ts +++ /dev/null @@ -1,109 +0,0 @@ -/** - * Semantic evals: judge eval spaces with real providers and report agreement with the labels. Costs money, - * except with `--check`. See README.md. - */ -import { existsSync } from 'node:fs'; -import { dirname } from 'node:path'; -import { parseArgs } from 'node:util'; -import { configPath, getConfigSourceFiles, loadConfigOrEmpty } from '../../src/config'; -import { createJudgeProvider } from '../../src/judge/factory'; -import { - checkEvalSpace, - type Judgement, - judgeEvalSpace, - loadEvalSpaces, - type RuleScore, - resolveEvalSources, - scoreEvalSpace, -} from './harness'; - -const { values } = parseArgs({ - options: { - from: { type: 'string', multiple: true }, - provider: { type: 'string', multiple: true }, - space: { type: 'string', multiple: true }, - runs: { type: 'string', default: '3' }, - 'per-node': { type: 'boolean', default: false }, - json: { type: 'boolean', default: false }, - check: { type: 'boolean', default: false }, - }, -}); - -const list = (values: string[] | undefined) => (values ?? []).flatMap((v) => v.split(',')).filter(Boolean); -const providerNames = list(values.provider); -const runs = Number(values.runs); -if (providerNames.length === 0 && !values.check) { - throw new Error('Name at least one model provider with --provider , or pass --check.'); -} -if (!Number.isInteger(runs) || runs < 1) { - throw new Error(`--runs must be a positive whole number, not "${values.runs}".`); -} - -// Build every provider and check every space before the first paid call. -const config = loadConfigOrEmpty(); -// The main config file and every file it includes. -const configFiles = [configPath(), ...getConfigSourceFiles()].filter((file) => existsSync(file)); -const configDirs = [...new Set(configFiles.map((file) => dirname(file)))]; -const providers = providerNames.map((name) => createJudgeProvider(config, {}, name)!); -const sources = (list(values.from).length > 0 ? list(values.from) : ['core']).flatMap((source) => - resolveEvalSources(source, configDirs), -); -const only = list(values.space); -const spaces = loadEvalSpaces(sources, config).filter( - (space) => only.length === 0 || only.some((name) => space.name === name || space.name.endsWith(`/${name}`)), -); -if (spaces.length === 0) throw new Error('No eval spaces found.'); -const problems = (await Promise.all(spaces.map(checkEvalSpace))).flat(); -if (problems.length > 0) throw new Error(`Eval spaces have problems:\n${problems.join('\n')}`); -if (values.check) { - console.log(`Checked ${spaces.length} eval space(s): ${spaces.map((s) => s.name).join(', ')}`); - process.exit(0); -} - -const results: Array<{ space: string; provider: string; scores: RuleScore[]; runs: Judgement[][] }> = []; -for (const space of spaces) { - for (const [i, provider] of providers.entries()) { - const judged: Judgement[][] = []; - for (let run = 0; run < runs; run++) { - judged.push(await judgeEvalSpace(space, provider, { perNode: values['per-node'] })); - } - results.push({ - space: space.name, - provider: providerNames[i]!, - scores: scoreEvalSpace(space, judged), - runs: judged, - }); - } -} - -if (values.json) { - console.log(JSON.stringify(results, null, 2)); -} else { - const pct = (n: number | undefined) => (n === undefined ? '-' : `${Math.round(n * 100)}%`); - const lines = [ - `Semantic evals: ${runs} run(s) per provider${values['per-node'] ? ', one request per node' : ''}`, - '', - '| Space | Provider | Rule | Cases | Agreement | Stable | p(expected) | Reserved | Errors |', - '|---|---|---|---|---|---|---|---|---|', - ]; - for (const { space, provider, scores } of results) { - for (const s of scores) { - lines.push( - `| ${space} | ${provider} | ${s.ruleId} | ${s.cases} | ${pct(s.agreement)} | ${runs > 1 ? pct(s.stable) : '-'} | ${pct(s.expectedProbability)} | ${s.reserved} | ${s.errors} |`, - ); - } - } - const notes: string[] = []; - for (const { space, provider, scores } of results) { - for (const s of scores) { - for (const m of s.misses) { - notes.push( - `- ${space} · ${provider} · ${s.ruleId} · ${m.nodeId}: expected ${m.expected.join(' or ')}, got ${m.got.join(', ')}`, - ); - if (m.evidence) notes.push(` ${m.evidence}`); - } - } - } - if (notes.length > 0) lines.push('', 'Misses:', ...notes); - console.log(lines.join('\n')); -} diff --git a/evals/semantic/strict-ost/nodes.jsonl b/evals/semantic/strict-ost/nodes.jsonl deleted file mode 100644 index 745676e..0000000 --- a/evals/semantic/strict-ost/nodes.jsonl +++ /dev/null @@ -1,14 +0,0 @@ -{"title": "Increase weekly active teams", "type": "outcome", "status": "identified", "metric": "Increase % of trial teams active in their second week", "content": "Teams that use the product every week renew."} -{"title": "New users find the editor toolbar confusing", "type": "opportunity", "status": "identified", "parent": "[[Increase weekly active teams]]", "source": "Usability test round 2", "content": "Participants hovered over icons without knowing what they did."} -{"title": "Onboarding", "type": "opportunity", "status": "identified", "parent": "[[Increase weekly active teams]]", "source": "Interview synthesis", "content": "Everything about how new users get started."} -{"title": "I don't know whether my changes have been saved", "type": "opportunity", "status": "identified", "parent": "[[I'm afraid I'll lose my work if my browser crashes]]", "source": "Usability test round 3", "content": "Four of six participants pressed the save shortcut repeatedly."} -{"title": "Add a progress bar to workspace setup", "type": "opportunity", "status": "identified", "parent": "[[Increase weekly active teams]]", "source": "Product team brainstorm", "content": "Show users how many setup steps remain."} -{"title": "Build a Slack integration", "type": "opportunity", "status": "identified", "parent": "[[Increase weekly active teams]]", "source": "Competitor review", "content": "Post page updates to a Slack channel."} -{"title": "Setting up a workspace takes so long that I give up", "type": "opportunity", "status": "identified", "parent": "[[Increase weekly active teams]]", "source": "Interview with Priya, 2026-09-02", "content": "Priya abandoned setup twice before her team lead finished it for her."} -{"title": "Increase trial-to-paid conversion by 10%", "type": "opportunity", "status": "identified", "parent": "[[Increase weekly active teams]]", "source": "Quarterly planning", "content": "The board wants conversion to rise this quarter."} -{"title": "I can't tell which plan fits a team of my size", "type": "opportunity", "status": "identified", "parent": "[[Increase weekly active teams]]", "source": "Sales call notes, 2026-09-10", "content": "Prospects asked which tier covers twelve seats with guest access."} -{"title": "I want to see what my teammates changed while I was away", "type": "opportunity", "status": "identified", "parent": "[[Increase weekly active teams]]", "source": "Interview with Tomas, 2026-09-15", "content": "Tomas reads every page again after a holiday to find what is new."} -{"title": "I'm afraid I'll lose my work if my browser crashes", "type": "opportunity", "status": "identified", "parent": "[[Increase weekly active teams]]", "source": "Support tickets, September 2026", "content": "Several tickets describe retyping long notes after a crash."} -{"title": "Offer a discount on annual plans", "type": "opportunity", "status": "identified", "parent": "[[Increase weekly active teams]]", "source": "Pricing workshop", "content": "Ten percent off when paying yearly."} -{"title": "Reduce churn among small teams", "type": "opportunity", "status": "identified", "parent": "[[Increase weekly active teams]]", "source": "Retention dashboard", "content": "Teams of fewer than five seats churn twice as often."} -{"title": "Show unsaved-change indicator", "type": "solution", "parent": "[[I don't know whether my changes have been saved]]", "status": "exploring", "content": "A dot on the tab while changes are pending."} diff --git a/package.json b/package.json index 41aad0a..1fc124c 100644 --- a/package.json +++ b/package.json @@ -52,12 +52,12 @@ "test:smoke": "bun test smoke/", "test:package": "bash scripts/test-package.sh", "test:judge": "SCTX_LIVE_JUDGE_TESTS=1 bun test tests/judge/live.test.ts", - "eval:semantic": "bun run evals/semantic/run.ts", "test:models": "SCTX_MODEL_VALIDATION=1 bun test tests/judge/model-validation.test.ts", "test:hook": "bun test hook-test/unit/", "test:mod": "claude plugin test plugin", "test:hook:e2e": "bun test hook-test/hooks.test.ts", "test:all": "bun test", + "eval:semantic": "bun run evals/run.ts", "lint": "biome check", "lint:fix": "biome check --write", "preversion": "bun run generate:schema-meta && bun run lint:fix && bun run test && bun run test:package", diff --git a/tests/evals/semantic.test.ts b/tests/evals/semantic.test.ts index c266e29..ef9cee4 100644 --- a/tests/evals/semantic.test.ts +++ b/tests/evals/semantic.test.ts @@ -2,77 +2,99 @@ import { afterAll, describe, expect, test } from 'bun:test'; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { basename, join } from 'node:path'; -import { - checkEvalSpace, - type EvalSpace, - type Judgement, - judgeEvalSpace, - loadEvalSpaces, - resolveEvalSources, - scoreEvalSpace, -} from '../../evals/semantic/harness'; +import { checkEval, type Judgement, judgeEval } from '../../evals/judge'; +import { loadEvals, resolveSuite } from '../../evals/load'; +import { scoreEval } from '../../evals/score'; import type { Config } from '../../src/config'; import { FakeJudgeProvider } from '../../src/judge/provider/fake'; +import type { SemanticJudgeInput } from '../../src/judge/types'; -const spaces = loadEvalSpaces(resolveEvalSources('core', []), { spaces: [] }); +const coreEvals = loadEvals([resolveSuite('core', [])], { spaces: [] }); -describe.each(spaces.map((s) => [s.name, s] as const))('eval space %s', (_name, space) => { - test('passes its checks: labels name real nodes, semantic rules and verdicts, and cover every judged node', async () => { - expect(await checkEvalSpace(space)).toEqual([]); +describe.each(coreEvals.map((e) => [e.name, e] as const))('eval %s', (_name, evaluation) => { + test('passes its checks', async () => { + expect(await checkEval(evaluation)).toEqual([]); }); }); -describe('eval spaces on a configured space', () => { +describe('a suite directory', () => { const dir = mkdtempSync(join(tmpdir(), 'sctx-evals-')); - const fixture = join(dir, 'space'); - const note = (title: string, frontmatter: string) => - writeFileSync(join(fixture, `${title}.md`), `---\nstatus: identified\n${frontmatter}\n---\n`); - mkdirSync(fixture); - note('Grow usage', 'type: outcome\nmetric: Weekly active teams'); - for (const title of ['Onboarding', 'Build a Slack integration', 'I lose track of changes']) { - note(title, 'type: opportunity\nparent: "[[Grow usage]]"\nsource: Interviews'); - } - mkdirSync(join(dir, 'mine')); - writeFileSync( - join(dir, 'mine', 'eval.json5'), - JSON.stringify({ - space: 'work', - expected: { 'opportunity-customer-voice': { 'Onboarding.md': 'fail', 'Build a Slack integration.md': 'fail' } }, - }), - ); - const config: Config = { spaces: [{ name: 'work', path: fixture, schema: 'strict_ost.json', sourceDir: dir }] }; - const [space] = loadEvalSpaces(resolveEvalSources(dir, []), config); afterAll(() => rmSync(dir, { recursive: true, force: true })); + const write = (path: string, value: unknown) => { + mkdirSync(join(dir, path, '..'), { recursive: true }); + writeFileSync(join(dir, path), JSON.stringify(value)); + }; + const opportunity = (title: string) => ({ + title, + type: 'opportunity', + status: 'identified', + parent: '[[Grow usage]]', + source: 'Interviews', + }); + write('fixtures/tree.json5', { + schema: 'strict_ost.json', + nodes: [ + { title: 'Grow usage', type: 'outcome', status: 'identified', metric: 'Weekly active teams' }, + opportunity('Onboarding'), + opportunity('Build a Slack integration'), + ], + }); + write('a.json5', { fixture: 'tree', expected: { 'opportunity-customer-voice': { Onboarding: 'fail' } } }); + write('b.json5', { + fixture: 'tree', + expected: { 'opportunity-customer-voice': { 'Grow usage': 'fail', 'Build a Slack integration': ['pass', 'fail'] } }, + }); + for (const title of ['Grow usage', 'Onboarding']) { + const type = + title === 'Grow usage' ? 'outcome\nmetric: Weekly active teams' : 'opportunity\nparent: "[[Grow usage]]"'; + mkdirSync(join(dir, 'space'), { recursive: true }); + writeFileSync( + join(dir, 'space', `${title}.md`), + `---\nstatus: identified\nsource: Interviews\ntype: ${type}\n---\n`, + ); + } + write('mine.json5', { space: 'work', expected: { 'opportunity-customer-voice': { 'Onboarding.md': 'fail' } } }); + const config: Config = { + spaces: [{ name: 'work', path: join(dir, 'space'), schema: 'strict_ost.json', sourceDir: dir }], + }; + const [a, b, mine] = loadEvals([resolveSuite(dir, [])], config); - test('loads the configured space, named by its source directory', () => { - expect(space).toMatchObject({ name: `${basename(dir)}/mine`, fixture: false }); - expect(space!.context.space.name).toBe('work'); + test('loads each eval file, sharing a fixture between evals', () => { + expect([a, b, mine].map((e) => e!.name)).toEqual(['a', 'b', 'mine'].map((n) => `${basename(dir)}/${n}`)); + expect(a!.context).toBe(b!.context); + expect(mine!.context.space.name).toBe('work'); }); - test('passes its checks without labelling every node', async () => { - expect(await checkEvalSpace(space!)).toEqual([]); + test('passes an eval that labels only some judged nodes', async () => { + expect(await checkEval(a!)).toEqual([]); + expect(await checkEval(mine!)).toEqual([]); + }); + + test('reports labels the rule never judges, and labels that accept every verdict', async () => { + expect(await checkEval(b!)).toEqual([ + `${b!.name}: rule "opportunity-customer-voice", node "Grow usage": not judged (unknown node, or the rule skips it)`, + `${b!.name}: rule "opportunity-customer-voice", node "Build a Slack integration": accepts every verdict`, + ]); }); test('sends only labelled nodes to the provider', async () => { const judge = new FakeJudgeProvider({ verdict: 'fail' }); - const judged = await judgeEvalSpace(space!, judge); - expect(judged.map((j) => j.nodeId).sort()).toEqual(['Build a Slack integration.md', 'Onboarding.md']); - expect(judge.callHistory.map((input) => input.node.id as string).sort()).toEqual( - judged.map((j) => j.nodeId).sort(), - ); + const judged = await judgeEval(a!, judge); + expect(judged.map((j) => j.nodeId)).toEqual(['Onboarding']); + expect(judge.callHistory.map((input) => input.node.id as string)).toEqual(['Onboarding']); }); }); -describe('judgeEvalSpace', () => { - const [space] = spaces; +describe('judgeEval', () => { + const [evaluation] = coreEvals; /** A judge that packs nodes, counting its calls of each kind. */ class PackingJudge extends FakeJudgeProvider { calls = { evaluate: 0, evaluateNodes: 0 }; - override evaluate(input: Parameters[0]) { + override evaluate(input: SemanticJudgeInput) { this.calls.evaluate++; return super.evaluate(input); } - async evaluateNodes(inputs: Parameters[0][]) { + async evaluateNodes(inputs: SemanticJudgeInput[]) { this.calls.evaluateNodes++; return Promise.all(inputs.map((input) => super.evaluate(input))); } @@ -81,48 +103,41 @@ describe('judgeEvalSpace', () => { test('passes packed inputs through, or one input per call with perNode', async () => { const packed = new PackingJudge(); const perNode = new PackingJudge(); - const a = await judgeEvalSpace(space!, packed); - const b = await judgeEvalSpace(space!, perNode, { perNode: true }); + const a = await judgeEval(evaluation!, packed); + const b = await judgeEval(evaluation!, perNode, { perNode: true }); expect(packed.calls).toEqual({ evaluate: 0, evaluateNodes: expect.any(Number) }); expect(perNode.calls).toEqual({ evaluate: b.length, evaluateNodes: 0 }); expect(a.length).toBe(b.length); }); }); -describe('scoreEvalSpace', () => { - const space = { - name: 'x', - expected: { r: { a: ['pass'], b: ['fail'], c: ['pass', 'fail'] } }, - } as unknown as EvalSpace; +describe('scoreEval', () => { + const evaluation = { expected: { r: { a: ['pass'], b: ['fail'], c: ['pass', 'fail'] } } }; const j = (nodeId: string, verdict: string, probabilities?: Record): Judgement => ({ nodeId, ruleId: 'r', result: { verdict, ...(probabilities && { probabilities }) }, }); - test('scores agreement, stability, probability, errors and unlabelled nodes over runs', () => { + test('scores agreement, stability, probability and errors over runs', () => { const runs = [ - [ - j('a', 'pass', { pass: 0.9, fail: 0.1 }), - j('b', 'pass', { pass: 0.6, fail: 0.4 }), - j('c', 'fail'), - j('d', 'pass'), - ], + [j('a', 'pass', { pass: 0.9, fail: 0.1 }), j('b', 'pass', { pass: 0.6, fail: 0.4 }), j('c', 'fail')], [j('a', 'pass', { pass: 0.7, fail: 0.3 }), j('b', 'fail', { pass: 0.2, fail: 0.8 })], ]; - const [score] = scoreEvalSpace(space, runs); - expect(score).toMatchObject({ cases: 3, runs: 2, stable: 1 / 3, errors: 1, reserved: 0, unlabelled: ['d'] }); + const [score] = scoreEval(evaluation, runs); + expect(score).toMatchObject({ cases: 3, runs: 2, stable: 1 / 3, errors: 1, reserved: 0 }); expect(score!.agreement).toBeCloseTo(4 / 6); expect(score!.expectedProbability).toBeCloseTo((0.9 + 0.4 + 0.7 + 0.8) / 4); expect(score!.misses).toEqual([ { nodeId: 'b', expected: ['fail'], got: ['pass', 'fail'] }, { nodeId: 'c', expected: ['pass', 'fail'], got: ['fail', 'not judged'] }, ]); + expect(score!.closest).toEqual([{ nodeId: 'a', probability: expect.closeTo(0.8) }]); }); test('counts reserved verdicts and errors', () => { const runs = [[j('a', 'missing-context'), { nodeId: 'b', ruleId: 'r', result: { error: 'boom' } }, j('c', 'pass')]]; - const [score] = scoreEvalSpace(space, runs); + const [score] = scoreEval(evaluation, runs); expect(score).toMatchObject({ reserved: 1, errors: 1, agreement: 1 / 3 }); }); }); From 6ca3221adb5b8e5e274bf938ae3a4187b40e23e2 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 03:19:47 +0000 Subject: [PATCH 7/7] refactor(evals): fixtures are spaces with parsed nodes A suite's fixtures.json5 lists fixtures as space configs (name, path to a JSONL nodes file, schema, plugins), so fixtures share schemas the way spaces do. A nodes file holds parsed nodes as `sctx dump` prints them, with their own ids and link targets; the harness no longer derives either from a title. Every eval names a space: a fixture or a configured space, labelled by node id the same way. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019XphpdrUMvqf8fuj8uG7JZ --- evals/README.md | 28 +++--- evals/load.ts | 87 +++++++++++-------- evals/semantic/fixtures.json5 | 6 ++ evals/semantic/fixtures/strict-ost.json5 | 26 ------ evals/semantic/fixtures/strict-ost.jsonl | 14 +++ .../semantic/opportunity-customer-voice.json5 | 24 ++--- tests/evals/semantic.test.ts | 66 +++++++------- 7 files changed, 131 insertions(+), 120 deletions(-) create mode 100644 evals/semantic/fixtures.json5 delete mode 100644 evals/semantic/fixtures/strict-ost.json5 create mode 100644 evals/semantic/fixtures/strict-ost.jsonl diff --git a/evals/README.md b/evals/README.md index 32dbe4c..a85bca1 100644 --- a/evals/README.md +++ b/evals/README.md @@ -1,6 +1,6 @@ # Semantic evals -Semantic evals measure how well model providers judge semantic rules. A case is a node, a rule, and the verdict the node should get. An eval holds the cases on one set of nodes: a fixture, or a configured space. +Semantic evals measure how well model providers judge semantic rules. A case is a node, a rule, and the verdict the node should get. An eval holds the cases on the nodes of one space. That space is a fixture, which keeps its nodes in the suite, or a configured space. ## Run the evals @@ -38,13 +38,13 @@ Each run validates through the same pipeline as `sctx validate --semantic`, with ## Suites -A suite is a directory of evals, with their fixtures in `fixtures/`: +A suite is a directory of evals and the fixtures they use: ``` evals/semantic/ - opportunity-customer-voice.json5 - fixtures/ - strict-ost.json5 + opportunity-customer-voice.json5 an eval + fixtures.json5 the fixtures, as space configs + fixtures/strict-ost.jsonl a fixture's nodes ``` | `--from` | Directory | Schema names resolve from | @@ -55,30 +55,30 @@ evals/semantic/ ## Add an eval -An eval is a `.json5` file in the suite. It names a `fixture` in the suite, or a configured `space`. `expected` maps a rule id to node ids and their verdicts: +An eval is a `.json5` file in the suite. It names a `space`: one of the suite's fixtures, or else a space in the config. `expected` maps a rule id to node ids and their verdicts: ```json5 { - fixture: 'strict-ost', + space: 'strict-ost', expected: { 'opportunity-customer-voice': { - 'Build a Slack integration': 'fail', + 'Build a Slack integration.md': 'fail', // A solution in disguise. - 'Add a progress bar to workspace setup': 'fail', + 'Add a progress bar to workspace setup.md': 'fail', }, }, } ``` -A list of verdicts accepts any of them. On a configured space, node ids are file paths: `{ space: 'work', expected: { 'opportunity-customer-voice': { 'Onboarding.md': 'fail' } } }`. - -Write a comment on a case whose label is not obvious. We recommend one eval per rule, so each rule's cases are easy to find and to run alone. +A list of verdicts accepts any of them. Write a comment on a case whose label is not obvious. We recommend one eval per rule, so each rule's cases are easy to find and to run alone. ## Add a fixture -A fixture is `fixtures/.json5`: a `schema`, optional `plugins` as in a space config, and `nodes`. Each node holds the fields its frontmatter would, with a body in `content`. Its `title` is its node id and link target. `sctx dump | jq '.nodes | map(.schemaData)'` prints notes in this shape. +A fixture is a space whose nodes are in a JSONL file in the suite. `fixtures.json5` lists the fixtures as a config lists spaces: a `name`, the `path` to the nodes file, a `schema`, and optional `plugins`. Several fixtures can share a schema, and several evals can share a fixture. + +The nodes file holds one parsed node per line, as `sctx dump` prints them. To make one from markdown notes, configure them as a space and run `sctx dump | jq -c '.nodes[]'`. -Several evals can share a fixture. For a rule whose `scope` sends related nodes, such as siblings, give each scenario its own small tree, so that one case's related nodes do not change another's. +For a rule whose `scope` sends related nodes, such as siblings, give each scenario its own small tree, so that one case's related nodes do not change another's. ## Checks diff --git a/evals/load.ts b/evals/load.ts index 1993815..4af82c2 100644 --- a/evals/load.ts +++ b/evals/load.ts @@ -6,11 +6,14 @@ import { resolvePluginDir } from '../src/plugins/util'; import { readSpace } from '../src/read/read-space'; import { resolveGraphEdges } from '../src/read/resolve-graph-edges'; import { createSpaceContext } from '../src/space-context'; -import type { BaseNode, NodeId, SpaceContext } from '../src/types'; +import type { BaseNode, SpaceContext } from '../src/types'; /** Where core and each plugin keep their semantic suite. */ export const SUITE_SUBDIR = join('evals', 'semantic'); +/** The suite's fixtures: spaces whose nodes are in a JSONL file. */ +const FIXTURES_FILE = 'fixtures.json5'; + const CORE_SUITE_DIR = join(import.meta.dir, 'semantic'); /** A directory of evals and their fixtures, and the directory schema names resolve from. */ @@ -23,7 +26,7 @@ export interface Suite { /** Rule id → node id → the verdicts that count as agreeing. */ export type Expected = Record>; -/** Labelled cases on the nodes of one fixture or configured space. */ +/** Labelled cases on the nodes of one space: a fixture, or a configured space. */ export interface Eval { /** `/`, e.g. `core/opportunity-customer-voice`. */ name: string; @@ -50,34 +53,30 @@ export function resolveSuite(source: string, configDirs: string[]): Suite { throw new Error(`Suite "${source}" is not core, a path, or a plugin found from the config.`); } -type EvalFile = { fixture?: string; space?: string; expected: Record> }; -type FixtureFile = Pick & { nodes: Record[] }; +type EvalFile = { space: string; expected: Record> }; +type FixturesFile = { spaces: Pick[] }; +type Space = Pick; -/** Every `*.json5` at the top of each suite is an eval. `config` holds the spaces evals name. */ +/** + * Every `*.json5` at the top of each suite, except `fixtures.json5`, is an eval. An eval's `space` names one of + * the suite's fixtures, or else a space in `config`. + */ export function loadEvals(suites: Suite[], config: Config): Eval[] { return suites.flatMap((suite) => { - const fixtures = new Map>(); - const fixture = (name: string) => { - if (!fixtures.has(name)) fixtures.set(name, loadFixture(suite, name)); - return fixtures.get(name)!; - }; + const fixtures = loadFixtures(suite); return readdirSync(suite.dir) - .filter((file) => file.endsWith('.json5')) + .filter((file) => file.endsWith('.json5') && file !== FIXTURES_FILE) .sort() .map((file) => { const name = `${suite.name}/${basename(file, '.json5')}`; try { - const evalFile = JSON5.parse(readFileSync(join(suite.dir, file), 'utf8')) as EvalFile; - if ((evalFile.fixture === undefined) === (evalFile.space === undefined)) { - throw new Error('needs either "fixture" or "space"'); - } + const { space, expected: labels } = JSON5.parse(readFileSync(join(suite.dir, file), 'utf8')) as EvalFile; + if (typeof space !== 'string') throw new Error('needs a "space"'); const expected: Expected = {}; - for (const [ruleId, labels] of Object.entries(evalFile.expected)) { - expected[ruleId] = Object.fromEntries(Object.entries(labels).map(([id, v]) => [id, [v].flat()])); + for (const [ruleId, verdicts] of Object.entries(labels)) { + expected[ruleId] = Object.fromEntries(Object.entries(verdicts).map(([id, v]) => [id, [v].flat()])); } - if (evalFile.fixture !== undefined) return { name, expected, ...fixture(evalFile.fixture) }; - const context = createSpaceContext(evalFile.space!, config); - return { name, expected, context, read: () => readSpace(context) }; + return { name, expected, ...(fixtures.get(space) ?? configuredSpace(space, config)) }; } catch (err) { throw new Error(`Eval ${name}: ${err instanceof Error ? err.message : String(err)}`); } @@ -85,23 +84,35 @@ export function loadEvals(suites: Suite[], config: Config): Eval[] { }); } -/** `fixtures/.json5`: a schema and the nodes' schema data, each with its `title` as node id and link target. */ -function loadFixture(suite: Suite, name: string): Pick { - const path = join(suite.dir, 'fixtures', `${name}.json5`); - if (!existsSync(path)) throw new Error(`no fixture at ${path}`); - const file = JSON5.parse(readFileSync(path, 'utf8')) as FixtureFile; - const spaceName = `${suite.name}/fixtures/${name}`; - const context = createSpaceContext( - spaceName, - { spaces: [{ name: spaceName, path: suite.dir, schema: file.schema, plugins: file.plugins }] }, - { configDir: suite.configDir }, +function configuredSpace(name: string, config: Config): Space { + const context = createSpaceContext(name, config); + return { context, read: () => readSpace(context) }; +} + +/** A fixture's nodes file holds one parsed node per line, as `sctx dump` prints them. Resolved fields are dropped, since resolution runs again. */ +function loadFixtures(suite: Suite): Map { + const path = join(suite.dir, FIXTURES_FILE); + if (!existsSync(path)) return new Map(); + const { spaces } = JSON5.parse(readFileSync(path, 'utf8')) as FixturesFile; + const config: Config = { spaces: spaces.map((s) => ({ ...s, path: resolve(suite.dir, s.path) })) }; + return new Map( + config.spaces.map(({ name, path: nodesPath }) => { + const context = createSpaceContext(name, config, { configDir: suite.configDir }); + const nodes = readNodes(nodesPath); + return [name, { context, read: async () => resolveGraphEdges(nodes, context.schema.metadata) }]; + }), ); - const nodes: BaseNode[] = file.nodes.map((schemaData, i) => { - const { title, type } = schemaData; - if (typeof title !== 'string' || typeof type !== 'string') { - throw new Error(`${path}: node ${i + 1} needs a "title" and a "type"`); - } - return { id: title as NodeId, title, type, linkTargets: [title], schemaData }; - }); - return { context, read: async () => resolveGraphEdges(nodes, context.schema.metadata) }; +} + +function readNodes(path: string): BaseNode[] { + return readFileSync(path, 'utf8') + .split('\n') + .flatMap((line, i) => { + if (line.trim() === '') return []; + const { id, title, type, schemaData, linkTargets, content, contentLinks } = JSON.parse(line) as BaseNode; + if (!id || typeof title !== 'string' || typeof type !== 'string' || !schemaData || !linkTargets) { + throw new Error(`${path}:${i + 1}: not a parsed node (needs id, title, type, schemaData and linkTargets)`); + } + return [{ id, title, type, schemaData, linkTargets, content, contentLinks }]; + }); } diff --git a/evals/semantic/fixtures.json5 b/evals/semantic/fixtures.json5 new file mode 100644 index 0000000..5f17d2a --- /dev/null +++ b/evals/semantic/fixtures.json5 @@ -0,0 +1,6 @@ +{ + spaces: [ + // A small product's opportunity solution tree. + { name: 'strict-ost', path: 'fixtures/strict-ost.jsonl', schema: 'strict_ost.json' }, + ], +} diff --git a/evals/semantic/fixtures/strict-ost.json5 b/evals/semantic/fixtures/strict-ost.json5 deleted file mode 100644 index 1d7a0a8..0000000 --- a/evals/semantic/fixtures/strict-ost.json5 +++ /dev/null @@ -1,26 +0,0 @@ -// A small product's opportunity solution tree. Each node holds the fields its frontmatter would. -{ - schema: 'strict_ost.json', - nodes: [ - { title: 'Increase weekly active teams', type: 'outcome', status: 'identified', metric: 'Increase % of trial teams active in their second week', content: 'Teams that use the product every week renew.' }, - - // Customer needs, in the customer's words. - { title: 'Setting up a workspace takes so long that I give up', type: 'opportunity', status: 'identified', parent: '[[Increase weekly active teams]]', source: 'Interview with Priya, 2026-09-02', content: 'Priya abandoned setup twice before her team lead finished it for her.' }, - { title: "I can't tell which plan fits a team of my size", type: 'opportunity', status: 'identified', parent: '[[Increase weekly active teams]]', source: 'Sales call notes, 2026-09-10', content: 'Prospects asked which tier covers twelve seats with guest access.' }, - { title: 'I want to see what my teammates changed while I was away', type: 'opportunity', status: 'identified', parent: '[[Increase weekly active teams]]', source: 'Interview with Tomas, 2026-09-15', content: 'Tomas reads every page again after a holiday to find what is new.' }, - { title: "I'm afraid I'll lose my work if my browser crashes", type: 'opportunity', status: 'identified', parent: '[[Increase weekly active teams]]', source: 'Support tickets, September 2026', content: 'Several tickets describe retyping long notes after a crash.' }, - { title: "I don't know whether my changes have been saved", type: 'opportunity', status: 'identified', parent: "[[I'm afraid I'll lose my work if my browser crashes]]", source: 'Usability test round 3', content: 'Four of six participants pressed the save shortcut repeatedly.' }, - { title: 'Show unsaved-change indicator', type: 'solution', status: 'exploring', parent: "[[I don't know whether my changes have been saved]]", content: 'A dot on the tab while changes are pending.' }, - - // Solutions, metrics and topics posing as opportunities. - { title: 'Add a progress bar to workspace setup', type: 'opportunity', status: 'identified', parent: '[[Increase weekly active teams]]', source: 'Product team brainstorm', content: 'Show users how many setup steps remain.' }, - { title: 'Build a Slack integration', type: 'opportunity', status: 'identified', parent: '[[Increase weekly active teams]]', source: 'Competitor review', content: 'Post page updates to a Slack channel.' }, - { title: 'Offer a discount on annual plans', type: 'opportunity', status: 'identified', parent: '[[Increase weekly active teams]]', source: 'Pricing workshop', content: 'Ten percent off when paying yearly.' }, - { title: 'Increase trial-to-paid conversion by 10%', type: 'opportunity', status: 'identified', parent: '[[Increase weekly active teams]]', source: 'Quarterly planning', content: 'The board wants conversion to rise this quarter.' }, - { title: 'Reduce churn among small teams', type: 'opportunity', status: 'identified', parent: '[[Increase weekly active teams]]', source: 'Retention dashboard', content: 'Teams of fewer than five seats churn twice as often.' }, - { title: 'Onboarding', type: 'opportunity', status: 'identified', parent: '[[Increase weekly active teams]]', source: 'Interview synthesis', content: 'Everything about how new users get started.' }, - - // A real pain point, but reported in the third person. - { title: 'New users find the editor toolbar confusing', type: 'opportunity', status: 'identified', parent: '[[Increase weekly active teams]]', source: 'Usability test round 2', content: 'Participants hovered over icons without knowing what they did.' }, - ], -} diff --git a/evals/semantic/fixtures/strict-ost.jsonl b/evals/semantic/fixtures/strict-ost.jsonl new file mode 100644 index 0000000..18d7996 --- /dev/null +++ b/evals/semantic/fixtures/strict-ost.jsonl @@ -0,0 +1,14 @@ +{"id":"Increase weekly active teams.md","title":"Increase weekly active teams","type":"outcome","schemaData":{"title":"Increase weekly active teams","type":"outcome","status":"identified","metric":"Increase % of trial teams active in their second week","content":"Teams that use the product every week renew."},"linkTargets":["Increase weekly active teams"]} +{"id":"Setting up a workspace takes so long that I give up.md","title":"Setting up a workspace takes so long that I give up","type":"opportunity","schemaData":{"title":"Setting up a workspace takes so long that I give up","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Interview with Priya, 2026-09-02","content":"Priya abandoned setup twice before her team lead finished it for her."},"linkTargets":["Setting up a workspace takes so long that I give up"]} +{"id":"I can't tell which plan fits a team of my size.md","title":"I can't tell which plan fits a team of my size","type":"opportunity","schemaData":{"title":"I can't tell which plan fits a team of my size","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Sales call notes, 2026-09-10","content":"Prospects asked which tier covers twelve seats with guest access."},"linkTargets":["I can't tell which plan fits a team of my size"]} +{"id":"I want to see what my teammates changed while I was away.md","title":"I want to see what my teammates changed while I was away","type":"opportunity","schemaData":{"title":"I want to see what my teammates changed while I was away","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Interview with Tomas, 2026-09-15","content":"Tomas reads every page again after a holiday to find what is new."},"linkTargets":["I want to see what my teammates changed while I was away"]} +{"id":"I'm afraid I'll lose my work if my browser crashes.md","title":"I'm afraid I'll lose my work if my browser crashes","type":"opportunity","schemaData":{"title":"I'm afraid I'll lose my work if my browser crashes","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Support tickets, September 2026","content":"Several tickets describe retyping long notes after a crash."},"linkTargets":["I'm afraid I'll lose my work if my browser crashes"]} +{"id":"I don't know whether my changes have been saved.md","title":"I don't know whether my changes have been saved","type":"opportunity","schemaData":{"title":"I don't know whether my changes have been saved","type":"opportunity","status":"identified","parent":"[[I'm afraid I'll lose my work if my browser crashes]]","source":"Usability test round 3","content":"Four of six participants pressed the save shortcut repeatedly."},"linkTargets":["I don't know whether my changes have been saved"]} +{"id":"Show unsaved-change indicator.md","title":"Show unsaved-change indicator","type":"solution","schemaData":{"title":"Show unsaved-change indicator","type":"solution","status":"exploring","parent":"[[I don't know whether my changes have been saved]]","content":"A dot on the tab while changes are pending."},"linkTargets":["Show unsaved-change indicator"]} +{"id":"Add a progress bar to workspace setup.md","title":"Add a progress bar to workspace setup","type":"opportunity","schemaData":{"title":"Add a progress bar to workspace setup","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Product team brainstorm","content":"Show users how many setup steps remain."},"linkTargets":["Add a progress bar to workspace setup"]} +{"id":"Build a Slack integration.md","title":"Build a Slack integration","type":"opportunity","schemaData":{"title":"Build a Slack integration","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Competitor review","content":"Post page updates to a Slack channel."},"linkTargets":["Build a Slack integration"]} +{"id":"Offer a discount on annual plans.md","title":"Offer a discount on annual plans","type":"opportunity","schemaData":{"title":"Offer a discount on annual plans","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Pricing workshop","content":"Ten percent off when paying yearly."},"linkTargets":["Offer a discount on annual plans"]} +{"id":"Increase trial-to-paid conversion by 10%.md","title":"Increase trial-to-paid conversion by 10%","type":"opportunity","schemaData":{"title":"Increase trial-to-paid conversion by 10%","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Quarterly planning","content":"The board wants conversion to rise this quarter."},"linkTargets":["Increase trial-to-paid conversion by 10%"]} +{"id":"Reduce churn among small teams.md","title":"Reduce churn among small teams","type":"opportunity","schemaData":{"title":"Reduce churn among small teams","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Retention dashboard","content":"Teams of fewer than five seats churn twice as often."},"linkTargets":["Reduce churn among small teams"]} +{"id":"Onboarding.md","title":"Onboarding","type":"opportunity","schemaData":{"title":"Onboarding","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Interview synthesis","content":"Everything about how new users get started."},"linkTargets":["Onboarding"]} +{"id":"New users find the editor toolbar confusing.md","title":"New users find the editor toolbar confusing","type":"opportunity","schemaData":{"title":"New users find the editor toolbar confusing","type":"opportunity","status":"identified","parent":"[[Increase weekly active teams]]","source":"Usability test round 2","content":"Participants hovered over icons without knowing what they did."},"linkTargets":["New users find the editor toolbar confusing"]} diff --git a/evals/semantic/opportunity-customer-voice.json5 b/evals/semantic/opportunity-customer-voice.json5 index c6394eb..9291342 100644 --- a/evals/semantic/opportunity-customer-voice.json5 +++ b/evals/semantic/opportunity-customer-voice.json5 @@ -1,21 +1,21 @@ { - fixture: 'strict-ost', + space: 'strict-ost', expected: { 'opportunity-customer-voice': { - 'Setting up a workspace takes so long that I give up': 'pass', - "I can't tell which plan fits a team of my size": 'pass', - "I'm afraid I'll lose my work if my browser crashes": 'pass', - "I don't know whether my changes have been saved": 'pass', - 'I want to see what my teammates changed while I was away': 'pass', + 'Setting up a workspace takes so long that I give up.md': 'pass', + "I can't tell which plan fits a team of my size.md": 'pass', + "I'm afraid I'll lose my work if my browser crashes.md": 'pass', + "I don't know whether my changes have been saved.md": 'pass', + 'I want to see what my teammates changed while I was away.md': 'pass', // A solution in disguise. - 'Add a progress bar to workspace setup': 'fail', - 'Build a Slack integration': 'fail', - 'Offer a discount on annual plans': 'fail', + 'Add a progress bar to workspace setup.md': 'fail', + 'Build a Slack integration.md': 'fail', + 'Offer a discount on annual plans.md': 'fail', // A business metric. - 'Increase trial-to-paid conversion by 10%': 'fail', - 'Reduce churn among small teams': 'fail', + 'Increase trial-to-paid conversion by 10%.md': 'fail', + 'Reduce churn among small teams.md': 'fail', // A topic, not a need. - 'Onboarding': 'fail', + 'Onboarding.md': 'fail', }, }, } diff --git a/tests/evals/semantic.test.ts b/tests/evals/semantic.test.ts index ef9cee4..ccdec5b 100644 --- a/tests/evals/semantic.test.ts +++ b/tests/evals/semantic.test.ts @@ -24,42 +24,48 @@ describe('a suite directory', () => { mkdirSync(join(dir, path, '..'), { recursive: true }); writeFileSync(join(dir, path), JSON.stringify(value)); }; - const opportunity = (title: string) => ({ - title, - type: 'opportunity', - status: 'identified', - parent: '[[Grow usage]]', - source: 'Interviews', - }); - write('fixtures/tree.json5', { - schema: 'strict_ost.json', - nodes: [ - { title: 'Grow usage', type: 'outcome', status: 'identified', metric: 'Weekly active teams' }, + const node = (title: string, type: string, fields: Record) => + JSON.stringify({ + id: `${title}.md`, + title, + type, + linkTargets: [title], + schemaData: { title, type, status: 'identified', ...fields }, + }); + const opportunity = (title: string) => node(title, 'opportunity', { parent: '[[Grow usage]]', source: 'Interviews' }); + mkdirSync(join(dir, 'nodes')); + writeFileSync( + join(dir, 'nodes', 'tree.jsonl'), + [ + node('Grow usage', 'outcome', { metric: 'Weekly active teams' }), opportunity('Onboarding'), opportunity('Build a Slack integration'), - ], - }); - write('a.json5', { fixture: 'tree', expected: { 'opportunity-customer-voice': { Onboarding: 'fail' } } }); + ].join('\n'), + ); + write('fixtures.json5', { spaces: [{ name: 'tree', path: 'nodes/tree.jsonl', schema: 'strict_ost.json' }] }); + write('a.json5', { space: 'tree', expected: { 'opportunity-customer-voice': { 'Onboarding.md': 'fail' } } }); write('b.json5', { - fixture: 'tree', - expected: { 'opportunity-customer-voice': { 'Grow usage': 'fail', 'Build a Slack integration': ['pass', 'fail'] } }, + space: 'tree', + expected: { + 'opportunity-customer-voice': { 'Grow usage.md': 'fail', 'Build a Slack integration.md': ['pass', 'fail'] }, + }, }); - for (const title of ['Grow usage', 'Onboarding']) { - const type = - title === 'Grow usage' ? 'outcome\nmetric: Weekly active teams' : 'opportunity\nparent: "[[Grow usage]]"'; - mkdirSync(join(dir, 'space'), { recursive: true }); - writeFileSync( - join(dir, 'space', `${title}.md`), - `---\nstatus: identified\nsource: Interviews\ntype: ${type}\n---\n`, - ); - } + mkdirSync(join(dir, 'space')); + writeFileSync( + join(dir, 'space', 'Grow usage.md'), + '---\nstatus: identified\ntype: outcome\nmetric: Weekly active teams\n---\n', + ); + writeFileSync( + join(dir, 'space', 'Onboarding.md'), + '---\nstatus: identified\ntype: opportunity\nparent: "[[Grow usage]]"\nsource: Interviews\n---\n', + ); write('mine.json5', { space: 'work', expected: { 'opportunity-customer-voice': { 'Onboarding.md': 'fail' } } }); const config: Config = { spaces: [{ name: 'work', path: join(dir, 'space'), schema: 'strict_ost.json', sourceDir: dir }], }; const [a, b, mine] = loadEvals([resolveSuite(dir, [])], config); - test('loads each eval file, sharing a fixture between evals', () => { + test('loads each eval file, with evals sharing a fixture', () => { expect([a, b, mine].map((e) => e!.name)).toEqual(['a', 'b', 'mine'].map((n) => `${basename(dir)}/${n}`)); expect(a!.context).toBe(b!.context); expect(mine!.context.space.name).toBe('work'); @@ -72,16 +78,16 @@ describe('a suite directory', () => { test('reports labels the rule never judges, and labels that accept every verdict', async () => { expect(await checkEval(b!)).toEqual([ - `${b!.name}: rule "opportunity-customer-voice", node "Grow usage": not judged (unknown node, or the rule skips it)`, - `${b!.name}: rule "opportunity-customer-voice", node "Build a Slack integration": accepts every verdict`, + `${b!.name}: rule "opportunity-customer-voice", node "Grow usage.md": not judged (unknown node, or the rule skips it)`, + `${b!.name}: rule "opportunity-customer-voice", node "Build a Slack integration.md": accepts every verdict`, ]); }); test('sends only labelled nodes to the provider', async () => { const judge = new FakeJudgeProvider({ verdict: 'fail' }); const judged = await judgeEval(a!, judge); - expect(judged.map((j) => j.nodeId)).toEqual(['Onboarding']); - expect(judge.callHistory.map((input) => input.node.id as string)).toEqual(['Onboarding']); + expect(judged.map((j) => j.nodeId)).toEqual(['Onboarding.md']); + expect(judge.callHistory.map((input) => input.node.id as string)).toEqual(['Onboarding.md']); }); });